org.apache.hadoop.hbase.regionserver.TestHRegion.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hbase.regionserver.TestHRegion.java

Source

/**
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.regionserver;

import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;

import java.io.IOException;
import java.io.InterruptedIOException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
import org.apache.hadoop.hbase.DroppedSnapshotException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.MultithreadedTestUtil;
import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterBase;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.NullComparator;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
import org.apache.hadoop.hbase.regionserver.HRegion.RowLock;
import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
import org.apache.hadoop.hbase.regionserver.wal.FaultyHLog;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.test.MetricsAssertHelper;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
import org.apache.hadoop.hbase.util.PairOfSameType;
import org.apache.hadoop.hbase.util.Threads;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.mockito.Mockito;

import com.google.common.collect.Lists;
import com.google.protobuf.ByteString;

/**
 * Basic stand-alone testing of HRegion.  No clusters!
 *
 * A lot of the meta information for an HRegion now lives inside other HRegions
 * or in the HBaseMaster, so only basic testing is possible.
 */
@Category(MediumTests.class)
@SuppressWarnings("deprecation")
public class TestHRegion {
    // Do not spin up clusters in here. If you need to spin up a cluster, do it
    // over in TestHRegionOnCluster.
    static final Log LOG = LogFactory.getLog(TestHRegion.class);
    @Rule
    public TestName name = new TestName();

    private static final String COLUMN_FAMILY = "MyCF";
    private static final byte[] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);

    HRegion region = null;
    // Do not run unit tests in parallel (? Why not?  It don't work?  Why not?  St.Ack)
    private static HBaseTestingUtility TEST_UTIL;
    public static Configuration CONF;
    private String dir;
    private static FileSystem FILESYSTEM;
    private final int MAX_VERSIONS = 2;

    // Test names
    protected byte[] tableName;
    protected String method;
    protected final byte[] qual1 = Bytes.toBytes("qual1");
    protected final byte[] qual2 = Bytes.toBytes("qual2");
    protected final byte[] qual3 = Bytes.toBytes("qual3");
    protected final byte[] value1 = Bytes.toBytes("value1");
    protected final byte[] value2 = Bytes.toBytes("value2");
    protected final byte[] row = Bytes.toBytes("rowA");
    protected final byte[] row2 = Bytes.toBytes("rowB");

    protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
            .getInstance(MetricsAssertHelper.class);

    @Before
    public void setup() throws IOException {
        TEST_UTIL = HBaseTestingUtility.createLocalHTU();
        FILESYSTEM = TEST_UTIL.getTestFileSystem();
        CONF = TEST_UTIL.getConfiguration();
        dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
        method = name.getMethodName();
        tableName = Bytes.toBytes(name.getMethodName());
    }

    @After
    public void tearDown() throws Exception {
        EnvironmentEdgeManagerTestHelper.reset();
        LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
        TEST_UTIL.cleanupTestDir();
    }

    String getName() {
        return name.getMethodName();
    }

    /**
     * Test for Bug 2 of HBASE-10466.
     * "Bug 2: Conditions for the first flush of region close (so-called pre-flush) If memstoreSize
     * is smaller than a certain value, or when region close starts a flush is ongoing, the first
     * flush is skipped and only the second flush takes place. However, two flushes are required in
     * case previous flush fails and leaves some data in snapshot. The bug could cause loss of data
     * in current memstore. The fix is removing all conditions except abort check so we ensure 2
     * flushes for region close."
     * @throws IOException
     */
    @Test(timeout = 60000)
    public void testCloseCarryingSnapshot() throws IOException {
        HRegion region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
        Store store = region.getStore(COLUMN_FAMILY_BYTES);
        // Get some random bytes.
        byte[] value = Bytes.toBytes(name.getMethodName());
        // Make a random put against our cf.
        Put put = new Put(value);
        put.add(COLUMN_FAMILY_BYTES, null, value);
        // First put something in current memstore, which will be in snapshot after flusher.prepare()
        region.put(put);
        StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
        storeFlushCtx.prepare();
        // Second put something in current memstore
        put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
        region.put(put);
        // Close with something in memstore and something in the snapshot.  Make sure all is cleared.
        region.close();
        assertEquals(0, region.getMemstoreSize().get());
        HRegion.closeHRegion(region);
    }

    /*
     * This test is for verifying memstore snapshot size is correctly updated in case of rollback
     * See HBASE-10845
     */
    @Test(timeout = 60000)
    public void testMemstoreSnapshotSize() throws IOException {
        class MyFaultyHLog extends FaultyHLog {
            StoreFlushContext storeFlushCtx;

            public MyFaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
                    throws IOException {
                super(fs, rootDir, logName, conf);
            }

            void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
                this.storeFlushCtx = storeFlushCtx;
            }

            @Override
            public void sync(long txid) throws IOException {
                storeFlushCtx.prepare();
                super.sync(txid);
            }
        }

        FileSystem fs = FileSystem.get(CONF);
        Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
        MyFaultyHLog faultyLog = new MyFaultyHLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
        HRegion region = initHRegion(tableName, null, null, name.getMethodName(), CONF, false, Durability.SYNC_WAL,
                faultyLog, COLUMN_FAMILY_BYTES);

        Store store = region.getStore(COLUMN_FAMILY_BYTES);
        // Get some random bytes.
        byte[] value = Bytes.toBytes(name.getMethodName());
        faultyLog.setStoreFlushCtx(store.createFlushContext(12345));

        Put put = new Put(value);
        put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
        faultyLog.setFailureType(FaultyHLog.FailureType.SYNC);

        boolean threwIOE = false;
        try {
            region.put(put);
        } catch (IOException ioe) {
            threwIOE = true;
        } finally {
            assertTrue("The regionserver should have thrown an exception", threwIOE);
        }
        long sz = store.getFlushableSize();
        assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
        HRegion.closeHRegion(region);
    }

    /**
     * Test we do not lose data if we fail a flush and then close.
     * Part of HBase-10466.  Tests the following from the issue description:
     * "Bug 1: Wrong calculation of HRegion.memstoreSize: When a flush fails, data to be flushed is
     * kept in each MemStore's snapshot and wait for next flush attempt to continue on it. But when
     * the next flush succeeds, the counter of total memstore size in HRegion is always deduced by
     * the sum of current memstore sizes instead of snapshots left from previous failed flush. This
     * calculation is problematic that almost every time there is failed flush, HRegion.memstoreSize
     * gets reduced by a wrong value. If region flush could not proceed for a couple cycles, the size
     * in current memstore could be much larger than the snapshot. It's likely to drift memstoreSize
     * much smaller than expected. In extreme case, if the error accumulates to even bigger than
     * HRegion's memstore size limit, any further flush is skipped because flush does not do anything
     * if memstoreSize is not larger than 0."
     * @throws Exception
     */
    @Test(timeout = 60000)
    public void testFlushSizeAccounting() throws Exception {
        final Configuration conf = HBaseConfiguration.create(CONF);
        // Only retry once.
        conf.setInt("hbase.hstore.flush.retries.number", 1);
        final User user = User.createUserForTesting(conf, this.name.getMethodName(), new String[] { "foo" });
        // Inject our faulty LocalFileSystem
        conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
        user.runAs(new PrivilegedExceptionAction<Object>() {
            @Override
            public Object run() throws Exception {
                // Make sure it worked (above is sensitive to caching details in hadoop core)
                FileSystem fs = FileSystem.get(conf);
                Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
                FaultyFileSystem ffs = (FaultyFileSystem) fs;
                HRegion region = null;
                try {
                    // Initialize region
                    region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
                    long size = region.getMemstoreSize().get();
                    Assert.assertEquals(0, size);
                    // Put one item into memstore.  Measure the size of one item in memstore.
                    Put p1 = new Put(row);
                    p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null));
                    region.put(p1);
                    final long sizeOfOnePut = region.getMemstoreSize().get();
                    // Fail a flush which means the current memstore will hang out as memstore 'snapshot'.
                    try {
                        LOG.info("Flushing");
                        region.flushcache();
                        Assert.fail("Didn't bubble up IOE!");
                    } catch (DroppedSnapshotException dse) {
                        // What we are expecting
                    }
                    // Make it so all writes succeed from here on out
                    ffs.fault.set(false);
                    // Check sizes.  Should still be the one entry.
                    Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize().get());
                    // Now add two entries so that on this next flush that fails, we can see if we
                    // subtract the right amount, the snapshot size only.
                    Put p2 = new Put(row);
                    p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null));
                    p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null));
                    region.put(p2);
                    Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize().get());
                    // Do a successful flush.  It will clear the snapshot only.  Thats how flushes work.
                    // If already a snapshot, we clear it else we move the memstore to be snapshot and flush
                    // it
                    region.flushcache();
                    // Make sure our memory accounting is right.
                    Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize().get());
                } finally {
                    HRegion.closeHRegion(region);
                }
                return null;
            }
        });
        FileSystem.closeAllForUGI(user.getUGI());
    }

    @Test(timeout = 60000)
    public void testCloseWithFailingFlush() throws Exception {
        final Configuration conf = HBaseConfiguration.create(CONF);
        // Only retry once.
        conf.setInt("hbase.hstore.flush.retries.number", 1);
        final User user = User.createUserForTesting(conf, this.name.getMethodName(), new String[] { "foo" });
        // Inject our faulty LocalFileSystem
        conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
        user.runAs(new PrivilegedExceptionAction<Object>() {
            @Override
            public Object run() throws Exception {
                // Make sure it worked (above is sensitive to caching details in hadoop core)
                FileSystem fs = FileSystem.get(conf);
                Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
                FaultyFileSystem ffs = (FaultyFileSystem) fs;
                HRegion region = null;
                try {
                    // Initialize region
                    region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
                    long size = region.getMemstoreSize().get();
                    Assert.assertEquals(0, size);
                    // Put one item into memstore.  Measure the size of one item in memstore.
                    Put p1 = new Put(row);
                    p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null));
                    region.put(p1);
                    // Manufacture an outstanding snapshot -- fake a failed flush by doing prepare step only.
                    Store store = region.getStore(COLUMN_FAMILY_BYTES);
                    StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
                    storeFlushCtx.prepare();
                    // Now add two entries to the foreground memstore.
                    Put p2 = new Put(row);
                    p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null));
                    p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null));
                    region.put(p2);
                    // Now try close on top of a failing flush.
                    region.close();
                    fail();
                } catch (DroppedSnapshotException dse) {
                    // Expected
                    LOG.info("Expected DroppedSnapshotException");
                } finally {
                    // Make it so all writes succeed from here on out so can close clean
                    ffs.fault.set(false);
                    HRegion.closeHRegion(region);
                }
                return null;
            }
        });
        FileSystem.closeAllForUGI(user.getUGI());
    }

    @Test
    public void testCompactionAffectedByScanners() throws Exception {
        byte[] family = Bytes.toBytes("family");
        this.region = initHRegion(tableName, method, CONF, family);

        Put put = new Put(Bytes.toBytes("r1"));
        put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
        region.put(put);
        region.flushcache();

        Scan scan = new Scan();
        scan.setMaxVersions(3);
        // open the first scanner
        RegionScanner scanner1 = region.getScanner(scan);

        Delete delete = new Delete(Bytes.toBytes("r1"));
        region.delete(delete);
        region.flushcache();

        // open the second scanner
        RegionScanner scanner2 = region.getScanner(scan);

        List<Cell> results = new ArrayList<Cell>();

        System.out.println("Smallest read point:" + region.getSmallestReadPoint());

        // make a major compaction
        region.compactStores(true);

        // open the third scanner
        RegionScanner scanner3 = region.getScanner(scan);

        // get data from scanner 1, 2, 3 after major compaction
        scanner1.next(results);
        System.out.println(results);
        assertEquals(1, results.size());

        results.clear();
        scanner2.next(results);
        System.out.println(results);
        assertEquals(0, results.size());

        results.clear();
        scanner3.next(results);
        System.out.println(results);
        assertEquals(0, results.size());
    }

    @Test
    public void testToShowNPEOnRegionScannerReseek() throws Exception {
        byte[] family = Bytes.toBytes("family");
        this.region = initHRegion(tableName, method, CONF, family);

        Put put = new Put(Bytes.toBytes("r1"));
        put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
        region.put(put);
        put = new Put(Bytes.toBytes("r2"));
        put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
        region.put(put);
        region.flushcache();

        Scan scan = new Scan();
        scan.setMaxVersions(3);
        // open the first scanner
        RegionScanner scanner1 = region.getScanner(scan);

        System.out.println("Smallest read point:" + region.getSmallestReadPoint());

        region.compactStores(true);

        scanner1.reseek(Bytes.toBytes("r2"));
        List<Cell> results = new ArrayList<Cell>();
        scanner1.next(results);
        Cell keyValue = results.get(0);
        Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
        scanner1.close();
    }

    @Test
    public void testSkipRecoveredEditsReplay() throws Exception {
        String method = "testSkipRecoveredEditsReplay";
        TableName tableName = TableName.valueOf(method);
        byte[] family = Bytes.toBytes("family");
        this.region = initHRegion(tableName, method, CONF, family);
        try {
            Path regiondir = region.getRegionFileSystem().getRegionDir();
            FileSystem fs = region.getRegionFileSystem().getFileSystem();
            byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();

            Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);

            long maxSeqId = 1050;
            long minSeqId = 1000;

            for (long i = minSeqId; i <= maxSeqId; i += 10) {
                Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
                fs.create(recoveredEdits);
                HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);

                long time = System.nanoTime();
                WALEdit edit = new WALEdit();
                edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
                writer.append(new HLog.Entry(
                        new HLogKey(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));

                writer.close();
            }
            MonitoredTask status = TaskMonitor.get().createStatus(method);
            Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
            for (Store store : region.getStores().values()) {
                maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
            }
            long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
            assertEquals(maxSeqId, seqId);
            Get get = new Get(row);
            Result result = region.get(get);
            for (long i = minSeqId; i <= maxSeqId; i += 10) {
                List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
                assertEquals(1, kvs.size());
                assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
        String method = "testSkipRecoveredEditsReplaySomeIgnored";
        TableName tableName = TableName.valueOf(method);
        byte[] family = Bytes.toBytes("family");
        this.region = initHRegion(tableName, method, CONF, family);
        try {
            Path regiondir = region.getRegionFileSystem().getRegionDir();
            FileSystem fs = region.getRegionFileSystem().getFileSystem();
            byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();

            Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);

            long maxSeqId = 1050;
            long minSeqId = 1000;

            for (long i = minSeqId; i <= maxSeqId; i += 10) {
                Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
                fs.create(recoveredEdits);
                HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);

                long time = System.nanoTime();
                WALEdit edit = new WALEdit();
                edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
                writer.append(new HLog.Entry(
                        new HLogKey(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));

                writer.close();
            }
            long recoverSeqId = 1030;
            MonitoredTask status = TaskMonitor.get().createStatus(method);
            Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
            for (Store store : region.getStores().values()) {
                maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
            }
            long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
            assertEquals(maxSeqId, seqId);
            Get get = new Get(row);
            Result result = region.get(get);
            for (long i = minSeqId; i <= maxSeqId; i += 10) {
                List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
                if (i < recoverSeqId) {
                    assertEquals(0, kvs.size());
                } else {
                    assertEquals(1, kvs.size());
                    assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
                }
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
        byte[] family = Bytes.toBytes("family");
        this.region = initHRegion(tableName, method, CONF, family);
        try {
            Path regiondir = region.getRegionFileSystem().getRegionDir();
            FileSystem fs = region.getRegionFileSystem().getFileSystem();

            Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
            for (int i = 1000; i < 1050; i += 10) {
                Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
                FSDataOutputStream dos = fs.create(recoveredEdits);
                dos.writeInt(i);
                dos.close();
            }
            long minSeqId = 2000;
            Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
            FSDataOutputStream dos = fs.create(recoveredEdits);
            dos.close();

            Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
            for (Store store : region.getStores().values()) {
                maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
            }
            long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
            assertEquals(minSeqId, seqId);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
        String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
        TableName tableName = TableName.valueOf(method);
        byte[] family = Bytes.toBytes("family");
        this.region = initHRegion(tableName, method, CONF, family);
        try {
            Path regiondir = region.getRegionFileSystem().getRegionDir();
            FileSystem fs = region.getRegionFileSystem().getFileSystem();
            byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();

            assertEquals(0, region.getStoreFileList(region.getStores().keySet().toArray(new byte[0][])).size());

            Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);

            long maxSeqId = 1050;
            long minSeqId = 1000;

            for (long i = minSeqId; i <= maxSeqId; i += 10) {
                Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
                fs.create(recoveredEdits);
                HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);

                long time = System.nanoTime();
                WALEdit edit = null;
                if (i == maxSeqId) {
                    edit = WALEdit.createCompaction(
                            CompactionDescriptor.newBuilder().setTableName(ByteString.copyFrom(tableName.getName()))
                                    .setFamilyName(ByteString.copyFrom(regionName))
                                    .setEncodedRegionName(ByteString.copyFrom(regionName))
                                    .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
                                    .build());
                } else {
                    edit = new WALEdit();
                    edit.add(
                            new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
                }
                writer.append(new HLog.Entry(
                        new HLogKey(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
                writer.close();
            }

            long recoverSeqId = 1030;
            Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
            MonitoredTask status = TaskMonitor.get().createStatus(method);
            for (Store store : region.getStores().values()) {
                maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
            }
            long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
            assertEquals(maxSeqId, seqId);

            // assert that the files are flushed
            assertEquals(1, region.getStoreFileList(region.getStores().keySet().toArray(new byte[0][])).size());

        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testRecoveredEditsReplayCompaction() throws Exception {
        String method = name.getMethodName();
        TableName tableName = TableName.valueOf(method);
        byte[] family = Bytes.toBytes("family");
        this.region = initHRegion(tableName, method, CONF, family);
        try {
            Path regiondir = region.getRegionFileSystem().getRegionDir();
            FileSystem fs = region.getRegionFileSystem().getFileSystem();
            byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();

            long maxSeqId = 3;
            long minSeqId = 0;

            for (long i = minSeqId; i < maxSeqId; i++) {
                Put put = new Put(Bytes.toBytes(i));
                put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
                region.put(put);
                region.flushcache();
            }

            // this will create a region with 3 files
            assertEquals(3, region.getStore(family).getStorefilesCount());
            List<Path> storeFiles = new ArrayList<Path>(3);
            for (StoreFile sf : region.getStore(family).getStorefiles()) {
                storeFiles.add(sf.getPath());
            }

            // disable compaction completion
            CONF.setBoolean("hbase.hstore.compaction.complete", false);
            region.compactStores();

            // ensure that nothing changed
            assertEquals(3, region.getStore(family).getStorefilesCount());

            // now find the compacted file, and manually add it to the recovered edits
            Path tmpDir = region.getRegionFileSystem().getTempDir();
            FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
            String errorMsg = "Expected to find 1 file in the region temp directory "
                    + "from the compaction, could not find any";
            assertNotNull(errorMsg, files);
            assertEquals(errorMsg, 1, files.length);
            // move the file inside region dir
            Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family), files[0].getPath());

            CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(
                    this.region.getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile),
                    region.getRegionFileSystem().getStoreDir(Bytes.toString(family)));

            HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(), this.region.getRegionInfo(),
                    compactionDescriptor, new AtomicLong(1));

            Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);

            Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
            fs.create(recoveredEdits);
            HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, CONF);

            long time = System.nanoTime();

            writer.append(
                    new HLog.Entry(new HLogKey(regionName, tableName, 10, time, HConstants.DEFAULT_CLUSTER_ID),
                            WALEdit.createCompaction(compactionDescriptor)));
            writer.close();

            // close the region now, and reopen again
            region.getTableDesc();
            region.getRegionInfo();
            region.close();
            region = HRegion.openHRegion(region, null);

            // now check whether we have only one store file, the compacted one
            Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
            for (StoreFile sf : sfs) {
                LOG.info(sf.getPath());
            }
            assertEquals(1, region.getStore(family).getStorefilesCount());
            files = FSUtils.listStatus(fs, tmpDir);
            assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);

            for (long i = minSeqId; i < maxSeqId; i++) {
                Get get = new Get(Bytes.toBytes(i));
                Result result = region.get(get);
                byte[] value = result.getValue(family, Bytes.toBytes(i));
                assertArrayEquals(Bytes.toBytes(i), value);
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testGetWhileRegionClose() throws IOException {
        TableName tableName = TableName.valueOf(name.getMethodName());
        Configuration hc = initSplit();
        int numRows = 100;
        byte[][] families = { fam1, fam2, fam3 };

        // Setting up region
        String method = name.getMethodName();
        this.region = initHRegion(tableName, method, hc, families);
        try {
            // Put data in region
            final int startRow = 100;
            putData(startRow, numRows, qual1, families);
            putData(startRow, numRows, qual2, families);
            putData(startRow, numRows, qual3, families);
            final AtomicBoolean done = new AtomicBoolean(false);
            final AtomicInteger gets = new AtomicInteger(0);
            GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
            try {
                // Set ten threads running concurrently getting from the region.
                for (int i = 0; i < threads.length / 2; i++) {
                    threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
                    threads[i].setDaemon(true);
                    threads[i].start();
                }
                // Artificially make the condition by setting closing flag explicitly.
                // I can't make the issue happen with a call to region.close().
                this.region.closing.set(true);
                for (int i = threads.length / 2; i < threads.length; i++) {
                    threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
                    threads[i].setDaemon(true);
                    threads[i].start();
                }
            } finally {
                if (this.region != null) {
                    HRegion.closeHRegion(this.region);
                }
            }
            done.set(true);
            for (GetTillDoneOrException t : threads) {
                try {
                    t.join();
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
                if (t.e != null) {
                    LOG.info("Exception=" + t.e);
                    assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
                }
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    /*
     * Thread that does get on single row until 'done' flag is flipped. If an
     * exception causes us to fail, it records it.
     */
    class GetTillDoneOrException extends Thread {
        private final Get g;
        private final AtomicBoolean done;
        private final AtomicInteger count;
        private Exception e;

        GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
            super("getter." + i);
            this.g = new Get(r);
            this.done = d;
            this.count = c;
        }

        @Override
        public void run() {
            while (!this.done.get()) {
                try {
                    assertTrue(region.get(g).size() > 0);
                    this.count.incrementAndGet();
                } catch (Exception e) {
                    this.e = e;
                    break;
                }
            }
        }
    }

    /*
     * An involved filter test. Has multiple column families and deletes in mix.
     */
    @Test
    public void testWeirdCacheBehaviour() throws Exception {
        byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
        byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
                Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
        this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
        try {
            String value = "this is the value";
            String value2 = "this is some other value";
            String keyPrefix1 = "prefix1";
            String keyPrefix2 = "prefix2";
            String keyPrefix3 = "prefix3";
            putRows(this.region, 3, value, keyPrefix1);
            putRows(this.region, 3, value, keyPrefix2);
            putRows(this.region, 3, value, keyPrefix3);
            putRows(this.region, 3, value2, keyPrefix1);
            putRows(this.region, 3, value2, keyPrefix2);
            putRows(this.region, 3, value2, keyPrefix3);
            System.out.println("Checking values for key: " + keyPrefix1);
            assertEquals("Got back incorrect number of rows from scan", 3,
                    getNumberOfRows(keyPrefix1, value2, this.region));
            System.out.println("Checking values for key: " + keyPrefix2);
            assertEquals("Got back incorrect number of rows from scan", 3,
                    getNumberOfRows(keyPrefix2, value2, this.region));
            System.out.println("Checking values for key: " + keyPrefix3);
            assertEquals("Got back incorrect number of rows from scan", 3,
                    getNumberOfRows(keyPrefix3, value2, this.region));
            deleteColumns(this.region, value2, keyPrefix1);
            deleteColumns(this.region, value2, keyPrefix2);
            deleteColumns(this.region, value2, keyPrefix3);
            System.out.println("Starting important checks.....");
            assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
                    getNumberOfRows(keyPrefix1, value2, this.region));
            assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
                    getNumberOfRows(keyPrefix2, value2, this.region));
            assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
                    getNumberOfRows(keyPrefix3, value2, this.region));
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testAppendWithReadOnlyTable() throws Exception {
        byte[] TABLE = Bytes.toBytes("readOnlyTable");
        this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
        boolean exceptionCaught = false;
        Append append = new Append(Bytes.toBytes("somerow"));
        append.setDurability(Durability.SKIP_WAL);
        append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), Bytes.toBytes("somevalue"));
        try {
            region.append(append);
        } catch (IOException e) {
            exceptionCaught = true;
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
        assertTrue(exceptionCaught == true);
    }

    @Test
    public void testIncrWithReadOnlyTable() throws Exception {
        byte[] TABLE = Bytes.toBytes("readOnlyTable");
        this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
        boolean exceptionCaught = false;
        Increment inc = new Increment(Bytes.toBytes("somerow"));
        inc.setDurability(Durability.SKIP_WAL);
        inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
        try {
            region.increment(inc);
        } catch (IOException e) {
            exceptionCaught = true;
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
        assertTrue(exceptionCaught == true);
    }

    private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
        InternalScanner scanner = buildScanner(keyPrefix, value, r);
        int count = 0;
        boolean more = false;
        List<Cell> results = new ArrayList<Cell>();
        do {
            more = scanner.next(results);
            if (results != null && !results.isEmpty())
                count++;
            else
                break;
            Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
            delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
            r.delete(delete);
            results.clear();
        } while (more);
        assertEquals("Did not perform correct number of deletes", 3, count);
    }

    private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
        InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
        int numberOfResults = 0;
        List<Cell> results = new ArrayList<Cell>();
        boolean more = false;
        do {
            more = resultScanner.next(results);
            if (results != null && !results.isEmpty())
                numberOfResults++;
            else
                break;
            for (Cell kv : results) {
                System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
            }
            results.clear();
        } while (more);
        return numberOfResults;
    }

    private InternalScanner buildScanner(String keyPrefix, String value, HRegion r) throws IOException {
        // Defaults FilterList.Operator.MUST_PASS_ALL.
        FilterList allFilters = new FilterList();
        allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
        // Only return rows where this column value exists in the row.
        SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
                Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
        filter.setFilterIfMissing(true);
        allFilters.addFilter(filter);
        Scan scan = new Scan();
        scan.addFamily(Bytes.toBytes("trans-blob"));
        scan.addFamily(Bytes.toBytes("trans-type"));
        scan.addFamily(Bytes.toBytes("trans-date"));
        scan.addFamily(Bytes.toBytes("trans-tags"));
        scan.addFamily(Bytes.toBytes("trans-group"));
        scan.setFilter(allFilters);
        return r.getScanner(scan);
    }

    private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
        for (int i = 0; i < numRows; i++) {
            String row = key + "_" + i/* UUID.randomUUID().toString() */;
            System.out.println(String.format("Saving row: %s, with value %s", row, value));
            Put put = new Put(Bytes.toBytes(row));
            put.setDurability(Durability.SKIP_WAL);
            put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
            put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
            put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
            put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
            put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
            r.put(put);
        }
    }

    @Test
    public void testFamilyWithAndWithoutColon() throws Exception {
        byte[] b = Bytes.toBytes(getName());
        byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
        this.region = initHRegion(b, getName(), CONF, cf);
        try {
            Put p = new Put(b);
            byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
            p.add(cfwithcolon, cfwithcolon, cfwithcolon);
            boolean exception = false;
            try {
                this.region.put(p);
            } catch (NoSuchColumnFamilyException e) {
                exception = true;
            }
            assertTrue(exception);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testBatchPut() throws Exception {
        byte[] b = Bytes.toBytes(getName());
        byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
        byte[] qual = Bytes.toBytes("qual");
        byte[] val = Bytes.toBytes("val");
        this.region = initHRegion(b, getName(), CONF, cf);
        MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
        try {
            long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
            metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);

            LOG.info("First a batch put with all valid puts");
            final Put[] puts = new Put[10];
            for (int i = 0; i < 10; i++) {
                puts[i] = new Put(Bytes.toBytes("row_" + i));
                puts[i].add(cf, qual, val);
            }

            OperationStatus[] codes = this.region.batchMutate(puts);
            assertEquals(10, codes.length);
            for (int i = 0; i < 10; i++) {
                assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
            }
            metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);

            LOG.info("Next a batch put with one invalid family");
            puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
            codes = this.region.batchMutate(puts);
            assertEquals(10, codes.length);
            for (int i = 0; i < 10; i++) {
                assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
                        codes[i].getOperationStatusCode());
            }

            metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);

            LOG.info("Next a batch put that has to break into two batches to avoid a lock");
            RowLock rowLock = region.getRowLock(Bytes.toBytes("row_2"));

            MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
            final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
            TestThread putter = new TestThread(ctx) {
                @Override
                public void doWork() throws IOException {
                    retFromThread.set(region.batchMutate(puts));
                }
            };
            LOG.info("...starting put thread while holding lock");
            ctx.addThread(putter);
            ctx.startThreads();

            LOG.info("...waiting for put thread to sync first time");
            long startWait = System.currentTimeMillis();
            while (metricsAssertHelper.getCounter("syncTimeNumOps", source) == syncs + 2) {
                Thread.sleep(100);
                if (System.currentTimeMillis() - startWait > 10000) {
                    fail("Timed out waiting for thread to sync first minibatch");
                }
            }
            LOG.info("...releasing row lock, which should let put thread continue");
            rowLock.release();
            LOG.info("...joining on thread");
            ctx.stop();
            LOG.info("...checking that next batch was synced");
            metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 4, source);
            codes = retFromThread.get();
            for (int i = 0; i < 10; i++) {
                assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
                        codes[i].getOperationStatusCode());
            }

        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testBatchPutWithTsSlop() throws Exception {
        byte[] b = Bytes.toBytes(getName());
        byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
        byte[] qual = Bytes.toBytes("qual");
        byte[] val = Bytes.toBytes("val");

        // add data with a timestamp that is too recent for range. Ensure assert
        CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
        this.region = initHRegion(b, getName(), CONF, cf);

        try {
            MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
            long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
            metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);

            final Put[] puts = new Put[10];
            for (int i = 0; i < 10; i++) {
                puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
                puts[i].add(cf, qual, val);
            }

            OperationStatus[] codes = this.region.batchMutate(puts);
            assertEquals(10, codes.length);
            for (int i = 0; i < 10; i++) {
                assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
            }
            metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);

        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }

    }

    // ////////////////////////////////////////////////////////////////////////////
    // checkAndMutate tests
    // ////////////////////////////////////////////////////////////////////////////
    @Test
    public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] qf1 = Bytes.toBytes("qualifier");
        byte[] emptyVal = new byte[] {};
        byte[] val1 = Bytes.toBytes("value1");
        byte[] val2 = Bytes.toBytes("value2");

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, fam1);
        try {
            // Putting empty data in key
            Put put = new Put(row1);
            put.add(fam1, qf1, emptyVal);

            // checkAndPut with empty value
            boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
                    put, true);
            assertTrue(res);

            // Putting data in key
            put = new Put(row1);
            put.add(fam1, qf1, val1);

            // checkAndPut with correct value
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal), put,
                    true);
            assertTrue(res);

            // not empty anymore
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal), put,
                    true);
            assertFalse(res);

            Delete delete = new Delete(row1);
            delete.deleteColumn(fam1, qf1);
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal), delete,
                    true);
            assertFalse(res);

            put = new Put(row1);
            put.add(fam1, qf1, val2);
            // checkAndPut with correct value
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1), put, true);
            assertTrue(res);

            // checkAndDelete with correct value
            delete = new Delete(row1);
            delete.deleteColumn(fam1, qf1);
            delete.deleteColumn(fam1, qf1);
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2), delete, true);
            assertTrue(res);

            delete = new Delete(row1);
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal), delete,
                    true);
            assertTrue(res);

            // checkAndPut looking for a null value
            put = new Put(row1);
            put.add(fam1, qf1, val1);

            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
            assertTrue(res);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testCheckAndMutate_WithWrongValue() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] qf1 = Bytes.toBytes("qualifier");
        byte[] val1 = Bytes.toBytes("value1");
        byte[] val2 = Bytes.toBytes("value2");

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, fam1);
        try {
            // Putting data in key
            Put put = new Put(row1);
            put.add(fam1, qf1, val1);
            region.put(put);

            // checkAndPut with wrong value
            boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2), put,
                    true);
            assertEquals(false, res);

            // checkAndDelete with wrong value
            Delete delete = new Delete(row1);
            delete.deleteFamily(fam1);
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2), delete, true);
            assertEquals(false, res);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testCheckAndMutate_WithCorrectValue() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] qf1 = Bytes.toBytes("qualifier");
        byte[] val1 = Bytes.toBytes("value1");

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, fam1);
        try {
            // Putting data in key
            Put put = new Put(row1);
            put.add(fam1, qf1, val1);
            region.put(put);

            // checkAndPut with correct value
            boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1), put,
                    true);
            assertEquals(true, res);

            // checkAndDelete with correct value
            Delete delete = new Delete(row1);
            delete.deleteColumn(fam1, qf1);
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1), put, true);
            assertEquals(true, res);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] qf1 = Bytes.toBytes("qualifier");
        byte[] val1 = Bytes.toBytes("value1");
        byte[] val2 = Bytes.toBytes("value2");
        byte[] val3 = Bytes.toBytes("value3");
        byte[] val4 = Bytes.toBytes("value4");

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, fam1);
        try {
            // Putting val3 in key
            Put put = new Put(row1);
            put.add(fam1, qf1, val3);
            region.put(put);

            // Test CompareOp.LESS: original = val3, compare with val3, fail
            boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS, new BinaryComparator(val3), put,
                    true);
            assertEquals(false, res);

            // Test CompareOp.LESS: original = val3, compare with val4, fail
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS, new BinaryComparator(val4), put, true);
            assertEquals(false, res);

            // Test CompareOp.LESS: original = val3, compare with val2,
            // succeed (now value = val2)
            put = new Put(row1);
            put.add(fam1, qf1, val2);
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS, new BinaryComparator(val2), put, true);
            assertEquals(true, res);

            // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL, new BinaryComparator(val3), put,
                    true);
            assertEquals(false, res);

            // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2,
            // succeed (value still = val2)
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL, new BinaryComparator(val2), put,
                    true);
            assertEquals(true, res);

            // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1,
            // succeed (now value = val3)
            put = new Put(row1);
            put.add(fam1, qf1, val3);
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL, new BinaryComparator(val1), put,
                    true);
            assertEquals(true, res);

            // Test CompareOp.GREATER: original = val3, compare with val3, fail
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER, new BinaryComparator(val3), put, true);
            assertEquals(false, res);

            // Test CompareOp.GREATER: original = val3, compare with val2, fail
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER, new BinaryComparator(val2), put, true);
            assertEquals(false, res);

            // Test CompareOp.GREATER: original = val3, compare with val4,
            // succeed (now value = val2)
            put = new Put(row1);
            put.add(fam1, qf1, val2);
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER, new BinaryComparator(val4), put, true);
            assertEquals(true, res);

            // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL, new BinaryComparator(val1),
                    put, true);
            assertEquals(false, res);

            // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2,
            // succeed (value still = val2)
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL, new BinaryComparator(val2),
                    put, true);
            assertEquals(true, res);

            // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL, new BinaryComparator(val3),
                    put, true);
            assertEquals(true, res);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testCheckAndPut_ThatPutWasWritten() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] fam2 = Bytes.toBytes("fam2");
        byte[] qf1 = Bytes.toBytes("qualifier");
        byte[] val1 = Bytes.toBytes("value1");
        byte[] val2 = Bytes.toBytes("value2");

        byte[][] families = { fam1, fam2 };

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, families);
        try {
            // Putting data in the key to check
            Put put = new Put(row1);
            put.add(fam1, qf1, val1);
            region.put(put);

            // Creating put to add
            long ts = System.currentTimeMillis();
            KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
            put = new Put(row1);
            put.add(kv);

            // checkAndPut with wrong value
            boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1), put,
                    true);
            assertEquals(true, res);

            Get get = new Get(row1);
            get.addColumn(fam2, qf1);
            Cell[] actual = region.get(get).rawCells();

            Cell[] expected = { kv };

            assertEquals(expected.length, actual.length);
            for (int i = 0; i < actual.length; i++) {
                assertEquals(expected[i], actual[i]);
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testCheckAndPut_wrongRowInPut() throws IOException {
        TableName tableName = TableName.valueOf(name.getMethodName());
        this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
        try {
            Put put = new Put(row2);
            put.add(fam1, qual1, value1);
            try {
                region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(value2), put, false);
                fail();
            } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
                // expected exception.
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] fam2 = Bytes.toBytes("fam2");
        byte[] qf1 = Bytes.toBytes("qualifier1");
        byte[] qf2 = Bytes.toBytes("qualifier2");
        byte[] qf3 = Bytes.toBytes("qualifier3");
        byte[] val1 = Bytes.toBytes("value1");
        byte[] val2 = Bytes.toBytes("value2");
        byte[] val3 = Bytes.toBytes("value3");
        byte[] emptyVal = new byte[] {};

        byte[][] families = { fam1, fam2 };

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, families);
        try {
            // Put content
            Put put = new Put(row1);
            put.add(fam1, qf1, val1);
            region.put(put);
            Threads.sleep(2);

            put = new Put(row1);
            put.add(fam1, qf1, val2);
            put.add(fam2, qf1, val3);
            put.add(fam2, qf2, val2);
            put.add(fam2, qf3, val1);
            put.add(fam1, qf3, val1);
            region.put(put);

            // Multi-column delete
            Delete delete = new Delete(row1);
            delete.deleteColumn(fam1, qf1);
            delete.deleteColumn(fam2, qf1);
            delete.deleteColumn(fam1, qf3);
            boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
                    delete, true);
            assertEquals(true, res);

            Get get = new Get(row1);
            get.addColumn(fam1, qf1);
            get.addColumn(fam1, qf3);
            get.addColumn(fam2, qf2);
            Result r = region.get(get);
            assertEquals(2, r.size());
            assertArrayEquals(val1, r.getValue(fam1, qf1));
            assertArrayEquals(val2, r.getValue(fam2, qf2));

            // Family delete
            delete = new Delete(row1);
            delete.deleteFamily(fam2);
            res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal), delete,
                    true);
            assertEquals(true, res);

            get = new Get(row1);
            r = region.get(get);
            assertEquals(1, r.size());
            assertArrayEquals(val1, r.getValue(fam1, qf1));

            // Row delete
            delete = new Delete(row1);
            res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1), delete, true);
            assertEquals(true, res);
            get = new Get(row1);
            r = region.get(get);
            assertEquals(0, r.size());
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    // ////////////////////////////////////////////////////////////////////////////
    // Delete tests
    // ////////////////////////////////////////////////////////////////////////////
    @Test
    public void testDelete_multiDeleteColumn() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] qual = Bytes.toBytes("qualifier");
        byte[] value = Bytes.toBytes("value");

        Put put = new Put(row1);
        put.add(fam1, qual, 1, value);
        put.add(fam1, qual, 2, value);

        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, fam1);
        try {
            region.put(put);

            // We do support deleting more than 1 'latest' version
            Delete delete = new Delete(row1);
            delete.deleteColumn(fam1, qual);
            delete.deleteColumn(fam1, qual);
            region.delete(delete);

            Get get = new Get(row1);
            get.addFamily(fam1);
            Result r = region.get(get);
            assertEquals(0, r.size());
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testDelete_CheckFamily() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] fam2 = Bytes.toBytes("fam2");
        byte[] fam3 = Bytes.toBytes("fam3");
        byte[] fam4 = Bytes.toBytes("fam4");

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
        try {
            List<Cell> kvs = new ArrayList<Cell>();
            kvs.add(new KeyValue(row1, fam4, null, null));

            // testing existing family
            byte[] family = fam2;
            try {
                NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
                        Bytes.BYTES_COMPARATOR);
                deleteMap.put(family, kvs);
                region.delete(deleteMap, Durability.SYNC_WAL);
            } catch (Exception e) {
                assertTrue("Family " + new String(family) + " does not exist", false);
            }

            // testing non existing family
            boolean ok = false;
            family = fam4;
            try {
                NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
                        Bytes.BYTES_COMPARATOR);
                deleteMap.put(family, kvs);
                region.delete(deleteMap, Durability.SYNC_WAL);
            } catch (Exception e) {
                ok = true;
            }
            assertEquals("Family " + new String(family) + " does exist", true, ok);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testDelete_mixed() throws IOException, InterruptedException {
        byte[] fam = Bytes.toBytes("info");
        byte[][] families = { fam };
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, families);
        try {
            EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());

            byte[] row = Bytes.toBytes("table_name");
            // column names
            byte[] serverinfo = Bytes.toBytes("serverinfo");
            byte[] splitA = Bytes.toBytes("splitA");
            byte[] splitB = Bytes.toBytes("splitB");

            // add some data:
            Put put = new Put(row);
            put.add(fam, splitA, Bytes.toBytes("reference_A"));
            region.put(put);

            put = new Put(row);
            put.add(fam, splitB, Bytes.toBytes("reference_B"));
            region.put(put);

            put = new Put(row);
            put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
            region.put(put);

            // ok now delete a split:
            Delete delete = new Delete(row);
            delete.deleteColumns(fam, splitA);
            region.delete(delete);

            // assert some things:
            Get get = new Get(row).addColumn(fam, serverinfo);
            Result result = region.get(get);
            assertEquals(1, result.size());

            get = new Get(row).addColumn(fam, splitA);
            result = region.get(get);
            assertEquals(0, result.size());

            get = new Get(row).addColumn(fam, splitB);
            result = region.get(get);
            assertEquals(1, result.size());

            // Assert that after a delete, I can put.
            put = new Put(row);
            put.add(fam, splitA, Bytes.toBytes("reference_A"));
            region.put(put);
            get = new Get(row);
            result = region.get(get);
            assertEquals(3, result.size());

            // Now delete all... then test I can add stuff back
            delete = new Delete(row);
            region.delete(delete);
            assertEquals(0, region.get(get).size());

            region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
            result = region.get(get);
            assertEquals(1, result.size());
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testDeleteRowWithFutureTs() throws IOException {
        byte[] fam = Bytes.toBytes("info");
        byte[][] families = { fam };
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, families);
        try {
            byte[] row = Bytes.toBytes("table_name");
            // column names
            byte[] serverinfo = Bytes.toBytes("serverinfo");

            // add data in the far future
            Put put = new Put(row);
            put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
            region.put(put);

            // now delete something in the present
            Delete delete = new Delete(row);
            region.delete(delete);

            // make sure we still see our data
            Get get = new Get(row).addColumn(fam, serverinfo);
            Result result = region.get(get);
            assertEquals(1, result.size());

            // delete the future row
            delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
            region.delete(delete);

            // make sure it is gone
            get = new Get(row).addColumn(fam, serverinfo);
            result = region.get(get);
            assertEquals(0, result.size());
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    /**
     * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by
     * the actual timestamp
     */
    @Test
    public void testPutWithLatestTS() throws IOException {
        byte[] fam = Bytes.toBytes("info");
        byte[][] families = { fam };
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, families);
        try {
            byte[] row = Bytes.toBytes("row1");
            // column names
            byte[] qual = Bytes.toBytes("qual");

            // add data with LATEST_TIMESTAMP, put without WAL
            Put put = new Put(row);
            put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
            region.put(put);

            // Make sure it shows up with an actual timestamp
            Get get = new Get(row).addColumn(fam, qual);
            Result result = region.get(get);
            assertEquals(1, result.size());
            Cell kv = result.rawCells()[0];
            LOG.info("Got: " + kv);
            assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
                    kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);

            // Check same with WAL enabled (historically these took different
            // code paths, so check both)
            row = Bytes.toBytes("row2");
            put = new Put(row);
            put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
            region.put(put);

            // Make sure it shows up with an actual timestamp
            get = new Get(row).addColumn(fam, qual);
            result = region.get(get);
            assertEquals(1, result.size());
            kv = result.rawCells()[0];
            LOG.info("Got: " + kv);
            assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
                    kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }

    }

    /**
     * Tests that there is server-side filtering for invalid timestamp upper
     * bound. Note that the timestamp lower bound is automatically handled for us
     * by the TTL field.
     */
    @Test
    public void testPutWithTsSlop() throws IOException {
        byte[] fam = Bytes.toBytes("info");
        byte[][] families = { fam };
        String method = this.getName();

        // add data with a timestamp that is too recent for range. Ensure assert
        CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
        this.region = initHRegion(tableName, method, CONF, families);
        boolean caughtExcep = false;
        try {
            try {
                // no TS specified == use latest. should not error
                region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
                // TS out of range. should error
                region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
                        Bytes.toBytes("value")));
                fail("Expected IOE for TS out of configured timerange");
            } catch (FailedSanityCheckException ioe) {
                LOG.debug("Received expected exception", ioe);
                caughtExcep = true;
            }
            assertTrue("Should catch FailedSanityCheckException", caughtExcep);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
        byte[] fam1 = Bytes.toBytes("columnA");
        byte[] fam2 = Bytes.toBytes("columnB");
        this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
        try {
            byte[] rowA = Bytes.toBytes("rowA");
            byte[] rowB = Bytes.toBytes("rowB");

            byte[] value = Bytes.toBytes("value");

            Delete delete = new Delete(rowA);
            delete.deleteFamily(fam1);

            region.delete(delete);

            // now create data.
            Put put = new Put(rowA);
            put.add(fam2, null, value);
            region.put(put);

            put = new Put(rowB);
            put.add(fam1, null, value);
            put.add(fam2, null, value);
            region.put(put);

            Scan scan = new Scan();
            scan.addFamily(fam1).addFamily(fam2);
            InternalScanner s = region.getScanner(scan);
            List<Cell> results = new ArrayList<Cell>();
            s.next(results);
            assertTrue(CellUtil.matchingRow(results.get(0), rowA));

            results.clear();
            s.next(results);
            assertTrue(CellUtil.matchingRow(results.get(0), rowB));
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
        Delete delete = new Delete(row);
        delete.deleteColumns(fam1, qual1);
        doTestDelete_AndPostInsert(delete);
    }

    @Test
    public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
        Delete delete = new Delete(row);
        delete.deleteFamily(fam1);
        doTestDelete_AndPostInsert(delete);
    }

    public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
        TableName tableName = TableName.valueOf(name.getMethodName());
        this.region = initHRegion(tableName, getName(), CONF, fam1);
        try {
            EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
            Put put = new Put(row);
            put.add(fam1, qual1, value1);
            region.put(put);

            // now delete the value:
            region.delete(delete);

            // ok put data:
            put = new Put(row);
            put.add(fam1, qual1, value2);
            region.put(put);

            // ok get:
            Get get = new Get(row);
            get.addColumn(fam1, qual1);

            Result r = region.get(get);
            assertEquals(1, r.size());
            assertArrayEquals(value2, r.getValue(fam1, qual1));

            // next:
            Scan scan = new Scan(row);
            scan.addColumn(fam1, qual1);
            InternalScanner s = region.getScanner(scan);

            List<Cell> results = new ArrayList<Cell>();
            assertEquals(false, s.next(results));
            assertEquals(1, results.size());
            Cell kv = results.get(0);

            assertArrayEquals(value2, CellUtil.cloneValue(kv));
            assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
            assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
            assertArrayEquals(row, CellUtil.cloneRow(kv));
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testDelete_CheckTimestampUpdated() throws IOException {
        TableName tableName = TableName.valueOf(name.getMethodName());
        byte[] row1 = Bytes.toBytes("row1");
        byte[] col1 = Bytes.toBytes("col1");
        byte[] col2 = Bytes.toBytes("col2");
        byte[] col3 = Bytes.toBytes("col3");

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, fam1);
        try {
            // Building checkerList
            List<Cell> kvs = new ArrayList<Cell>();
            kvs.add(new KeyValue(row1, fam1, col1, null));
            kvs.add(new KeyValue(row1, fam1, col2, null));
            kvs.add(new KeyValue(row1, fam1, col3, null));

            NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR);
            deleteMap.put(fam1, kvs);
            region.delete(deleteMap, Durability.SYNC_WAL);

            // extract the key values out the memstore:
            // This is kinda hacky, but better than nothing...
            long now = System.currentTimeMillis();
            DefaultMemStore memstore = (DefaultMemStore) ((HStore) region.getStore(fam1)).memstore;
            KeyValue firstKv = memstore.kvset.first();
            assertTrue(firstKv.getTimestamp() <= now);
            now = firstKv.getTimestamp();
            for (Cell cell : memstore.kvset) {
                assertTrue(cell.getTimestamp() <= now);
                now = cell.getTimestamp();
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    // ////////////////////////////////////////////////////////////////////////////
    // Get tests
    // ////////////////////////////////////////////////////////////////////////////
    @Test
    public void testGet_FamilyChecker() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] fam2 = Bytes.toBytes("False");
        byte[] col1 = Bytes.toBytes("col1");

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, fam1);
        try {
            Get get = new Get(row1);
            get.addColumn(fam2, col1);

            // Test
            try {
                region.get(get);
            } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
                assertFalse(false);
                return;
            }
            assertFalse(true);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testGet_Basic() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] col1 = Bytes.toBytes("col1");
        byte[] col2 = Bytes.toBytes("col2");
        byte[] col3 = Bytes.toBytes("col3");
        byte[] col4 = Bytes.toBytes("col4");
        byte[] col5 = Bytes.toBytes("col5");

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, fam1);
        try {
            // Add to memstore
            Put put = new Put(row1);
            put.add(fam1, col1, null);
            put.add(fam1, col2, null);
            put.add(fam1, col3, null);
            put.add(fam1, col4, null);
            put.add(fam1, col5, null);
            region.put(put);

            Get get = new Get(row1);
            get.addColumn(fam1, col2);
            get.addColumn(fam1, col4);
            // Expected result
            KeyValue kv1 = new KeyValue(row1, fam1, col2);
            KeyValue kv2 = new KeyValue(row1, fam1, col4);
            KeyValue[] expected = { kv1, kv2 };

            // Test
            Result res = region.get(get);
            assertEquals(expected.length, res.size());
            for (int i = 0; i < res.size(); i++) {
                assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
                assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
                assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
            }

            // Test using a filter on a Get
            Get g = new Get(row1);
            final int count = 2;
            g.setFilter(new ColumnCountGetFilter(count));
            res = region.get(g);
            assertEquals(count, res.size());
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testGet_Empty() throws IOException {
        byte[] row = Bytes.toBytes("row");
        byte[] fam = Bytes.toBytes("fam");

        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, fam);
        try {
            Get get = new Get(row);
            get.addFamily(fam);
            Result r = region.get(get);

            assertTrue(r.isEmpty());
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    // ////////////////////////////////////////////////////////////////////////////
    // Merge test
    // ////////////////////////////////////////////////////////////////////////////
    @Test
    public void testMerge() throws IOException {
        byte[][] families = { fam1, fam2, fam3 };
        Configuration hc = initSplit();
        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, hc, families);
        try {
            LOG.info("" + HBaseTestCase.addContent(region, fam3));
            region.flushcache();
            region.compactStores();
            byte[] splitRow = region.checkSplit();
            assertNotNull(splitRow);
            LOG.info("SplitRow: " + Bytes.toString(splitRow));
            HRegion[] subregions = splitRegion(region, splitRow);
            try {
                // Need to open the regions.
                for (int i = 0; i < subregions.length; i++) {
                    HRegion.openHRegion(subregions[i], null);
                    subregions[i].compactStores();
                }
                Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
                Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
                Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
                long startTime = System.currentTimeMillis();
                region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
                LOG.info("Merge regions elapsed time: " + ((System.currentTimeMillis() - startTime) / 1000.0));
                FILESYSTEM.delete(oldRegion1, true);
                FILESYSTEM.delete(oldRegion2, true);
                FILESYSTEM.delete(oldRegionPath, true);
                LOG.info("splitAndMerge completed.");
            } finally {
                for (int i = 0; i < subregions.length; i++) {
                    try {
                        HRegion.closeHRegion(subregions[i]);
                    } catch (IOException e) {
                        // Ignore.
                    }
                }
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    /**
     * @param parent
     *          Region to split.
     * @param midkey
     *          Key to split around.
     * @return The Regions we created.
     * @throws IOException
     */
    HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
        PairOfSameType<HRegion> result = null;
        SplitTransaction st = new SplitTransaction(parent, midkey);
        // If prepare does not return true, for some reason -- logged inside in
        // the prepare call -- we are not ready to split just now. Just return.
        if (!st.prepare())
            return null;
        try {
            result = st.execute(null, null);
        } catch (IOException ioe) {
            try {
                LOG.info("Running rollback of failed split of " + parent.getRegionNameAsString() + "; "
                        + ioe.getMessage());
                st.rollback(null, null);
                LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString());
                return null;
            } catch (RuntimeException e) {
                // If failed rollback, kill this server to avoid having a hole in table.
                LOG.info("Failed rollback of failed split of " + parent.getRegionNameAsString()
                        + " -- aborting server", e);
            }
        }
        return new HRegion[] { result.getFirst(), result.getSecond() };
    }

    // ////////////////////////////////////////////////////////////////////////////
    // Scanner tests
    // ////////////////////////////////////////////////////////////////////////////
    @Test
    public void testGetScanner_WithOkFamilies() throws IOException {
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] fam2 = Bytes.toBytes("fam2");

        byte[][] families = { fam1, fam2 };

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, families);
        try {
            Scan scan = new Scan();
            scan.addFamily(fam1);
            scan.addFamily(fam2);
            try {
                region.getScanner(scan);
            } catch (Exception e) {
                assertTrue("Families could not be found in Region", false);
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testGetScanner_WithNotOkFamilies() throws IOException {
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] fam2 = Bytes.toBytes("fam2");

        byte[][] families = { fam1 };

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, families);
        try {
            Scan scan = new Scan();
            scan.addFamily(fam2);
            boolean ok = false;
            try {
                region.getScanner(scan);
            } catch (Exception e) {
                ok = true;
            }
            assertTrue("Families could not be found in Region", ok);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testGetScanner_WithNoFamilies() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] fam2 = Bytes.toBytes("fam2");
        byte[] fam3 = Bytes.toBytes("fam3");
        byte[] fam4 = Bytes.toBytes("fam4");

        byte[][] families = { fam1, fam2, fam3, fam4 };

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, families);
        try {

            // Putting data in Region
            Put put = new Put(row1);
            put.add(fam1, null, null);
            put.add(fam2, null, null);
            put.add(fam3, null, null);
            put.add(fam4, null, null);
            region.put(put);

            Scan scan = null;
            HRegion.RegionScannerImpl is = null;

            // Testing to see how many scanners that is produced by getScanner,
            // starting
            // with known number, 2 - current = 1
            scan = new Scan();
            scan.addFamily(fam2);
            scan.addFamily(fam4);
            is = (RegionScannerImpl) region.getScanner(scan);
            assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());

            scan = new Scan();
            is = (RegionScannerImpl) region.getScanner(scan);
            assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    /**
     * This method tests https://issues.apache.org/jira/browse/HBASE-2516.
     * 
     * @throws IOException
     */
    @Test
    public void testGetScanner_WithRegionClosed() throws IOException {
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] fam2 = Bytes.toBytes("fam2");

        byte[][] families = { fam1, fam2 };

        // Setting up region
        String method = this.getName();
        try {
            this.region = initHRegion(tableName, method, CONF, families);
        } catch (IOException e) {
            e.printStackTrace();
            fail("Got IOException during initHRegion, " + e.getMessage());
        }
        try {
            region.closed.set(true);
            try {
                region.getScanner(null);
                fail("Expected to get an exception during getScanner on a region that is closed");
            } catch (NotServingRegionException e) {
                // this is the correct exception that is expected
            } catch (IOException e) {
                fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
                        + e.getMessage());
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testRegionScanner_Next() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] row2 = Bytes.toBytes("row2");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] fam2 = Bytes.toBytes("fam2");
        byte[] fam3 = Bytes.toBytes("fam3");
        byte[] fam4 = Bytes.toBytes("fam4");

        byte[][] families = { fam1, fam2, fam3, fam4 };
        long ts = System.currentTimeMillis();

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, families);
        try {
            // Putting data in Region
            Put put = null;
            put = new Put(row1);
            put.add(fam1, (byte[]) null, ts, null);
            put.add(fam2, (byte[]) null, ts, null);
            put.add(fam3, (byte[]) null, ts, null);
            put.add(fam4, (byte[]) null, ts, null);
            region.put(put);

            put = new Put(row2);
            put.add(fam1, (byte[]) null, ts, null);
            put.add(fam2, (byte[]) null, ts, null);
            put.add(fam3, (byte[]) null, ts, null);
            put.add(fam4, (byte[]) null, ts, null);
            region.put(put);

            Scan scan = new Scan();
            scan.addFamily(fam2);
            scan.addFamily(fam4);
            InternalScanner is = region.getScanner(scan);

            List<Cell> res = null;

            // Result 1
            List<Cell> expected1 = new ArrayList<Cell>();
            expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
            expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));

            res = new ArrayList<Cell>();
            is.next(res);
            for (int i = 0; i < res.size(); i++) {
                assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
            }

            // Result 2
            List<Cell> expected2 = new ArrayList<Cell>();
            expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
            expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));

            res = new ArrayList<Cell>();
            is.next(res);
            for (int i = 0; i < res.size(); i++) {
                assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] qf1 = Bytes.toBytes("qualifier1");
        byte[] qf2 = Bytes.toBytes("qualifier2");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[][] families = { fam1 };

        long ts1 = System.currentTimeMillis();
        long ts2 = ts1 + 1;
        long ts3 = ts1 + 2;

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, families);
        try {
            // Putting data in Region
            Put put = null;
            KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
            KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
            KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);

            KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
            KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
            KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);

            put = new Put(row1);
            put.add(kv13);
            put.add(kv12);
            put.add(kv11);
            put.add(kv23);
            put.add(kv22);
            put.add(kv21);
            region.put(put);

            // Expected
            List<Cell> expected = new ArrayList<Cell>();
            expected.add(kv13);
            expected.add(kv12);

            Scan scan = new Scan(row1);
            scan.addColumn(fam1, qf1);
            scan.setMaxVersions(MAX_VERSIONS);
            List<Cell> actual = new ArrayList<Cell>();
            InternalScanner scanner = region.getScanner(scan);

            boolean hasNext = scanner.next(actual);
            assertEquals(false, hasNext);

            // Verify result
            for (int i = 0; i < expected.size(); i++) {
                assertEquals(expected.get(i), actual.get(i));
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] qf1 = Bytes.toBytes("qualifier1");
        byte[] qf2 = Bytes.toBytes("qualifier2");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[][] families = { fam1 };

        long ts1 = 1; // System.currentTimeMillis();
        long ts2 = ts1 + 1;
        long ts3 = ts1 + 2;

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, families);
        try {
            // Putting data in Region
            Put put = null;
            KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
            KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
            KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);

            KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
            KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
            KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);

            put = new Put(row1);
            put.add(kv13);
            put.add(kv12);
            put.add(kv11);
            put.add(kv23);
            put.add(kv22);
            put.add(kv21);
            region.put(put);
            region.flushcache();

            // Expected
            List<Cell> expected = new ArrayList<Cell>();
            expected.add(kv13);
            expected.add(kv12);
            expected.add(kv23);
            expected.add(kv22);

            Scan scan = new Scan(row1);
            scan.addColumn(fam1, qf1);
            scan.addColumn(fam1, qf2);
            scan.setMaxVersions(MAX_VERSIONS);
            List<Cell> actual = new ArrayList<Cell>();
            InternalScanner scanner = region.getScanner(scan);

            boolean hasNext = scanner.next(actual);
            assertEquals(false, hasNext);

            // Verify result
            for (int i = 0; i < expected.size(); i++) {
                assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[][] families = { fam1 };
        byte[] qf1 = Bytes.toBytes("qualifier1");
        byte[] qf2 = Bytes.toBytes("qualifier2");

        long ts1 = 1;
        long ts2 = ts1 + 1;
        long ts3 = ts1 + 2;
        long ts4 = ts1 + 3;

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, families);
        try {
            // Putting data in Region
            KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
            KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
            KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
            KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);

            KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
            KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
            KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
            KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);

            Put put = null;
            put = new Put(row1);
            put.add(kv14);
            put.add(kv24);
            region.put(put);
            region.flushcache();

            put = new Put(row1);
            put.add(kv23);
            put.add(kv13);
            region.put(put);
            region.flushcache();

            put = new Put(row1);
            put.add(kv22);
            put.add(kv12);
            region.put(put);
            region.flushcache();

            put = new Put(row1);
            put.add(kv21);
            put.add(kv11);
            region.put(put);

            // Expected
            List<Cell> expected = new ArrayList<Cell>();
            expected.add(kv14);
            expected.add(kv13);
            expected.add(kv12);
            expected.add(kv24);
            expected.add(kv23);
            expected.add(kv22);

            Scan scan = new Scan(row1);
            scan.addColumn(fam1, qf1);
            scan.addColumn(fam1, qf2);
            int versions = 3;
            scan.setMaxVersions(versions);
            List<Cell> actual = new ArrayList<Cell>();
            InternalScanner scanner = region.getScanner(scan);

            boolean hasNext = scanner.next(actual);
            assertEquals(false, hasNext);

            // Verify result
            for (int i = 0; i < expected.size(); i++) {
                assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] qf1 = Bytes.toBytes("qualifier1");
        byte[] qf2 = Bytes.toBytes("qualifier2");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[][] families = { fam1 };

        long ts1 = System.currentTimeMillis();
        long ts2 = ts1 + 1;
        long ts3 = ts1 + 2;

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, families);
        try {
            // Putting data in Region
            Put put = null;
            KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
            KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
            KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);

            KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
            KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
            KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);

            put = new Put(row1);
            put.add(kv13);
            put.add(kv12);
            put.add(kv11);
            put.add(kv23);
            put.add(kv22);
            put.add(kv21);
            region.put(put);

            // Expected
            List<Cell> expected = new ArrayList<Cell>();
            expected.add(kv13);
            expected.add(kv12);
            expected.add(kv23);
            expected.add(kv22);

            Scan scan = new Scan(row1);
            scan.addFamily(fam1);
            scan.setMaxVersions(MAX_VERSIONS);
            List<Cell> actual = new ArrayList<Cell>();
            InternalScanner scanner = region.getScanner(scan);

            boolean hasNext = scanner.next(actual);
            assertEquals(false, hasNext);

            // Verify result
            for (int i = 0; i < expected.size(); i++) {
                assertEquals(expected.get(i), actual.get(i));
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] qf1 = Bytes.toBytes("qualifier1");
        byte[] qf2 = Bytes.toBytes("qualifier2");
        byte[] fam1 = Bytes.toBytes("fam1");

        long ts1 = 1; // System.currentTimeMillis();
        long ts2 = ts1 + 1;
        long ts3 = ts1 + 2;

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, fam1);
        try {
            // Putting data in Region
            Put put = null;
            KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
            KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
            KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);

            KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
            KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
            KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);

            put = new Put(row1);
            put.add(kv13);
            put.add(kv12);
            put.add(kv11);
            put.add(kv23);
            put.add(kv22);
            put.add(kv21);
            region.put(put);
            region.flushcache();

            // Expected
            List<Cell> expected = new ArrayList<Cell>();
            expected.add(kv13);
            expected.add(kv12);
            expected.add(kv23);
            expected.add(kv22);

            Scan scan = new Scan(row1);
            scan.addFamily(fam1);
            scan.setMaxVersions(MAX_VERSIONS);
            List<Cell> actual = new ArrayList<Cell>();
            InternalScanner scanner = region.getScanner(scan);

            boolean hasNext = scanner.next(actual);
            assertEquals(false, hasNext);

            // Verify result
            for (int i = 0; i < expected.size(); i++) {
                assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testScanner_StopRow1542() throws IOException {
        byte[] family = Bytes.toBytes("testFamily");
        this.region = initHRegion(tableName, getName(), CONF, family);
        try {
            byte[] row1 = Bytes.toBytes("row111");
            byte[] row2 = Bytes.toBytes("row222");
            byte[] row3 = Bytes.toBytes("row333");
            byte[] row4 = Bytes.toBytes("row444");
            byte[] row5 = Bytes.toBytes("row555");

            byte[] col1 = Bytes.toBytes("Pub111");
            byte[] col2 = Bytes.toBytes("Pub222");

            Put put = new Put(row1);
            put.add(family, col1, Bytes.toBytes(10L));
            region.put(put);

            put = new Put(row2);
            put.add(family, col1, Bytes.toBytes(15L));
            region.put(put);

            put = new Put(row3);
            put.add(family, col2, Bytes.toBytes(20L));
            region.put(put);

            put = new Put(row4);
            put.add(family, col2, Bytes.toBytes(30L));
            region.put(put);

            put = new Put(row5);
            put.add(family, col1, Bytes.toBytes(40L));
            region.put(put);

            Scan scan = new Scan(row3, row4);
            scan.setMaxVersions();
            scan.addColumn(family, col1);
            InternalScanner s = region.getScanner(scan);

            List<Cell> results = new ArrayList<Cell>();
            assertEquals(false, s.next(results));
            assertEquals(0, results.size());
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] qf1 = Bytes.toBytes("qualifier1");
        byte[] qf2 = Bytes.toBytes("quateslifier2");

        long ts1 = 1;
        long ts2 = ts1 + 1;
        long ts3 = ts1 + 2;
        long ts4 = ts1 + 3;

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, CONF, fam1);
        try {
            // Putting data in Region
            KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
            KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
            KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
            KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);

            KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
            KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
            KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
            KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);

            Put put = null;
            put = new Put(row1);
            put.add(kv14);
            put.add(kv24);
            region.put(put);
            region.flushcache();

            put = new Put(row1);
            put.add(kv23);
            put.add(kv13);
            region.put(put);
            region.flushcache();

            put = new Put(row1);
            put.add(kv22);
            put.add(kv12);
            region.put(put);
            region.flushcache();

            put = new Put(row1);
            put.add(kv21);
            put.add(kv11);
            region.put(put);

            // Expected
            List<KeyValue> expected = new ArrayList<KeyValue>();
            expected.add(kv14);
            expected.add(kv13);
            expected.add(kv12);
            expected.add(kv24);
            expected.add(kv23);
            expected.add(kv22);

            Scan scan = new Scan(row1);
            int versions = 3;
            scan.setMaxVersions(versions);
            List<Cell> actual = new ArrayList<Cell>();
            InternalScanner scanner = region.getScanner(scan);

            boolean hasNext = scanner.next(actual);
            assertEquals(false, hasNext);

            // Verify result
            for (int i = 0; i < expected.size(); i++) {
                assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    /**
     * Added for HBASE-5416
     * 
     * Here we test scan optimization when only subset of CFs are used in filter
     * conditions.
     */
    @Test
    public void testScanner_JoinedScanners() throws IOException {
        byte[] cf_essential = Bytes.toBytes("essential");
        byte[] cf_joined = Bytes.toBytes("joined");
        byte[] cf_alpha = Bytes.toBytes("alpha");
        this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
        try {
            byte[] row1 = Bytes.toBytes("row1");
            byte[] row2 = Bytes.toBytes("row2");
            byte[] row3 = Bytes.toBytes("row3");

            byte[] col_normal = Bytes.toBytes("d");
            byte[] col_alpha = Bytes.toBytes("a");

            byte[] filtered_val = Bytes.toBytes(3);

            Put put = new Put(row1);
            put.add(cf_essential, col_normal, Bytes.toBytes(1));
            put.add(cf_joined, col_alpha, Bytes.toBytes(1));
            region.put(put);

            put = new Put(row2);
            put.add(cf_essential, col_alpha, Bytes.toBytes(2));
            put.add(cf_joined, col_normal, Bytes.toBytes(2));
            put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
            region.put(put);

            put = new Put(row3);
            put.add(cf_essential, col_normal, filtered_val);
            put.add(cf_joined, col_normal, filtered_val);
            region.put(put);

            // Check two things:
            // 1. result list contains expected values
            // 2. result list is sorted properly

            Scan scan = new Scan();
            Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal, CompareOp.NOT_EQUAL,
                    filtered_val);
            scan.setFilter(filter);
            scan.setLoadColumnFamiliesOnDemand(true);
            InternalScanner s = region.getScanner(scan);

            List<Cell> results = new ArrayList<Cell>();
            assertTrue(s.next(results));
            assertEquals(results.size(), 1);
            results.clear();

            assertTrue(s.next(results));
            assertEquals(results.size(), 3);
            assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
            assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
            assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
            results.clear();

            assertFalse(s.next(results));
            assertEquals(results.size(), 0);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    /**
     * HBASE-5416
     * 
     * Test case when scan limits amount of KVs returned on each next() call.
     */
    @Test
    public void testScanner_JoinedScannersWithLimits() throws IOException {
        final byte[] cf_first = Bytes.toBytes("first");
        final byte[] cf_second = Bytes.toBytes("second");

        this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
        try {
            final byte[] col_a = Bytes.toBytes("a");
            final byte[] col_b = Bytes.toBytes("b");

            Put put;

            for (int i = 0; i < 10; i++) {
                put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
                put.add(cf_first, col_a, Bytes.toBytes(i));
                if (i < 5) {
                    put.add(cf_first, col_b, Bytes.toBytes(i));
                    put.add(cf_second, col_a, Bytes.toBytes(i));
                    put.add(cf_second, col_b, Bytes.toBytes(i));
                }
                region.put(put);
            }

            Scan scan = new Scan();
            scan.setLoadColumnFamiliesOnDemand(true);
            Filter bogusFilter = new FilterBase() {
                @Override
                public ReturnCode filterKeyValue(Cell ignored) throws IOException {
                    return ReturnCode.INCLUDE;
                }

                @Override
                public boolean isFamilyEssential(byte[] name) {
                    return Bytes.equals(name, cf_first);
                }
            };

            scan.setFilter(bogusFilter);
            InternalScanner s = region.getScanner(scan);

            // Our data looks like this:
            // r0: first:a, first:b, second:a, second:b
            // r1: first:a, first:b, second:a, second:b
            // r2: first:a, first:b, second:a, second:b
            // r3: first:a, first:b, second:a, second:b
            // r4: first:a, first:b, second:a, second:b
            // r5: first:a
            // r6: first:a
            // r7: first:a
            // r8: first:a
            // r9: first:a

            // But due to next's limit set to 3, we should get this:
            // r0: first:a, first:b, second:a
            // r0: second:b
            // r1: first:a, first:b, second:a
            // r1: second:b
            // r2: first:a, first:b, second:a
            // r2: second:b
            // r3: first:a, first:b, second:a
            // r3: second:b
            // r4: first:a, first:b, second:a
            // r4: second:b
            // r5: first:a
            // r6: first:a
            // r7: first:a
            // r8: first:a
            // r9: first:a

            List<Cell> results = new ArrayList<Cell>();
            int index = 0;
            while (true) {
                boolean more = s.next(results, 3);
                if ((index >> 1) < 5) {
                    if (index % 2 == 0)
                        assertEquals(results.size(), 3);
                    else
                        assertEquals(results.size(), 1);
                } else
                    assertEquals(results.size(), 1);
                results.clear();
                index++;
                if (!more)
                    break;
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    // ////////////////////////////////////////////////////////////////////////////
    // Split test
    // ////////////////////////////////////////////////////////////////////////////
    /**
     * Splits twice and verifies getting from each of the split regions.
     * 
     * @throws Exception
     */
    @Test
    public void testBasicSplit() throws Exception {
        byte[][] families = { fam1, fam2, fam3 };

        Configuration hc = initSplit();
        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, hc, families);

        try {
            LOG.info("" + HBaseTestCase.addContent(region, fam3));
            region.flushcache();
            region.compactStores();
            byte[] splitRow = region.checkSplit();
            assertNotNull(splitRow);
            LOG.info("SplitRow: " + Bytes.toString(splitRow));
            HRegion[] regions = splitRegion(region, splitRow);
            try {
                // Need to open the regions.
                // TODO: Add an 'open' to HRegion... don't do open by constructing
                // instance.
                for (int i = 0; i < regions.length; i++) {
                    regions[i] = HRegion.openHRegion(regions[i], null);
                }
                // Assert can get rows out of new regions. Should be able to get first
                // row from first region and the midkey from second region.
                assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
                assertGet(regions[1], fam3, splitRow);
                // Test I can get scanner and that it starts at right place.
                assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
                assertScan(regions[1], fam3, splitRow);
                // Now prove can't split regions that have references.
                for (int i = 0; i < regions.length; i++) {
                    // Add so much data to this region, we create a store file that is >
                    // than one of our unsplitable references. it will.
                    for (int j = 0; j < 2; j++) {
                        HBaseTestCase.addContent(regions[i], fam3);
                    }
                    HBaseTestCase.addContent(regions[i], fam2);
                    HBaseTestCase.addContent(regions[i], fam1);
                    regions[i].flushcache();
                }

                byte[][] midkeys = new byte[regions.length][];
                // To make regions splitable force compaction.
                for (int i = 0; i < regions.length; i++) {
                    regions[i].compactStores();
                    midkeys[i] = regions[i].checkSplit();
                }

                TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
                // Split these two daughter regions so then I'll have 4 regions. Will
                // split because added data above.
                for (int i = 0; i < regions.length; i++) {
                    HRegion[] rs = null;
                    if (midkeys[i] != null) {
                        rs = splitRegion(regions[i], midkeys[i]);
                        for (int j = 0; j < rs.length; j++) {
                            sortedMap.put(Bytes.toString(rs[j].getRegionName()), HRegion.openHRegion(rs[j], null));
                        }
                    }
                }
                LOG.info("Made 4 regions");
                // The splits should have been even. Test I can get some arbitrary row
                // out of each.
                int interval = (LAST_CHAR - FIRST_CHAR) / 3;
                byte[] b = Bytes.toBytes(START_KEY);
                for (HRegion r : sortedMap.values()) {
                    assertGet(r, fam3, b);
                    b[0] += interval;
                }
            } finally {
                for (int i = 0; i < regions.length; i++) {
                    try {
                        regions[i].close();
                    } catch (IOException e) {
                        // Ignore.
                    }
                }
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testSplitRegion() throws IOException {
        byte[] qualifier = Bytes.toBytes("qualifier");
        Configuration hc = initSplit();
        int numRows = 10;
        byte[][] families = { fam1, fam3 };

        // Setting up region
        String method = this.getName();
        this.region = initHRegion(tableName, method, hc, families);

        // Put data in region
        int startRow = 100;
        putData(startRow, numRows, qualifier, families);
        int splitRow = startRow + numRows;
        putData(splitRow, numRows, qualifier, families);
        region.flushcache();

        HRegion[] regions = null;
        try {
            regions = splitRegion(region, Bytes.toBytes("" + splitRow));
            // Opening the regions returned.
            for (int i = 0; i < regions.length; i++) {
                regions[i] = HRegion.openHRegion(regions[i], null);
            }
            // Verifying that the region has been split
            assertEquals(2, regions.length);

            // Verifying that all data is still there and that data is in the right
            // place
            verifyData(regions[0], startRow, numRows, qualifier, families);
            verifyData(regions[1], splitRow, numRows, qualifier, families);

        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    /**
     * Flushes the cache in a thread while scanning. The tests verify that the
     * scan is coherent - e.g. the returned results are always of the same or
     * later update as the previous results.
     * 
     * @throws IOException
     *           scan / compact
     * @throws InterruptedException
     *           thread join
     */
    @Test
    public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
        byte[] family = Bytes.toBytes("family");
        int numRows = 1000;
        int flushAndScanInterval = 10;
        int compactInterval = 10 * flushAndScanInterval;

        String method = "testFlushCacheWhileScanning";
        this.region = initHRegion(tableName, method, CONF, family);
        try {
            FlushThread flushThread = new FlushThread();
            flushThread.start();

            Scan scan = new Scan();
            scan.addFamily(family);
            scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
                    new BinaryComparator(Bytes.toBytes(5L))));

            int expectedCount = 0;
            List<Cell> res = new ArrayList<Cell>();

            boolean toggle = true;
            for (long i = 0; i < numRows; i++) {
                Put put = new Put(Bytes.toBytes(i));
                put.setDurability(Durability.SKIP_WAL);
                put.add(family, qual1, Bytes.toBytes(i % 10));
                region.put(put);

                if (i != 0 && i % compactInterval == 0) {
                    // System.out.println("iteration = " + i);
                    region.compactStores(true);
                }

                if (i % 10 == 5L) {
                    expectedCount++;
                }

                if (i != 0 && i % flushAndScanInterval == 0) {
                    res.clear();
                    InternalScanner scanner = region.getScanner(scan);
                    if (toggle) {
                        flushThread.flush();
                    }
                    while (scanner.next(res))
                        ;
                    if (!toggle) {
                        flushThread.flush();
                    }
                    assertEquals("i=" + i, expectedCount, res.size());
                    toggle = !toggle;
                }
            }

            flushThread.done();
            flushThread.join();
            flushThread.checkNoError();
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    protected class FlushThread extends Thread {
        private volatile boolean done;
        private Throwable error = null;

        public void done() {
            done = true;
            synchronized (this) {
                interrupt();
            }
        }

        public void checkNoError() {
            if (error != null) {
                assertNull(error);
            }
        }

        @Override
        public void run() {
            done = false;
            while (!done) {
                synchronized (this) {
                    try {
                        wait();
                    } catch (InterruptedException ignored) {
                        if (done) {
                            break;
                        }
                    }
                }
                try {
                    region.flushcache();
                } catch (IOException e) {
                    if (!done) {
                        LOG.error("Error while flusing cache", e);
                        error = e;
                    }
                    break;
                }
            }

        }

        public void flush() {
            synchronized (this) {
                notify();
            }

        }
    }

    /**
     * Writes very wide records and scans for the latest every time.. Flushes and
     * compacts the region every now and then to keep things realistic.
     * 
     * @throws IOException
     *           by flush / scan / compaction
     * @throws InterruptedException
     *           when joining threads
     */
    @Test
    public void testWritesWhileScanning() throws IOException, InterruptedException {
        int testCount = 100;
        int numRows = 1;
        int numFamilies = 10;
        int numQualifiers = 100;
        int flushInterval = 7;
        int compactInterval = 5 * flushInterval;
        byte[][] families = new byte[numFamilies][];
        for (int i = 0; i < numFamilies; i++) {
            families[i] = Bytes.toBytes("family" + i);
        }
        byte[][] qualifiers = new byte[numQualifiers][];
        for (int i = 0; i < numQualifiers; i++) {
            qualifiers[i] = Bytes.toBytes("qual" + i);
        }

        String method = "testWritesWhileScanning";
        this.region = initHRegion(tableName, method, CONF, families);
        try {
            PutThread putThread = new PutThread(numRows, families, qualifiers);
            putThread.start();
            putThread.waitForFirstPut();

            FlushThread flushThread = new FlushThread();
            flushThread.start();

            Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));

            int expectedCount = numFamilies * numQualifiers;
            List<Cell> res = new ArrayList<Cell>();

            long prevTimestamp = 0L;
            for (int i = 0; i < testCount; i++) {

                if (i != 0 && i % compactInterval == 0) {
                    region.compactStores(true);
                }

                if (i != 0 && i % flushInterval == 0) {
                    flushThread.flush();
                }

                boolean previousEmpty = res.isEmpty();
                res.clear();
                InternalScanner scanner = region.getScanner(scan);
                while (scanner.next(res))
                    ;
                if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
                    assertEquals("i=" + i, expectedCount, res.size());
                    long timestamp = res.get(0).getTimestamp();
                    assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
                            timestamp >= prevTimestamp);
                    prevTimestamp = timestamp;
                }
            }

            putThread.done();

            region.flushcache();

            putThread.join();
            putThread.checkNoError();

            flushThread.done();
            flushThread.join();
            flushThread.checkNoError();
        } finally {
            try {
                HRegion.closeHRegion(this.region);
            } catch (DroppedSnapshotException dse) {
                // We could get this on way out because we interrupt the background flusher and it could
                // fail anywhere causing a DSE over in the background flusher... only it is not properly
                // dealt with so could still be memory hanging out when we get to here -- memory we can't
                // flush because the accounting is 'off' since original DSE.
            }
            this.region = null;
        }
    }

    protected class PutThread extends Thread {
        private volatile boolean done;
        private volatile int numPutsFinished = 0;

        private Throwable error = null;
        private int numRows;
        private byte[][] families;
        private byte[][] qualifiers;

        private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
            this.numRows = numRows;
            this.families = families;
            this.qualifiers = qualifiers;
        }

        /**
         * Block until this thread has put at least one row.
         */
        public void waitForFirstPut() throws InterruptedException {
            // wait until put thread actually puts some data
            while (numPutsFinished == 0) {
                checkNoError();
                Thread.sleep(50);
            }
        }

        public void done() {
            done = true;
            synchronized (this) {
                interrupt();
            }
        }

        public void checkNoError() {
            if (error != null) {
                assertNull(error);
            }
        }

        @Override
        public void run() {
            done = false;
            while (!done) {
                try {
                    for (int r = 0; r < numRows; r++) {
                        byte[] row = Bytes.toBytes("row" + r);
                        Put put = new Put(row);
                        put.setDurability(Durability.SKIP_WAL);
                        byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
                        for (byte[] family : families) {
                            for (byte[] qualifier : qualifiers) {
                                put.add(family, qualifier, (long) numPutsFinished, value);
                            }
                        }
                        region.put(put);
                        numPutsFinished++;
                        if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
                            System.out.println("put iteration = " + numPutsFinished);
                            Delete delete = new Delete(row, (long) numPutsFinished - 30);
                            region.delete(delete);
                        }
                        numPutsFinished++;
                    }
                } catch (InterruptedIOException e) {
                    // This is fine. It means we are done, or didn't get the lock on time
                } catch (IOException e) {
                    LOG.error("error while putting records", e);
                    error = e;
                    break;
                }
            }

        }

    }

    /**
     * Writes very wide records and gets the latest row every time.. Flushes and
     * compacts the region aggressivly to catch issues.
     * 
     * @throws IOException
     *           by flush / scan / compaction
     * @throws InterruptedException
     *           when joining threads
     */
    @Test
    public void testWritesWhileGetting() throws Exception {
        int testCount = 100;
        int numRows = 1;
        int numFamilies = 10;
        int numQualifiers = 100;
        int compactInterval = 100;
        byte[][] families = new byte[numFamilies][];
        for (int i = 0; i < numFamilies; i++) {
            families[i] = Bytes.toBytes("family" + i);
        }
        byte[][] qualifiers = new byte[numQualifiers][];
        for (int i = 0; i < numQualifiers; i++) {
            qualifiers[i] = Bytes.toBytes("qual" + i);
        }

        String method = "testWritesWhileGetting";
        // This test flushes constantly and can cause many files to be created,
        // possibly
        // extending over the ulimit. Make sure compactions are aggressive in
        // reducing
        // the number of HFiles created.
        Configuration conf = HBaseConfiguration.create(CONF);
        conf.setInt("hbase.hstore.compaction.min", 1);
        conf.setInt("hbase.hstore.compaction.max", 1000);
        this.region = initHRegion(tableName, method, conf, families);
        PutThread putThread = null;
        MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
        try {
            putThread = new PutThread(numRows, families, qualifiers);
            putThread.start();
            putThread.waitForFirstPut();

            // Add a thread that flushes as fast as possible
            ctx.addThread(new RepeatingTestThread(ctx) {
                private int flushesSinceCompact = 0;
                private final int maxFlushesSinceCompact = 20;

                @Override
                public void doAnAction() throws Exception {
                    if (region.flushcache().isCompactionNeeded()) {
                        ++flushesSinceCompact;
                    }
                    // Compact regularly to avoid creating too many files and exceeding
                    // the ulimit.
                    if (flushesSinceCompact == maxFlushesSinceCompact) {
                        region.compactStores(false);
                        flushesSinceCompact = 0;
                    }
                }
            });
            ctx.startThreads();

            Get get = new Get(Bytes.toBytes("row0"));
            Result result = null;

            int expectedCount = numFamilies * numQualifiers;

            long prevTimestamp = 0L;
            for (int i = 0; i < testCount; i++) {

                boolean previousEmpty = result == null || result.isEmpty();
                result = region.get(get);
                if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
                    assertEquals("i=" + i, expectedCount, result.size());
                    // TODO this was removed, now what dangit?!
                    // search looking for the qualifier in question?
                    long timestamp = 0;
                    for (Cell kv : result.rawCells()) {
                        if (CellUtil.matchingFamily(kv, families[0])
                                && CellUtil.matchingQualifier(kv, qualifiers[0])) {
                            timestamp = kv.getTimestamp();
                        }
                    }
                    assertTrue(timestamp >= prevTimestamp);
                    prevTimestamp = timestamp;
                    Cell previousKV = null;

                    for (Cell kv : result.rawCells()) {
                        byte[] thisValue = CellUtil.cloneValue(kv);
                        if (previousKV != null) {
                            if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
                                LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
                                        + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
                                        + "(memStoreTS:" + kv.getMvccVersion() + ")");
                                assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
                            }
                        }
                        previousKV = kv;
                    }
                }
            }
        } finally {
            if (putThread != null)
                putThread.done();

            region.flushcache();

            if (putThread != null) {
                putThread.join();
                putThread.checkNoError();
            }

            ctx.stop();
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testHolesInMeta() throws Exception {
        byte[] family = Bytes.toBytes("family");
        this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF, false, family);
        try {
            byte[] rowNotServed = Bytes.toBytes("a");
            Get g = new Get(rowNotServed);
            try {
                region.get(g);
                fail();
            } catch (WrongRegionException x) {
                // OK
            }
            byte[] row = Bytes.toBytes("y");
            g = new Get(row);
            region.get(g);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testIndexesScanWithOneDeletedRow() throws IOException {
        byte[] family = Bytes.toBytes("family");

        // Setting up region
        String method = "testIndexesScanWithOneDeletedRow";
        this.region = initHRegion(tableName, method, CONF, family);
        try {
            Put put = new Put(Bytes.toBytes(1L));
            put.add(family, qual1, 1L, Bytes.toBytes(1L));
            region.put(put);

            region.flushcache();

            Delete delete = new Delete(Bytes.toBytes(1L), 1L);
            region.delete(delete);

            put = new Put(Bytes.toBytes(2L));
            put.add(family, qual1, 2L, Bytes.toBytes(2L));
            region.put(put);

            Scan idxScan = new Scan();
            idxScan.addFamily(family);
            idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,
                    Arrays.<Filter>asList(
                            new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
                                    new BinaryComparator(Bytes.toBytes(0L))),
                            new SingleColumnValueFilter(family, qual1, CompareOp.LESS_OR_EQUAL,
                                    new BinaryComparator(Bytes.toBytes(3L))))));
            InternalScanner scanner = region.getScanner(idxScan);
            List<Cell> res = new ArrayList<Cell>();

            while (scanner.next(res))
                ;
            assertEquals(1L, res.size());
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    // ////////////////////////////////////////////////////////////////////////////
    // Bloom filter test
    // ////////////////////////////////////////////////////////////////////////////
    @Test
    public void testBloomFilterSize() throws IOException {
        byte[] fam1 = Bytes.toBytes("fam1");
        byte[] qf1 = Bytes.toBytes("col");
        byte[] val1 = Bytes.toBytes("value1");
        // Create Table
        HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
                .setBloomFilterType(BloomType.ROWCOL);

        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
        htd.addFamily(hcd);
        HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
        this.region = TEST_UTIL.createLocalHRegion(info, htd);
        try {
            int num_unique_rows = 10;
            int duplicate_multiplier = 2;
            int num_storefiles = 4;

            int version = 0;
            for (int f = 0; f < num_storefiles; f++) {
                for (int i = 0; i < duplicate_multiplier; i++) {
                    for (int j = 0; j < num_unique_rows; j++) {
                        Put put = new Put(Bytes.toBytes("row" + j));
                        put.setDurability(Durability.SKIP_WAL);
                        put.add(fam1, qf1, version++, val1);
                        region.put(put);
                    }
                }
                region.flushcache();
            }
            // before compaction
            HStore store = (HStore) region.getStore(fam1);
            Collection<StoreFile> storeFiles = store.getStorefiles();
            for (StoreFile storefile : storeFiles) {
                StoreFile.Reader reader = storefile.getReader();
                reader.loadFileInfo();
                reader.loadBloomfilter();
                assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
                assertEquals(num_unique_rows, reader.getFilterEntries());
            }

            region.compactStores(true);

            // after compaction
            storeFiles = store.getStorefiles();
            for (StoreFile storefile : storeFiles) {
                StoreFile.Reader reader = storefile.getReader();
                reader.loadFileInfo();
                reader.loadBloomfilter();
                assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
                assertEquals(num_unique_rows, reader.getFilterEntries());
            }
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testAllColumnsWithBloomFilter() throws IOException {
        byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
        byte[] FAMILY = Bytes.toBytes("family");

        // Create table
        HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
                .setBloomFilterType(BloomType.ROWCOL);
        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
        htd.addFamily(hcd);
        HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
        this.region = TEST_UTIL.createLocalHRegion(info, htd);
        try {
            // For row:0, col:0: insert versions 1 through 5.
            byte row[] = Bytes.toBytes("row:" + 0);
            byte column[] = Bytes.toBytes("column:" + 0);
            Put put = new Put(row);
            put.setDurability(Durability.SKIP_WAL);
            for (long idx = 1; idx <= 4; idx++) {
                put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
            }
            region.put(put);

            // Flush
            region.flushcache();

            // Get rows
            Get get = new Get(row);
            get.setMaxVersions();
            Cell[] kvs = region.get(get).rawCells();

            // Check if rows are correct
            assertEquals(4, kvs.length);
            checkOneCell(kvs[0], FAMILY, 0, 0, 4);
            checkOneCell(kvs[1], FAMILY, 0, 0, 3);
            checkOneCell(kvs[2], FAMILY, 0, 0, 2);
            checkOneCell(kvs[3], FAMILY, 0, 0, 1);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    /**
     * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when
     * issuing delete row on columns with bloom filter set to row+col
     * (BloomType.ROWCOL)
     */
    @Test
    public void testDeleteRowWithBloomFilter() throws IOException {
        byte[] familyName = Bytes.toBytes("familyName");

        // Create Table
        HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
                .setBloomFilterType(BloomType.ROWCOL);

        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
        htd.addFamily(hcd);
        HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
        this.region = TEST_UTIL.createLocalHRegion(info, htd);
        try {
            // Insert some data
            byte row[] = Bytes.toBytes("row1");
            byte col[] = Bytes.toBytes("col1");

            Put put = new Put(row);
            put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
            region.put(put);
            region.flushcache();

            Delete del = new Delete(row);
            region.delete(del);
            region.flushcache();

            // Get remaining rows (should have none)
            Get get = new Get(row);
            get.addColumn(familyName, col);

            Cell[] keyValues = region.get(get).rawCells();
            assertTrue(keyValues.length == 0);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    @Test
    public void testgetHDFSBlocksDistribution() throws Exception {
        HBaseTestingUtility htu = new HBaseTestingUtility();
        // Why do we set the block size in this test?  If we set it smaller than the kvs, then we'll
        // break up the file in to more pieces that can be distributed across the three nodes and we
        // won't be able to have the condition this test asserts; that at least one node has
        // a copy of all replicas -- if small block size, then blocks are spread evenly across the
        // the three nodes.  hfilev3 with tags seems to put us over the block size.  St.Ack.
        // final int DEFAULT_BLOCK_SIZE = 1024;
        // htu.getConfiguration().setLong("dfs.blocksize", DEFAULT_BLOCK_SIZE);
        htu.getConfiguration().setInt("dfs.replication", 2);

        // set up a cluster with 3 nodes
        MiniHBaseCluster cluster = null;
        String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
        int regionServersCount = 3;

        try {
            cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
            byte[][] families = { fam1, fam2 };
            HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);

            // Setting up region
            byte row[] = Bytes.toBytes("row1");
            byte col[] = Bytes.toBytes("col1");

            Put put = new Put(row);
            put.add(fam1, col, 1, Bytes.toBytes("test1"));
            put.add(fam2, col, 1, Bytes.toBytes("test2"));
            ht.put(put);

            HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName())).get(0);
            firstRegion.flushcache();
            HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();

            // Given the default replication factor is 2 and we have 2 HFiles,
            // we will have total of 4 replica of blocks on 3 datanodes; thus there
            // must be at least one host that have replica for 2 HFiles. That host's
            // weight will be equal to the unique block weight.
            long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
            StringBuilder sb = new StringBuilder();
            for (String host : blocksDistribution1.getTopHosts()) {
                if (sb.length() > 0)
                    sb.append(", ");
                sb.append(host);
                sb.append("=");
                sb.append(blocksDistribution1.getWeight(host));
            }

            String topHost = blocksDistribution1.getTopHosts().get(0);
            long topHostWeight = blocksDistribution1.getWeight(topHost);
            String msg = "uniqueBlocksWeight=" + uniqueBlocksWeight1 + ", topHostWeight=" + topHostWeight
                    + ", topHost=" + topHost + "; " + sb.toString();
            LOG.info(msg);
            assertTrue(msg, uniqueBlocksWeight1 == topHostWeight);

            // use the static method to compute the value, it should be the same.
            // static method is used by load balancer or other components
            HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
                    htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
            long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();

            assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);

            ht.close();
        } finally {
            if (cluster != null) {
                htu.shutdownMiniCluster();
            }
        }
    }

    /**
     * Testcase to check state of region initialization task set to ABORTED or not
     * if any exceptions during initialization
     * 
     * @throws Exception
     */
    @Test
    public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
        TableName tableName = TableName.valueOf(name.getMethodName());
        HRegionInfo info = null;
        try {
            FileSystem fs = Mockito.mock(FileSystem.class);
            Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
            HTableDescriptor htd = new HTableDescriptor(tableName);
            htd.addFamily(new HColumnDescriptor("cf"));
            info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY,
                    false);
            Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
            region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
            // region initialization throws IOException and set task state to ABORTED.
            region.initialize();
            fail("Region initialization should fail due to IOException");
        } catch (IOException io) {
            List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
            for (MonitoredTask monitoredTask : tasks) {
                if (!(monitoredTask instanceof MonitoredRPCHandler)
                        && monitoredTask.getDescription().contains(region.toString())) {
                    assertTrue("Region state should be ABORTED.",
                            monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
                    break;
                }
            }
        } finally {
            HRegion.closeHRegion(region);
        }
    }

    /**
     * Verifies that the .regioninfo file is written on region creation and that
     * is recreated if missing during region opening.
     */
    @Test
    public void testRegionInfoFileCreation() throws IOException {
        Path rootDir = new Path(dir + "testRegionInfoFileCreation");

        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
        htd.addFamily(new HColumnDescriptor("cf"));

        HRegionInfo hri = new HRegionInfo(htd.getTableName());

        // Create a region and skip the initialization (like CreateTableHandler)
        HRegion region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
        //    HRegion region = TEST_UTIL.createLocalHRegion(hri, htd);
        Path regionDir = region.getRegionFileSystem().getRegionDir();
        FileSystem fs = region.getRegionFileSystem().getFileSystem();
        HRegion.closeHRegion(region);

        Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);

        // Verify that the .regioninfo file is present
        assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
                fs.exists(regionInfoFile));

        // Try to open the region
        region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
        assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
        HRegion.closeHRegion(region);

        // Verify that the .regioninfo file is still there
        assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
                fs.exists(regionInfoFile));

        // Remove the .regioninfo file and verify is recreated on region open
        fs.delete(regionInfoFile, true);
        assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
                fs.exists(regionInfoFile));

        region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
        //    region = TEST_UTIL.openHRegion(hri, htd);
        assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
        HRegion.closeHRegion(region);

        // Verify that the .regioninfo file is still there
        assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
                fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
    }

    /**
     * TestCase for increment
     */
    private static class Incrementer implements Runnable {
        private HRegion region;
        private final static byte[] incRow = Bytes.toBytes("incRow");
        private final static byte[] family = Bytes.toBytes("family");
        private final static byte[] qualifier = Bytes.toBytes("qualifier");
        private final static long ONE = 1l;
        private int incCounter;

        public Incrementer(HRegion region, int incCounter) {
            this.region = region;
            this.incCounter = incCounter;
        }

        @Override
        public void run() {
            int count = 0;
            while (count < incCounter) {
                Increment inc = new Increment(incRow);
                inc.addColumn(family, qualifier, ONE);
                count++;
                try {
                    region.increment(inc);
                } catch (IOException e) {
                    e.printStackTrace();
                    break;
                }
            }
        }
    }

    /**
     * Test case to check increment function with memstore flushing
     * @throws Exception
     */
    @Test
    public void testParallelIncrementWithMemStoreFlush() throws Exception {
        byte[] family = Incrementer.family;
        this.region = initHRegion(tableName, method, CONF, family);
        final HRegion region = this.region;
        final AtomicBoolean incrementDone = new AtomicBoolean(false);
        Runnable flusher = new Runnable() {
            @Override
            public void run() {
                while (!incrementDone.get()) {
                    try {
                        region.flushcache();
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            }
        };

        // after all increment finished, the row will increment to 20*100 = 2000
        int threadNum = 20;
        int incCounter = 100;
        long expected = threadNum * incCounter;
        Thread[] incrementers = new Thread[threadNum];
        Thread flushThread = new Thread(flusher);
        for (int i = 0; i < threadNum; i++) {
            incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
            incrementers[i].start();
        }
        flushThread.start();
        for (int i = 0; i < threadNum; i++) {
            incrementers[i].join();
        }

        incrementDone.set(true);
        flushThread.join();

        Get get = new Get(Incrementer.incRow);
        get.addColumn(Incrementer.family, Incrementer.qualifier);
        get.setMaxVersions(1);
        Result res = this.region.get(get);
        List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);

        // we just got the latest version
        assertEquals(kvs.size(), 1);
        Cell kv = kvs.get(0);
        assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
        this.region = null;
    }

    /**
     * TestCase for append
     */
    private static class Appender implements Runnable {
        private HRegion region;
        private final static byte[] appendRow = Bytes.toBytes("appendRow");
        private final static byte[] family = Bytes.toBytes("family");
        private final static byte[] qualifier = Bytes.toBytes("qualifier");
        private final static byte[] CHAR = Bytes.toBytes("a");
        private int appendCounter;

        public Appender(HRegion region, int appendCounter) {
            this.region = region;
            this.appendCounter = appendCounter;
        }

        @Override
        public void run() {
            int count = 0;
            while (count < appendCounter) {
                Append app = new Append(appendRow);
                app.add(family, qualifier, CHAR);
                count++;
                try {
                    region.append(app);
                } catch (IOException e) {
                    e.printStackTrace();
                    break;
                }
            }
        }
    }

    /**
     * Test case to check append function with memstore flushing
     * @throws Exception
     */
    @Test
    public void testParallelAppendWithMemStoreFlush() throws Exception {
        byte[] family = Appender.family;
        this.region = initHRegion(tableName, method, CONF, family);
        final HRegion region = this.region;
        final AtomicBoolean appendDone = new AtomicBoolean(false);
        Runnable flusher = new Runnable() {
            @Override
            public void run() {
                while (!appendDone.get()) {
                    try {
                        region.flushcache();
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            }
        };

        // after all append finished, the value will append to threadNum *
        // appendCounter Appender.CHAR
        int threadNum = 20;
        int appendCounter = 100;
        byte[] expected = new byte[threadNum * appendCounter];
        for (int i = 0; i < threadNum * appendCounter; i++) {
            System.arraycopy(Appender.CHAR, 0, expected, i, 1);
        }
        Thread[] appenders = new Thread[threadNum];
        Thread flushThread = new Thread(flusher);
        for (int i = 0; i < threadNum; i++) {
            appenders[i] = new Thread(new Appender(this.region, appendCounter));
            appenders[i].start();
        }
        flushThread.start();
        for (int i = 0; i < threadNum; i++) {
            appenders[i].join();
        }

        appendDone.set(true);
        flushThread.join();

        Get get = new Get(Appender.appendRow);
        get.addColumn(Appender.family, Appender.qualifier);
        get.setMaxVersions(1);
        Result res = this.region.get(get);
        List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);

        // we just got the latest version
        assertEquals(kvs.size(), 1);
        Cell kv = kvs.get(0);
        byte[] appendResult = new byte[kv.getValueLength()];
        System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
        assertArrayEquals(expected, appendResult);
        this.region = null;
    }

    /**
     * Test case to check put function with memstore flushing for same row, same ts
     * @throws Exception
     */
    @Test
    public void testPutWithMemStoreFlush() throws Exception {
        byte[] family = Bytes.toBytes("family");
        ;
        byte[] qualifier = Bytes.toBytes("qualifier");
        byte[] row = Bytes.toBytes("putRow");
        byte[] value = null;
        this.region = initHRegion(tableName, method, CONF, family);
        Put put = null;
        Get get = null;
        List<Cell> kvs = null;
        Result res = null;

        put = new Put(row);
        value = Bytes.toBytes("value0");
        put.add(family, qualifier, 1234567l, value);
        region.put(put);
        get = new Get(row);
        get.addColumn(family, qualifier);
        get.setMaxVersions();
        res = this.region.get(get);
        kvs = res.getColumnCells(family, qualifier);
        assertEquals(1, kvs.size());
        assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));

        region.flushcache();
        get = new Get(row);
        get.addColumn(family, qualifier);
        get.setMaxVersions();
        res = this.region.get(get);
        kvs = res.getColumnCells(family, qualifier);
        assertEquals(1, kvs.size());
        assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));

        put = new Put(row);
        value = Bytes.toBytes("value1");
        put.add(family, qualifier, 1234567l, value);
        region.put(put);
        get = new Get(row);
        get.addColumn(family, qualifier);
        get.setMaxVersions();
        res = this.region.get(get);
        kvs = res.getColumnCells(family, qualifier);
        assertEquals(1, kvs.size());
        assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));

        region.flushcache();
        get = new Get(row);
        get.addColumn(family, qualifier);
        get.setMaxVersions();
        res = this.region.get(get);
        kvs = res.getColumnCells(family, qualifier);
        assertEquals(1, kvs.size());
        assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
    }

    @Test
    public void testDurability() throws Exception {
        String method = "testDurability";
        // there are 5 x 5 cases:
        // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation
        // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT)

        // expected cases for append and sync wal
        durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
        durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
        durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);

        durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
        durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
        durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);

        durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
        durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);

        durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
        durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);

        durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
        durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
        durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);

        // expected cases for async wal
        durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
        durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
        durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
        durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
        durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
        durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);

        durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
        durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
        durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
        durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
        durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
        durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);

        // expect skip wal cases
        durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
        durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
        durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
        durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
        durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
        durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);

    }

    private void durabilityTest(String method, Durability tableDurability, Durability mutationDurability,
            long timeout, boolean expectAppend, final boolean expectSync, final boolean expectSyncFromLogSyncer)
            throws Exception {
        Configuration conf = HBaseConfiguration.create(CONF);
        method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
        TableName tableName = TableName.valueOf(method);
        byte[] family = Bytes.toBytes("family");
        Path logDir = new Path(new Path(dir + method), "log");
        HLog hlog = HLogFactory.createHLog(FILESYSTEM, logDir, UUID.randomUUID().toString(), conf);
        final HLog log = spy(hlog);
        this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, method,
                conf, false, tableDurability, log, new byte[][] { family });

        Put put = new Put(Bytes.toBytes("r1"));
        put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
        put.setDurability(mutationDurability);
        region.put(put);

        //verify append called or not
        verify(log, expectAppend ? times(1) : never()).appendNoSync((HTableDescriptor) any(), (HRegionInfo) any(),
                (HLogKey) any(), (WALEdit) any(), (AtomicLong) any(), Mockito.anyBoolean());

        // verify sync called or not
        if (expectSync || expectSyncFromLogSyncer) {
            TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
                @Override
                public boolean evaluate() throws Exception {
                    try {
                        if (expectSync) {
                            verify(log, times(1)).sync(anyLong()); // Hregion calls this one
                        } else if (expectSyncFromLogSyncer) {
                            verify(log, times(1)).sync(); // log syncer calls this one
                        }
                    } catch (Throwable ignore) {
                    }
                    return true;
                }
            });
        } else {
            verify(log, never()).sync(anyLong());
            verify(log, never()).sync();
        }

        HRegion.closeHRegion(this.region);
        this.region = null;
    }

    private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
        for (int i = startRow; i < startRow + numRows; i++) {
            Put put = new Put(Bytes.toBytes("" + i));
            put.setDurability(Durability.SKIP_WAL);
            for (byte[] family : families) {
                put.add(family, qf, null);
            }
            region.put(put);
        }
    }

    private void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
            throws IOException {
        for (int i = startRow; i < startRow + numRows; i++) {
            byte[] row = Bytes.toBytes("" + i);
            Get get = new Get(row);
            for (byte[] family : families) {
                get.addColumn(family, qf);
            }
            Result result = newReg.get(get);
            Cell[] raw = result.rawCells();
            assertEquals(families.length, result.size());
            for (int j = 0; j < families.length; j++) {
                assertTrue(CellUtil.matchingRow(raw[j], row));
                assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
                assertTrue(CellUtil.matchingQualifier(raw[j], qf));
            }
        }
    }

    private void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
        // Now I have k, get values out and assert they are as expected.
        Get get = new Get(k).addFamily(family).setMaxVersions();
        Cell[] results = r.get(get).rawCells();
        for (int j = 0; j < results.length; j++) {
            byte[] tmp = CellUtil.cloneValue(results[j]);
            // Row should be equal to value every time.
            assertTrue(Bytes.equals(k, tmp));
        }
    }

    /*
     * Assert first value in the passed region is <code>firstValue</code>.
     * 
     * @param r
     * 
     * @param fs
     * 
     * @param firstValue
     * 
     * @throws IOException
     */
    private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue) throws IOException {
        byte[][] families = { fs };
        Scan scan = new Scan();
        for (int i = 0; i < families.length; i++)
            scan.addFamily(families[i]);
        InternalScanner s = r.getScanner(scan);
        try {
            List<Cell> curVals = new ArrayList<Cell>();
            boolean first = true;
            OUTER_LOOP: while (s.next(curVals)) {
                for (Cell kv : curVals) {
                    byte[] val = CellUtil.cloneValue(kv);
                    byte[] curval = val;
                    if (first) {
                        first = false;
                        assertTrue(Bytes.compareTo(curval, firstValue) == 0);
                    } else {
                        // Not asserting anything. Might as well break.
                        break OUTER_LOOP;
                    }
                }
            }
        } finally {
            s.close();
        }
    }

    /**
     * Test that we get the expected flush results back
     * @throws IOException
     */
    @Test
    public void testFlushResult() throws IOException {
        String method = name.getMethodName();
        byte[] tableName = Bytes.toBytes(method);
        byte[] family = Bytes.toBytes("family");

        this.region = initHRegion(tableName, method, family);

        // empty memstore, flush doesn't run
        HRegion.FlushResult fr = region.flushcache();
        assertFalse(fr.isFlushSucceeded());
        assertFalse(fr.isCompactionNeeded());

        // Flush enough files to get up to the threshold, doesn't need compactions
        for (int i = 0; i < 2; i++) {
            Put put = new Put(tableName).add(family, family, tableName);
            region.put(put);
            fr = region.flushcache();
            assertTrue(fr.isFlushSucceeded());
            assertFalse(fr.isCompactionNeeded());
        }

        // Two flushes after the threshold, compactions are needed
        for (int i = 0; i < 2; i++) {
            Put put = new Put(tableName).add(family, family, tableName);
            region.put(put);
            fr = region.flushcache();
            assertTrue(fr.isFlushSucceeded());
            assertTrue(fr.isCompactionNeeded());
        }
    }

    private Configuration initSplit() {
        // Always compact if there is more than one store file.
        CONF.setInt("hbase.hstore.compactionThreshold", 2);

        // Make lease timeout longer, lease checks less frequent
        CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);

        CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);

        // Increase the amount of time between client retries
        CONF.setLong("hbase.client.pause", 15 * 1000);

        // This size should make it so we always split using the addContent
        // below. After adding all data, the first region is 1.3M
        CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
        return CONF;
    }

    /**
     * @param tableName
     * @param callingMethod
     * @param conf
     * @param families
     * @throws IOException
     * @return A region on which you must call
     *         {@link HRegion#closeHRegion(HRegion)} when done.
     */
    public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
            byte[]... families) throws IOException {
        return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
    }

    /**
     * @param tableName
     * @param callingMethod
     * @param conf
     * @param families
     * @throws IOException
     * @return A region on which you must call
     *         {@link HRegion#closeHRegion(HRegion)} when done.
     */
    public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
            byte[]... families) throws IOException {
        return initHRegion(tableName, null, null, callingMethod, conf, false, families);
    }

    /**
     * @param tableName
     * @param callingMethod
     * @param conf
     * @param isReadOnly
     * @param families
     * @throws IOException
     * @return A region on which you must call
     *         {@link HRegion#closeHRegion(HRegion)} when done.
     */
    public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
            boolean isReadOnly, byte[]... families) throws IOException {
        return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
    }

    private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey, String callingMethod,
            Configuration conf, boolean isReadOnly, byte[]... families) throws IOException {
        return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly, Durability.SYNC_WAL, null,
                families);
    }

    /**
     * @param tableName
     * @param startKey
     * @param stopKey
     * @param callingMethod
     * @param conf
     * @param isReadOnly
     * @param families
     * @throws IOException
     * @return A region on which you must call
     *         {@link HRegion#closeHRegion(HRegion)} when done.
     */
    private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey, String callingMethod,
            Configuration conf, boolean isReadOnly, Durability durability, HLog hlog, byte[]... families)
            throws IOException {
        return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
                durability, hlog, families);
    }

    /**
     * Assert that the passed in Cell has expected contents for the specified row,
     * column & timestamp.
     */
    private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
        String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
        assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx, Bytes.toString(CellUtil.cloneRow(kv)));
        assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
                Bytes.toString(CellUtil.cloneFamily(kv)));
        assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
                Bytes.toString(CellUtil.cloneQualifier(kv)));
        assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
        assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
                Bytes.toString(CellUtil.cloneValue(kv)));
    }

    public void testReverseScanner_FromMemStore_SingleCF_Normal() throws IOException {
        byte[] rowC = Bytes.toBytes("rowC");
        byte[] rowA = Bytes.toBytes("rowA");
        byte[] rowB = Bytes.toBytes("rowB");
        byte[] cf = Bytes.toBytes("CF");
        byte[][] families = { cf };
        byte[] col = Bytes.toBytes("C");
        long ts = 1;
        String method = this.getName();
        this.region = initHRegion(tableName, method, families);
        try {
            KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
            KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null);
            KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
            KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
            Put put = null;
            put = new Put(rowC);
            put.add(kv1);
            put.add(kv11);
            region.put(put);
            put = new Put(rowA);
            put.add(kv2);
            region.put(put);
            put = new Put(rowB);
            put.add(kv3);
            region.put(put);

            Scan scan = new Scan(rowC);
            scan.setMaxVersions(5);
            scan.setReversed(true);
            InternalScanner scanner = region.getScanner(scan);
            List<Cell> currRow = new ArrayList<Cell>();
            boolean hasNext = scanner.next(currRow);
            assertEquals(2, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
            assertTrue(hasNext);
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
            assertTrue(hasNext);
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
            assertFalse(hasNext);
            scanner.close();
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    public void testReverseScanner_FromMemStore_SingleCF_LargerKey() throws IOException {
        byte[] rowC = Bytes.toBytes("rowC");
        byte[] rowA = Bytes.toBytes("rowA");
        byte[] rowB = Bytes.toBytes("rowB");
        byte[] rowD = Bytes.toBytes("rowD");
        byte[] cf = Bytes.toBytes("CF");
        byte[][] families = { cf };
        byte[] col = Bytes.toBytes("C");
        long ts = 1;
        String method = this.getName();
        this.region = initHRegion(tableName, method, families);
        try {
            KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
            KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null);
            KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
            KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
            Put put = null;
            put = new Put(rowC);
            put.add(kv1);
            put.add(kv11);
            region.put(put);
            put = new Put(rowA);
            put.add(kv2);
            region.put(put);
            put = new Put(rowB);
            put.add(kv3);
            region.put(put);

            Scan scan = new Scan(rowD);
            List<Cell> currRow = new ArrayList<Cell>();
            scan.setReversed(true);
            scan.setMaxVersions(5);
            InternalScanner scanner = region.getScanner(scan);
            boolean hasNext = scanner.next(currRow);
            assertEquals(2, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
            assertTrue(hasNext);
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
            assertTrue(hasNext);
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
            assertFalse(hasNext);
            scanner.close();
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    public void testReverseScanner_FromMemStore_SingleCF_FullScan() throws IOException {
        byte[] rowC = Bytes.toBytes("rowC");
        byte[] rowA = Bytes.toBytes("rowA");
        byte[] rowB = Bytes.toBytes("rowB");
        byte[] cf = Bytes.toBytes("CF");
        byte[][] families = { cf };
        byte[] col = Bytes.toBytes("C");
        long ts = 1;
        String method = this.getName();
        this.region = initHRegion(tableName, method, families);
        try {
            KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
            KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null);
            KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
            KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
            Put put = null;
            put = new Put(rowC);
            put.add(kv1);
            put.add(kv11);
            region.put(put);
            put = new Put(rowA);
            put.add(kv2);
            region.put(put);
            put = new Put(rowB);
            put.add(kv3);
            region.put(put);
            Scan scan = new Scan();
            List<Cell> currRow = new ArrayList<Cell>();
            scan.setReversed(true);
            InternalScanner scanner = region.getScanner(scan);
            boolean hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
            assertTrue(hasNext);
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
            assertTrue(hasNext);
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
            assertFalse(hasNext);
            scanner.close();
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
        // case for "INCLUDE_AND_SEEK_NEXT_ROW & SEEK_NEXT_ROW" endless loop
        byte[] rowA = Bytes.toBytes("rowA");
        byte[] rowB = Bytes.toBytes("rowB");
        byte[] rowC = Bytes.toBytes("rowC");
        byte[] rowD = Bytes.toBytes("rowD");
        byte[] rowE = Bytes.toBytes("rowE");
        byte[] cf = Bytes.toBytes("CF");
        byte[][] families = { cf };
        byte[] col1 = Bytes.toBytes("col1");
        byte[] col2 = Bytes.toBytes("col2");
        long ts = 1;
        String method = this.getName();
        this.region = initHRegion(tableName, method, families);
        try {
            KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
            KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
            KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
            KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
            KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
            KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
            Put put = null;
            put = new Put(rowA);
            put.add(kv1);
            region.put(put);
            put = new Put(rowB);
            put.add(kv2);
            region.put(put);
            put = new Put(rowC);
            put.add(kv3);
            region.put(put);
            put = new Put(rowD);
            put.add(kv4_1);
            region.put(put);
            put = new Put(rowD);
            put.add(kv4_2);
            region.put(put);
            put = new Put(rowE);
            put.add(kv5);
            region.put(put);
            region.flushcache();
            Scan scan = new Scan(rowD, rowA);
            scan.addColumn(families[0], col1);
            scan.setReversed(true);
            List<Cell> currRow = new ArrayList<Cell>();
            InternalScanner scanner = region.getScanner(scan);
            boolean hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
            assertTrue(hasNext);
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
            assertTrue(hasNext);
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
            assertFalse(hasNext);
            scanner.close();

            scan = new Scan(rowD, rowA);
            scan.addColumn(families[0], col2);
            scan.setReversed(true);
            currRow.clear();
            scanner = region.getScanner(scan);
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
            scanner.close();
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    public void testReverseScanner_smaller_blocksize() throws IOException {
        // case to ensure no conflict with HFile index optimization
        byte[] rowA = Bytes.toBytes("rowA");
        byte[] rowB = Bytes.toBytes("rowB");
        byte[] rowC = Bytes.toBytes("rowC");
        byte[] rowD = Bytes.toBytes("rowD");
        byte[] rowE = Bytes.toBytes("rowE");
        byte[] cf = Bytes.toBytes("CF");
        byte[][] families = { cf };
        byte[] col1 = Bytes.toBytes("col1");
        byte[] col2 = Bytes.toBytes("col2");
        long ts = 1;
        String method = this.getName();
        HBaseConfiguration config = new HBaseConfiguration();
        config.setInt("test.block.size", 1);
        this.region = initHRegion(tableName, method, config, families);
        try {
            KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
            KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
            KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
            KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
            KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
            KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
            Put put = null;
            put = new Put(rowA);
            put.add(kv1);
            region.put(put);
            put = new Put(rowB);
            put.add(kv2);
            region.put(put);
            put = new Put(rowC);
            put.add(kv3);
            region.put(put);
            put = new Put(rowD);
            put.add(kv4_1);
            region.put(put);
            put = new Put(rowD);
            put.add(kv4_2);
            region.put(put);
            put = new Put(rowE);
            put.add(kv5);
            region.put(put);
            region.flushcache();
            Scan scan = new Scan(rowD, rowA);
            scan.addColumn(families[0], col1);
            scan.setReversed(true);
            List<Cell> currRow = new ArrayList<Cell>();
            InternalScanner scanner = region.getScanner(scan);
            boolean hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
            assertTrue(hasNext);
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
            assertTrue(hasNext);
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
            assertFalse(hasNext);
            scanner.close();

            scan = new Scan(rowD, rowA);
            scan.addColumn(families[0], col2);
            scan.setReversed(true);
            currRow.clear();
            scanner = region.getScanner(scan);
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
            scanner.close();
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1() throws IOException {
        byte[] row0 = Bytes.toBytes("row0"); // 1 kv
        byte[] row1 = Bytes.toBytes("row1"); // 2 kv
        byte[] row2 = Bytes.toBytes("row2"); // 4 kv
        byte[] row3 = Bytes.toBytes("row3"); // 2 kv
        byte[] row4 = Bytes.toBytes("row4"); // 5 kv
        byte[] row5 = Bytes.toBytes("row5"); // 2 kv
        byte[] cf1 = Bytes.toBytes("CF1");
        byte[] cf2 = Bytes.toBytes("CF2");
        byte[] cf3 = Bytes.toBytes("CF3");
        byte[][] families = { cf1, cf2, cf3 };
        byte[] col = Bytes.toBytes("C");
        long ts = 1;
        String method = this.getName();
        HBaseConfiguration conf = new HBaseConfiguration();
        // disable compactions in this test.
        conf.setInt("hbase.hstore.compactionThreshold", 10000);
        this.region = initHRegion(tableName, method, conf, families);
        try {
            // kv naming style: kv(row number) totalKvCountInThisRow seq no
            KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put, null);
            KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put, null);
            KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1, KeyValue.Type.Put, null);
            KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
            KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put, null);
            KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put, null);
            KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4, KeyValue.Type.Put, null);
            KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put, null);
            KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4, KeyValue.Type.Put, null);
            KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put, null);
            KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put, null);
            KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5, KeyValue.Type.Put, null);
            KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put, null);
            KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3, KeyValue.Type.Put, null);
            KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put, null);
            KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put, null);
            // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv)
            Put put = null;
            put = new Put(row1);
            put.add(kv1_2_1);
            region.put(put);
            put = new Put(row2);
            put.add(kv2_4_1);
            region.put(put);
            put = new Put(row4);
            put.add(kv4_5_4);
            put.add(kv4_5_5);
            region.put(put);
            region.flushcache();
            // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv)
            put = new Put(row4);
            put.add(kv4_5_1);
            put.add(kv4_5_3);
            region.put(put);
            put = new Put(row1);
            put.add(kv1_2_2);
            region.put(put);
            put = new Put(row2);
            put.add(kv2_4_4);
            region.put(put);
            region.flushcache();
            // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv)
            put = new Put(row4);
            put.add(kv4_5_2);
            region.put(put);
            put = new Put(row2);
            put.add(kv2_4_2);
            put.add(kv2_4_3);
            region.put(put);
            put = new Put(row3);
            put.add(kv3_2_2);
            region.put(put);
            region.flushcache();
            // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max)
            // ( 2 kv)
            put = new Put(row0);
            put.add(kv0_1_1);
            region.put(put);
            put = new Put(row3);
            put.add(kv3_2_1);
            region.put(put);
            put = new Put(row5);
            put.add(kv5_2_1);
            put.add(kv5_2_2);
            region.put(put);
            // scan range = ["row4", min), skip the max "row5"
            Scan scan = new Scan(row4);
            scan.setMaxVersions(5);
            scan.setBatch(3);
            scan.setReversed(true);
            InternalScanner scanner = region.getScanner(scan);
            List<Cell> currRow = new ArrayList<Cell>();
            boolean hasNext = false;
            // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not
            // included in scan range
            // "row4" takes 2 next() calls since batch=3
            hasNext = scanner.next(currRow);
            assertEquals(3, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
            assertTrue(hasNext);
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(2, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
            assertTrue(hasNext);
            // 2. scan out "row3" (2 kv)
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(2, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
            assertTrue(hasNext);
            // 3. scan out "row2" (4 kvs)
            // "row2" takes 2 next() calls since batch=3
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(3, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
            assertTrue(hasNext);
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
            assertTrue(hasNext);
            // 4. scan out "row1" (2 kv)
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(2, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
            assertTrue(hasNext);
            // 5. scan out "row0" (1 kv)
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
            assertFalse(hasNext);

            scanner.close();
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2() throws IOException {
        byte[] row1 = Bytes.toBytes("row1");
        byte[] row2 = Bytes.toBytes("row2");
        byte[] row3 = Bytes.toBytes("row3");
        byte[] row4 = Bytes.toBytes("row4");
        byte[] cf1 = Bytes.toBytes("CF1");
        byte[] cf2 = Bytes.toBytes("CF2");
        byte[] cf3 = Bytes.toBytes("CF3");
        byte[] cf4 = Bytes.toBytes("CF4");
        byte[][] families = { cf1, cf2, cf3, cf4 };
        byte[] col = Bytes.toBytes("C");
        long ts = 1;
        String method = this.getName();
        HBaseConfiguration conf = new HBaseConfiguration();
        // disable compactions in this test.
        conf.setInt("hbase.hstore.compactionThreshold", 10000);
        this.region = initHRegion(tableName, method, conf, families);
        try {
            KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
            KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
            KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
            KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
            // storefile1
            Put put = new Put(row1);
            put.add(kv1);
            region.put(put);
            region.flushcache();
            // storefile2
            put = new Put(row2);
            put.add(kv2);
            region.put(put);
            region.flushcache();
            // storefile3
            put = new Put(row3);
            put.add(kv3);
            region.put(put);
            region.flushcache();
            // memstore
            put = new Put(row4);
            put.add(kv4);
            region.put(put);
            // scan range = ["row4", min)
            Scan scan = new Scan(row4);
            scan.setReversed(true);
            scan.setBatch(10);
            InternalScanner scanner = region.getScanner(scan);
            List<Cell> currRow = new ArrayList<Cell>();
            boolean hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
            assertTrue(hasNext);
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
            assertTrue(hasNext);
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
            assertTrue(hasNext);
            currRow.clear();
            hasNext = scanner.next(currRow);
            assertEquals(1, currRow.size());
            assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
            assertFalse(hasNext);
        } finally {
            HRegion.closeHRegion(this.region);
            this.region = null;
        }
    }

    private static HRegion initHRegion(byte[] tableName, String callingMethod, byte[]... families)
            throws IOException {
        return initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families);
    }
}