org.apache.hadoop.hbase.client.TestFromClientSide.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hbase.client.TestFromClientSide.java

Source

/**
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.client;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicReference;

import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.LargeTests;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.WhileMatchFilter;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
import org.apache.hadoop.hbase.ipc.RpcClient;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.log4j.Level;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;

/**
 * Run tests that use the HBase clients; {@link HTable} and {@link HTablePool}.
 * Sets up the HBase mini cluster once at start and runs through all client tests.
 * Each creates a table named for the method and does its stuff against that.
 */
@Category(LargeTests.class)
@SuppressWarnings("deprecation")
public class TestFromClientSide {
    final Log LOG = LogFactory.getLog(getClass());
    protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
    private static byte[] ROW = Bytes.toBytes("testRow");
    private static byte[] FAMILY = Bytes.toBytes("testFamily");
    private static byte[] QUALIFIER = Bytes.toBytes("testQualifier");
    private static byte[] VALUE = Bytes.toBytes("testValue");
    protected static int SLAVES = 3;

    /**
     * @throws java.lang.Exception
     */
    @BeforeClass
    public static void setUpBeforeClass() throws Exception {
        ((Log4JLogger) RpcServer.LOG).getLogger().setLevel(Level.ALL);
        ((Log4JLogger) RpcClient.LOG).getLogger().setLevel(Level.ALL);
        ((Log4JLogger) ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
        Configuration conf = TEST_UTIL.getConfiguration();
        conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MultiRowMutationEndpoint.class.getName());
        conf.setBoolean("hbase.table.sanity.checks", true); // enable for below tests
        // We need more than one region server in this test
        TEST_UTIL.startMiniCluster(SLAVES);
    }

    /**
     * @throws java.lang.Exception
     */
    @AfterClass
    public static void tearDownAfterClass() throws Exception {
        TEST_UTIL.shutdownMiniCluster();
    }

    /**
     * @throws java.lang.Exception
     */
    @Before
    public void setUp() throws Exception {
        // Nothing to do.
    }

    /**
     * @throws java.lang.Exception
     */
    @After
    public void tearDown() throws Exception {
        // Nothing to do.
    }

    /**
     * Basic client side validation of HBASE-4536
     */
    @Test
    public void testKeepDeletedCells() throws Exception {
        final byte[] TABLENAME = Bytes.toBytes("testKeepDeletesCells");
        final byte[] FAMILY = Bytes.toBytes("family");
        final byte[] C0 = Bytes.toBytes("c0");

        final byte[] T1 = Bytes.toBytes("T1");
        final byte[] T2 = Bytes.toBytes("T2");
        final byte[] T3 = Bytes.toBytes("T3");
        HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setKeepDeletedCells(true).setMaxVersions(3);

        HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(TABLENAME));
        desc.addFamily(hcd);
        TEST_UTIL.getHBaseAdmin().createTable(desc);
        Configuration c = TEST_UTIL.getConfiguration();
        HTable h = new HTable(c, TABLENAME);

        long ts = System.currentTimeMillis();
        Put p = new Put(T1, ts);
        p.add(FAMILY, C0, T1);
        h.put(p);
        p = new Put(T1, ts + 2);
        p.add(FAMILY, C0, T2);
        h.put(p);
        p = new Put(T1, ts + 4);
        p.add(FAMILY, C0, T3);
        h.put(p);

        Delete d = new Delete(T1, ts + 3);
        h.delete(d);

        d = new Delete(T1, ts + 3);
        d.deleteColumns(FAMILY, C0, ts + 3);
        h.delete(d);

        Get g = new Get(T1);
        // does *not* include the delete
        g.setTimeRange(0, ts + 3);
        Result r = h.get(g);
        assertArrayEquals(T2, r.getValue(FAMILY, C0));

        Scan s = new Scan(T1);
        s.setTimeRange(0, ts + 3);
        s.setMaxVersions();
        ResultScanner scanner = h.getScanner(s);
        Cell[] kvs = scanner.next().rawCells();
        assertArrayEquals(T2, CellUtil.cloneValue(kvs[0]));
        assertArrayEquals(T1, CellUtil.cloneValue(kvs[1]));
        scanner.close();

        s = new Scan(T1);
        s.setRaw(true);
        s.setMaxVersions();
        scanner = h.getScanner(s);
        kvs = scanner.next().rawCells();
        assertTrue(CellUtil.isDeleteFamily(kvs[0]));
        assertArrayEquals(T3, CellUtil.cloneValue(kvs[1]));
        assertTrue(CellUtil.isDelete(kvs[2]));
        assertArrayEquals(T2, CellUtil.cloneValue(kvs[3]));
        assertArrayEquals(T1, CellUtil.cloneValue(kvs[4]));
        scanner.close();
        h.close();
    }

    /**
    * Basic client side validation of HBASE-10118
    */
    @Test
    public void testPurgeFutureDeletes() throws Exception {
        final byte[] TABLENAME = Bytes.toBytes("testPurgeFutureDeletes");
        final byte[] ROW = Bytes.toBytes("row");
        final byte[] FAMILY = Bytes.toBytes("family");
        final byte[] COLUMN = Bytes.toBytes("column");
        final byte[] VALUE = Bytes.toBytes("value");

        HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY);

        // future timestamp
        long ts = System.currentTimeMillis() * 2;
        Put put = new Put(ROW, ts);
        put.add(FAMILY, COLUMN, VALUE);
        table.put(put);

        Get get = new Get(ROW);
        Result result = table.get(get);
        assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));

        Delete del = new Delete(ROW);
        del.deleteColumn(FAMILY, COLUMN, ts);
        table.delete(del);

        get = new Get(ROW);
        result = table.get(get);
        assertNull(result.getValue(FAMILY, COLUMN));

        // major compaction, purged future deletes 
        TEST_UTIL.getHBaseAdmin().flush(TABLENAME);
        TEST_UTIL.getHBaseAdmin().majorCompact(TABLENAME);

        // waiting for the major compaction to complete
        TEST_UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
            @Override
            public boolean evaluate() throws IOException {
                try {
                    return TEST_UTIL.getHBaseAdmin().getCompactionState(
                            TABLENAME) == AdminProtos.GetRegionInfoResponse.CompactionState.NONE;
                } catch (InterruptedException e) {
                    throw new IOException(e);
                }
            }
        });

        put = new Put(ROW, ts);
        put.add(FAMILY, COLUMN, VALUE);
        table.put(put);

        get = new Get(ROW);
        result = table.get(get);
        assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));

        table.close();
    }

    @Test
    public void testSharedZooKeeper() throws Exception {
        Configuration newConfig = new Configuration(TEST_UTIL.getConfiguration());
        newConfig.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "12345");

        // First with a simple ZKW
        ZooKeeperWatcher z0 = new ZooKeeperWatcher(newConfig, "hconnection", new Abortable() {
            @Override
            public void abort(String why, Throwable e) {
            }

            @Override
            public boolean isAborted() {
                return false;
            }
        });
        z0.getRecoverableZooKeeper().getZooKeeper().exists("/oldZooKeeperWatcher", false);
        z0.close();

        // Then a ZooKeeperKeepAliveConnection
        ConnectionManager.HConnectionImplementation connection1 = (ConnectionManager.HConnectionImplementation) HConnectionManager
                .getConnection(newConfig);

        ZooKeeperKeepAliveConnection z1 = connection1.getKeepAliveZooKeeperWatcher();
        z1.getRecoverableZooKeeper().getZooKeeper().exists("/z1", false);

        z1.close();

        // will still work, because the real connection is not closed yet
        // Not do be done in real code
        z1.getRecoverableZooKeeper().getZooKeeper().exists("/z1afterclose", false);

        ZooKeeperKeepAliveConnection z2 = connection1.getKeepAliveZooKeeperWatcher();
        assertTrue("ZooKeeperKeepAliveConnection equals on same connection", z1 == z2);

        Configuration newConfig2 = new Configuration(TEST_UTIL.getConfiguration());
        newConfig2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "6789");
        ConnectionManager.HConnectionImplementation connection2 = (ConnectionManager.HConnectionImplementation) HConnectionManager
                .getConnection(newConfig2);

        assertTrue("connections should be different ", connection1 != connection2);

        ZooKeeperKeepAliveConnection z3 = connection2.getKeepAliveZooKeeperWatcher();
        assertTrue("ZooKeeperKeepAliveConnection should be different" + " on different connections", z1 != z3);

        // Bypass the private access
        Method m = ConnectionManager.HConnectionImplementation.class.getDeclaredMethod("closeZooKeeperWatcher");
        m.setAccessible(true);
        m.invoke(connection2);

        ZooKeeperKeepAliveConnection z4 = connection2.getKeepAliveZooKeeperWatcher();
        assertTrue("ZooKeeperKeepAliveConnection should be recreated" + " when previous connections was closed",
                z3 != z4);

        z2.getRecoverableZooKeeper().getZooKeeper().exists("/z2", false);
        z4.getRecoverableZooKeeper().getZooKeeper().exists("/z4", false);

        HConnectionManager.deleteConnection(newConfig);
        try {
            z2.getRecoverableZooKeeper().getZooKeeper().exists("/z2", false);
            assertTrue("We should not have a valid connection for z2", false);
        } catch (Exception e) {
        }

        z4.getRecoverableZooKeeper().getZooKeeper().exists("/z4", false);
        // We expect success here.

        HConnectionManager.deleteConnection(newConfig2);
        try {
            z4.getRecoverableZooKeeper().getZooKeeper().exists("/z4", false);
            assertTrue("We should not have a valid connection for z4", false);
        } catch (Exception e) {
        }
    }

    /**
     * Verifies that getConfiguration returns the same Configuration object used
     * to create the HTable instance.
     */
    @Test
    public void testGetConfiguration() throws Exception {
        byte[] TABLE = Bytes.toBytes("testGetConfiguration");
        byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
        Configuration conf = TEST_UTIL.getConfiguration();
        HTable table = TEST_UTIL.createTable(TABLE, FAMILIES, conf);
        assertSame(conf, table.getConfiguration());
    }

    /**
     * Test from client side of an involved filter against a multi family that
     * involves deletes.
     *
     * @throws Exception
     */
    @Test
    public void testWeirdCacheBehaviour() throws Exception {
        byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
        byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
                Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
        HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
        String value = "this is the value";
        String value2 = "this is some other value";
        String keyPrefix1 = UUID.randomUUID().toString();
        String keyPrefix2 = UUID.randomUUID().toString();
        String keyPrefix3 = UUID.randomUUID().toString();
        putRows(ht, 3, value, keyPrefix1);
        putRows(ht, 3, value, keyPrefix2);
        putRows(ht, 3, value, keyPrefix3);
        ht.flushCommits();
        putRows(ht, 3, value2, keyPrefix1);
        putRows(ht, 3, value2, keyPrefix2);
        putRows(ht, 3, value2, keyPrefix3);
        HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE);
        System.out.println("Checking values for key: " + keyPrefix1);
        assertEquals("Got back incorrect number of rows from scan", 3, getNumberOfRows(keyPrefix1, value2, table));
        System.out.println("Checking values for key: " + keyPrefix2);
        assertEquals("Got back incorrect number of rows from scan", 3, getNumberOfRows(keyPrefix2, value2, table));
        System.out.println("Checking values for key: " + keyPrefix3);
        assertEquals("Got back incorrect number of rows from scan", 3, getNumberOfRows(keyPrefix3, value2, table));
        deleteColumns(ht, value2, keyPrefix1);
        deleteColumns(ht, value2, keyPrefix2);
        deleteColumns(ht, value2, keyPrefix3);
        System.out.println("Starting important checks.....");
        assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
                getNumberOfRows(keyPrefix1, value2, table));
        assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
                getNumberOfRows(keyPrefix2, value2, table));
        assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
                getNumberOfRows(keyPrefix3, value2, table));
        ht.setScannerCaching(0);
        assertEquals("Got back incorrect number of rows from scan", 0, getNumberOfRows(keyPrefix1, value2, table));
        ht.setScannerCaching(100);
        assertEquals("Got back incorrect number of rows from scan", 0, getNumberOfRows(keyPrefix2, value2, table));
    }

    private void deleteColumns(HTable ht, String value, String keyPrefix) throws IOException {
        ResultScanner scanner = buildScanner(keyPrefix, value, ht);
        Iterator<Result> it = scanner.iterator();
        int count = 0;
        while (it.hasNext()) {
            Result result = it.next();
            Delete delete = new Delete(result.getRow());
            delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
            ht.delete(delete);
            count++;
        }
        assertEquals("Did not perform correct number of deletes", 3, count);
    }

    private int getNumberOfRows(String keyPrefix, String value, HTable ht) throws Exception {
        ResultScanner resultScanner = buildScanner(keyPrefix, value, ht);
        Iterator<Result> scanner = resultScanner.iterator();
        int numberOfResults = 0;
        while (scanner.hasNext()) {
            Result result = scanner.next();
            System.out.println("Got back key: " + Bytes.toString(result.getRow()));
            for (Cell kv : result.rawCells()) {
                System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
            }
            numberOfResults++;
        }
        return numberOfResults;
    }

    private ResultScanner buildScanner(String keyPrefix, String value, HTable ht) throws IOException {
        // OurFilterList allFilters = new OurFilterList();
        FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */);
        allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
        SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
                Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
        filter.setFilterIfMissing(true);
        allFilters.addFilter(filter);

        // allFilters.addFilter(new
        // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"),
        // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)));

        Scan scan = new Scan();
        scan.addFamily(Bytes.toBytes("trans-blob"));
        scan.addFamily(Bytes.toBytes("trans-type"));
        scan.addFamily(Bytes.toBytes("trans-date"));
        scan.addFamily(Bytes.toBytes("trans-tags"));
        scan.addFamily(Bytes.toBytes("trans-group"));
        scan.setFilter(allFilters);

        return ht.getScanner(scan);
    }

    private void putRows(HTable ht, int numRows, String value, String key) throws IOException {
        for (int i = 0; i < numRows; i++) {
            String row = key + "_" + UUID.randomUUID().toString();
            System.out.println(String.format("Saving row: %s, with value %s", row, value));
            Put put = new Put(Bytes.toBytes(row));
            put.setDurability(Durability.SKIP_WAL);
            put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
            put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
            put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
            put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
            put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
            ht.put(put);
        }
    }

    /**
     * Test filters when multiple regions.  It does counts.  Needs eye-balling of
     * logs to ensure that we're not scanning more regions that we're supposed to.
     * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package.
     * @throws IOException
     * @throws InterruptedException
     */
    @Test
    public void testFilterAcrossMultipleRegions() throws IOException, InterruptedException {
        byte[] name = Bytes.toBytes("testFilterAcrossMutlipleRegions");
        HTable t = TEST_UTIL.createTable(name, FAMILY);
        int rowCount = TEST_UTIL.loadTable(t, FAMILY, false);
        assertRowCount(t, rowCount);
        // Split the table.  Should split on a reasonable key; 'lqj'
        Map<HRegionInfo, ServerName> regions = splitTable(t);
        assertRowCount(t, rowCount);
        // Get end key of first region.
        byte[] endKey = regions.keySet().iterator().next().getEndKey();
        // Count rows with a filter that stops us before passed 'endKey'.
        // Should be count of rows in first region.
        int endKeyCount = countRows(t, createScanWithRowFilter(endKey));
        assertTrue(endKeyCount < rowCount);

        // How do I know I did not got to second region?  Thats tough.  Can't really
        // do that in client-side region test.  I verified by tracing in debugger.
        // I changed the messages that come out when set to DEBUG so should see
        // when scanner is done. Says "Finished with scanning..." with region name.
        // Check that its finished in right region.

        // New test.  Make it so scan goes into next region by one and then two.
        // Make sure count comes out right.
        byte[] key = new byte[] { endKey[0], endKey[1], (byte) (endKey[2] + 1) };
        int plusOneCount = countRows(t, createScanWithRowFilter(key));
        assertEquals(endKeyCount + 1, plusOneCount);
        key = new byte[] { endKey[0], endKey[1], (byte) (endKey[2] + 2) };
        int plusTwoCount = countRows(t, createScanWithRowFilter(key));
        assertEquals(endKeyCount + 2, plusTwoCount);

        // New test.  Make it so I scan one less than endkey.
        key = new byte[] { endKey[0], endKey[1], (byte) (endKey[2] - 1) };
        int minusOneCount = countRows(t, createScanWithRowFilter(key));
        assertEquals(endKeyCount - 1, minusOneCount);
        // For above test... study logs.  Make sure we do "Finished with scanning.."
        // in first region and that we do not fall into the next region.

        key = new byte[] { 'a', 'a', 'a' };
        int countBBB = countRows(t, createScanWithRowFilter(key, null, CompareFilter.CompareOp.EQUAL));
        assertEquals(1, countBBB);

        int countGreater = countRows(t,
                createScanWithRowFilter(endKey, null, CompareFilter.CompareOp.GREATER_OR_EQUAL));
        // Because started at start of table.
        assertEquals(0, countGreater);
        countGreater = countRows(t,
                createScanWithRowFilter(endKey, endKey, CompareFilter.CompareOp.GREATER_OR_EQUAL));
        assertEquals(rowCount - endKeyCount, countGreater);
    }

    /*
     * @param key
     * @return Scan with RowFilter that does LESS than passed key.
     */
    private Scan createScanWithRowFilter(final byte[] key) {
        return createScanWithRowFilter(key, null, CompareFilter.CompareOp.LESS);
    }

    /*
     * @param key
     * @param op
     * @param startRow
     * @return Scan with RowFilter that does CompareOp op on passed key.
     */
    private Scan createScanWithRowFilter(final byte[] key, final byte[] startRow, CompareFilter.CompareOp op) {
        // Make sure key is of some substance... non-null and > than first key.
        assertTrue(key != null && key.length > 0
                && Bytes.BYTES_COMPARATOR.compare(key, new byte[] { 'a', 'a', 'a' }) >= 0);
        LOG.info("Key=" + Bytes.toString(key));
        Scan s = startRow == null ? new Scan() : new Scan(startRow);
        Filter f = new RowFilter(op, new BinaryComparator(key));
        f = new WhileMatchFilter(f);
        s.setFilter(f);
        return s;
    }

    /*
     * @param t
     * @param s
     * @return Count of rows in table.
     * @throws IOException
     */
    private int countRows(final HTable t, final Scan s) throws IOException {
        // Assert all rows in table.
        ResultScanner scanner = t.getScanner(s);
        int count = 0;
        for (Result result : scanner) {
            count++;
            assertTrue(result.size() > 0);
            // LOG.info("Count=" + count + ", row=" + Bytes.toString(result.getRow()));
        }
        return count;
    }

    private void assertRowCount(final HTable t, final int expected) throws IOException {
        assertEquals(expected, countRows(t, new Scan()));
    }

    /*
     * Split table into multiple regions.
     * @param t Table to split.
     * @return Map of regions to servers.
     * @throws IOException
     */
    private Map<HRegionInfo, ServerName> splitTable(final HTable t) throws IOException, InterruptedException {
        // Split this table in two.
        HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
        admin.split(t.getTableName());
        admin.close();
        Map<HRegionInfo, ServerName> regions = waitOnSplit(t);
        assertTrue(regions.size() > 1);
        return regions;
    }

    /*
     * Wait on table split.  May return because we waited long enough on the split
     * and it didn't happen.  Caller should check.
     * @param t
     * @return Map of table regions; caller needs to check table actually split.
     */
    private Map<HRegionInfo, ServerName> waitOnSplit(final HTable t) throws IOException {
        Map<HRegionInfo, ServerName> regions = t.getRegionLocations();
        int originalCount = regions.size();
        for (int i = 0; i < TEST_UTIL.getConfiguration().getInt("hbase.test.retries", 30); i++) {
            Thread.currentThread();
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            regions = t.getRegionLocations();
            if (regions.size() > originalCount)
                break;
        }
        return regions;
    }

    @Test
    public void testSuperSimple() throws Exception {
        byte[] TABLE = Bytes.toBytes("testSuperSimple");
        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
        Put put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, VALUE);
        ht.put(put);
        Scan scan = new Scan();
        scan.addColumn(FAMILY, TABLE);
        ResultScanner scanner = ht.getScanner(scan);
        Result result = scanner.next();
        assertTrue("Expected null result", result == null);
        scanner.close();
    }

    @Test
    public void testMaxKeyValueSize() throws Exception {
        byte[] TABLE = Bytes.toBytes("testMaxKeyValueSize");
        Configuration conf = TEST_UTIL.getConfiguration();
        String oldMaxSize = conf.get("hbase.client.keyvalue.maxsize");
        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
        byte[] value = new byte[4 * 1024 * 1024];
        Put put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, value);
        ht.put(put);
        try {
            conf.setInt("hbase.client.keyvalue.maxsize", 2 * 1024 * 1024);
            TABLE = Bytes.toBytes("testMaxKeyValueSize2");
            ht = TEST_UTIL.createTable(TABLE, FAMILY);
            put = new Put(ROW);
            put.add(FAMILY, QUALIFIER, value);
            ht.put(put);
            fail("Inserting a too large KeyValue worked, should throw exception");
        } catch (Exception e) {
        }
        conf.set("hbase.client.keyvalue.maxsize", oldMaxSize);
    }

    @Test
    public void testFilters() throws Exception {
        byte[] TABLE = Bytes.toBytes("testFilters");
        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
        byte[][] ROWS = makeN(ROW, 10);
        byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
                Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
                Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
                Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
                Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>") };
        for (int i = 0; i < 10; i++) {
            Put put = new Put(ROWS[i]);
            put.setDurability(Durability.SKIP_WAL);
            put.add(FAMILY, QUALIFIERS[i], VALUE);
            ht.put(put);
        }
        Scan scan = new Scan();
        scan.addFamily(FAMILY);
        Filter filter = new QualifierFilter(CompareOp.EQUAL, new RegexStringComparator("col[1-5]"));
        scan.setFilter(filter);
        ResultScanner scanner = ht.getScanner(scan);
        int expectedIndex = 1;
        for (Result result : ht.getScanner(scan)) {
            assertEquals(result.size(), 1);
            assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[expectedIndex]));
            assertTrue(Bytes.equals(CellUtil.cloneQualifier(result.rawCells()[0]), QUALIFIERS[expectedIndex]));
            expectedIndex++;
        }
        assertEquals(expectedIndex, 6);
        scanner.close();
    }

    @Test
    public void testKeyOnlyFilter() throws Exception {
        byte[] TABLE = Bytes.toBytes("testKeyOnlyFilter");
        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
        byte[][] ROWS = makeN(ROW, 10);
        byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
                Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
                Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
                Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
                Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>") };
        for (int i = 0; i < 10; i++) {
            Put put = new Put(ROWS[i]);
            put.setDurability(Durability.SKIP_WAL);
            put.add(FAMILY, QUALIFIERS[i], VALUE);
            ht.put(put);
        }
        Scan scan = new Scan();
        scan.addFamily(FAMILY);
        Filter filter = new KeyOnlyFilter(true);
        scan.setFilter(filter);
        ResultScanner scanner = ht.getScanner(scan);
        int count = 0;
        for (Result result : ht.getScanner(scan)) {
            assertEquals(result.size(), 1);
            assertEquals(result.rawCells()[0].getValueLength(), Bytes.SIZEOF_INT);
            assertEquals(Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])), VALUE.length);
            count++;
        }
        assertEquals(count, 10);
        scanner.close();
    }

    /**
     * Test simple table and non-existent row cases.
     */
    @Test
    public void testSimpleMissing() throws Exception {
        byte[] TABLE = Bytes.toBytes("testSimpleMissing");
        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
        byte[][] ROWS = makeN(ROW, 4);

        // Try to get a row on an empty table
        Get get = new Get(ROWS[0]);
        Result result = ht.get(get);
        assertEmptyResult(result);

        get = new Get(ROWS[0]);
        get.addFamily(FAMILY);
        result = ht.get(get);
        assertEmptyResult(result);

        get = new Get(ROWS[0]);
        get.addColumn(FAMILY, QUALIFIER);
        result = ht.get(get);
        assertEmptyResult(result);

        Scan scan = new Scan();
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        scan = new Scan(ROWS[0]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        scan = new Scan(ROWS[0], ROWS[1]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        scan = new Scan();
        scan.addFamily(FAMILY);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        scan = new Scan();
        scan.addColumn(FAMILY, QUALIFIER);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        // Insert a row

        Put put = new Put(ROWS[2]);
        put.add(FAMILY, QUALIFIER, VALUE);
        ht.put(put);

        // Try to get empty rows around it

        get = new Get(ROWS[1]);
        result = ht.get(get);
        assertEmptyResult(result);

        get = new Get(ROWS[0]);
        get.addFamily(FAMILY);
        result = ht.get(get);
        assertEmptyResult(result);

        get = new Get(ROWS[3]);
        get.addColumn(FAMILY, QUALIFIER);
        result = ht.get(get);
        assertEmptyResult(result);

        // Try to scan empty rows around it

        scan = new Scan(ROWS[3]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        scan = new Scan(ROWS[0], ROWS[2]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        // Make sure we can actually get the row

        get = new Get(ROWS[2]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);

        get = new Get(ROWS[2]);
        get.addFamily(FAMILY);
        result = ht.get(get);
        assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);

        get = new Get(ROWS[2]);
        get.addColumn(FAMILY, QUALIFIER);
        result = ht.get(get);
        assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);

        // Make sure we can scan the row

        scan = new Scan();
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);

        scan = new Scan(ROWS[0], ROWS[3]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);

        scan = new Scan(ROWS[2], ROWS[3]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
    }

    /**
     * Test basic puts, gets, scans, and deletes for a single row
     * in a multiple family table.
     */
    @Test
    public void testSingleRowMultipleFamily() throws Exception {
        byte[] TABLE = Bytes.toBytes("testSingleRowMultipleFamily");
        byte[][] ROWS = makeN(ROW, 3);
        byte[][] FAMILIES = makeNAscii(FAMILY, 10);
        byte[][] QUALIFIERS = makeN(QUALIFIER, 10);
        byte[][] VALUES = makeN(VALUE, 10);

        HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);

        Get get;
        Scan scan;
        Delete delete;
        Put put;
        Result result;

        ////////////////////////////////////////////////////////////////////////////
        // Insert one column to one family
        ////////////////////////////////////////////////////////////////////////////

        put = new Put(ROWS[0]);
        put.add(FAMILIES[4], QUALIFIERS[0], VALUES[0]);
        ht.put(put);

        // Get the single column
        getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);

        // Scan the single column
        scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);

        // Get empty results around inserted column
        getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);

        // Scan empty results around inserted column
        scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);

        ////////////////////////////////////////////////////////////////////////////
        // Flush memstore and run same tests from storefiles
        ////////////////////////////////////////////////////////////////////////////

        TEST_UTIL.flush();

        // Redo get and scan tests from storefile
        getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
        scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
        getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
        scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);

        ////////////////////////////////////////////////////////////////////////////
        // Now, Test reading from memstore and storefiles at once
        ////////////////////////////////////////////////////////////////////////////

        // Insert multiple columns to two other families
        put = new Put(ROWS[0]);
        put.add(FAMILIES[2], QUALIFIERS[2], VALUES[2]);
        put.add(FAMILIES[2], QUALIFIERS[4], VALUES[4]);
        put.add(FAMILIES[4], QUALIFIERS[4], VALUES[4]);
        put.add(FAMILIES[6], QUALIFIERS[6], VALUES[6]);
        put.add(FAMILIES[6], QUALIFIERS[7], VALUES[7]);
        put.add(FAMILIES[7], QUALIFIERS[7], VALUES[7]);
        put.add(FAMILIES[9], QUALIFIERS[0], VALUES[0]);
        ht.put(put);

        // Get multiple columns across multiple families and get empties around it
        singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);

        // Scan multiple columns across multiple families and scan empties around it
        singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);

        ////////////////////////////////////////////////////////////////////////////
        // Flush the table again
        ////////////////////////////////////////////////////////////////////////////

        TEST_UTIL.flush();

        // Redo tests again
        singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
        singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);

        // Insert more data to memstore
        put = new Put(ROWS[0]);
        put.add(FAMILIES[6], QUALIFIERS[5], VALUES[5]);
        put.add(FAMILIES[6], QUALIFIERS[8], VALUES[8]);
        put.add(FAMILIES[6], QUALIFIERS[9], VALUES[9]);
        put.add(FAMILIES[4], QUALIFIERS[3], VALUES[3]);
        ht.put(put);

        ////////////////////////////////////////////////////////////////////////////
        // Delete a storefile column
        ////////////////////////////////////////////////////////////////////////////
        delete = new Delete(ROWS[0]);
        delete.deleteColumns(FAMILIES[6], QUALIFIERS[7]);
        ht.delete(delete);

        // Try to get deleted column
        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[6], QUALIFIERS[7]);
        result = ht.get(get);
        assertEmptyResult(result);

        // Try to scan deleted column
        scan = new Scan();
        scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        // Make sure we can still get a column before it and after it
        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[6], QUALIFIERS[6]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);

        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[6], QUALIFIERS[8]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);

        // Make sure we can still scan a column before it and after it
        scan = new Scan();
        scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);

        scan = new Scan();
        scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);

        ////////////////////////////////////////////////////////////////////////////
        // Delete a memstore column
        ////////////////////////////////////////////////////////////////////////////
        delete = new Delete(ROWS[0]);
        delete.deleteColumns(FAMILIES[6], QUALIFIERS[8]);
        ht.delete(delete);

        // Try to get deleted column
        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[6], QUALIFIERS[8]);
        result = ht.get(get);
        assertEmptyResult(result);

        // Try to scan deleted column
        scan = new Scan();
        scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        // Make sure we can still get a column before it and after it
        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[6], QUALIFIERS[6]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);

        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[6], QUALIFIERS[9]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);

        // Make sure we can still scan a column before it and after it
        scan = new Scan();
        scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);

        scan = new Scan();
        scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);

        ////////////////////////////////////////////////////////////////////////////
        // Delete joint storefile/memstore family
        ////////////////////////////////////////////////////////////////////////////

        delete = new Delete(ROWS[0]);
        delete.deleteFamily(FAMILIES[4]);
        ht.delete(delete);

        // Try to get storefile column in deleted family
        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[4], QUALIFIERS[4]);
        result = ht.get(get);
        assertEmptyResult(result);

        // Try to get memstore column in deleted family
        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[4], QUALIFIERS[3]);
        result = ht.get(get);
        assertEmptyResult(result);

        // Try to get deleted family
        get = new Get(ROWS[0]);
        get.addFamily(FAMILIES[4]);
        result = ht.get(get);
        assertEmptyResult(result);

        // Try to scan storefile column in deleted family
        scan = new Scan();
        scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        // Try to scan memstore column in deleted family
        scan = new Scan();
        scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        // Try to scan deleted family
        scan = new Scan();
        scan.addFamily(FAMILIES[4]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        // Make sure we can still get another family
        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[2], QUALIFIERS[2]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);

        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[6], QUALIFIERS[9]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);

        // Make sure we can still scan another family
        scan = new Scan();
        scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);

        scan = new Scan();
        scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);

        ////////////////////////////////////////////////////////////////////////////
        // Flush everything and rerun delete tests
        ////////////////////////////////////////////////////////////////////////////

        TEST_UTIL.flush();

        // Try to get storefile column in deleted family
        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[4], QUALIFIERS[4]);
        result = ht.get(get);
        assertEmptyResult(result);

        // Try to get memstore column in deleted family
        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[4], QUALIFIERS[3]);
        result = ht.get(get);
        assertEmptyResult(result);

        // Try to get deleted family
        get = new Get(ROWS[0]);
        get.addFamily(FAMILIES[4]);
        result = ht.get(get);
        assertEmptyResult(result);

        // Try to scan storefile column in deleted family
        scan = new Scan();
        scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        // Try to scan memstore column in deleted family
        scan = new Scan();
        scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        // Try to scan deleted family
        scan = new Scan();
        scan.addFamily(FAMILIES[4]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        // Make sure we can still get another family
        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[2], QUALIFIERS[2]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);

        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[6], QUALIFIERS[9]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);

        // Make sure we can still scan another family
        scan = new Scan();
        scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);

        scan = new Scan();
        scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);

    }

    @Test
    public void testNull() throws Exception {
        byte[] TABLE = Bytes.toBytes("testNull");

        // Null table name (should NOT work)
        try {
            TEST_UTIL.createTable((TableName) null, FAMILY);
            fail("Creating a table with null name passed, should have failed");
        } catch (Exception e) {
        }

        // Null family (should NOT work)
        try {
            TEST_UTIL.createTable(TABLE, (byte[]) null);
            fail("Creating a table with a null family passed, should fail");
        } catch (Exception e) {
        }

        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);

        // Null row (should NOT work)
        try {
            Put put = new Put((byte[]) null);
            put.add(FAMILY, QUALIFIER, VALUE);
            ht.put(put);
            fail("Inserting a null row worked, should throw exception");
        } catch (Exception e) {
        }

        // Null qualifier (should work)
        {
            Put put = new Put(ROW);
            put.add(FAMILY, null, VALUE);
            ht.put(put);

            getTestNull(ht, ROW, FAMILY, VALUE);

            scanTestNull(ht, ROW, FAMILY, VALUE);

            Delete delete = new Delete(ROW);
            delete.deleteColumns(FAMILY, null);
            ht.delete(delete);

            Get get = new Get(ROW);
            Result result = ht.get(get);
            assertEmptyResult(result);
        }

        // Use a new table
        byte[] TABLE2 = Bytes.toBytes("testNull2");
        ht = TEST_UTIL.createTable(TABLE2, FAMILY);

        // Empty qualifier, byte[0] instead of null (should work)
        try {
            Put put = new Put(ROW);
            put.add(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
            ht.put(put);

            getTestNull(ht, ROW, FAMILY, VALUE);

            scanTestNull(ht, ROW, FAMILY, VALUE);

            // Flush and try again

            TEST_UTIL.flush();

            getTestNull(ht, ROW, FAMILY, VALUE);

            scanTestNull(ht, ROW, FAMILY, VALUE);

            Delete delete = new Delete(ROW);
            delete.deleteColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
            ht.delete(delete);

            Get get = new Get(ROW);
            Result result = ht.get(get);
            assertEmptyResult(result);

        } catch (Exception e) {
            throw new IOException("Using a row with null qualifier threw exception, should ");
        }

        // Null value
        try {
            Put put = new Put(ROW);
            put.add(FAMILY, QUALIFIER, null);
            ht.put(put);

            Get get = new Get(ROW);
            get.addColumn(FAMILY, QUALIFIER);
            Result result = ht.get(get);
            assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);

            Scan scan = new Scan();
            scan.addColumn(FAMILY, QUALIFIER);
            result = getSingleScanResult(ht, scan);
            assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);

            Delete delete = new Delete(ROW);
            delete.deleteColumns(FAMILY, QUALIFIER);
            ht.delete(delete);

            get = new Get(ROW);
            result = ht.get(get);
            assertEmptyResult(result);

        } catch (Exception e) {
            throw new IOException("Null values should be allowed, but threw exception");
        }
    }

    @Test
    public void testVersions() throws Exception {
        byte[] TABLE = Bytes.toBytes("testVersions");

        long[] STAMPS = makeStamps(20);
        byte[][] VALUES = makeNAscii(VALUE, 20);

        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);

        // Insert 4 versions of same column
        Put put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
        ht.put(put);

        // Verify we can get each one properly
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);

        // Verify we don't accidentally get others
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);

        // Ensure maxVersions in query is respected
        Get get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIER);
        get.setMaxVersions(2);
        Result result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
                new byte[][] { VALUES[4], VALUES[5] }, 0, 1);

        Scan scan = new Scan(ROW);
        scan.addColumn(FAMILY, QUALIFIER);
        scan.setMaxVersions(2);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
                new byte[][] { VALUES[4], VALUES[5] }, 0, 1);

        // Flush and redo

        TEST_UTIL.flush();

        // Verify we can get each one properly
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);

        // Verify we don't accidentally get others
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);

        // Ensure maxVersions in query is respected
        get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIER);
        get.setMaxVersions(2);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
                new byte[][] { VALUES[4], VALUES[5] }, 0, 1);

        scan = new Scan(ROW);
        scan.addColumn(FAMILY, QUALIFIER);
        scan.setMaxVersions(2);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
                new byte[][] { VALUES[4], VALUES[5] }, 0, 1);

        // Add some memstore and retest

        // Insert 4 more versions of same column and a dupe
        put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
        put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
        put.add(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
        put.add(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
        ht.put(put);

        // Ensure maxVersions in query is respected
        get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIER);
        get.setMaxVersions();
        result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7],
                        STAMPS[8] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7],
                        VALUES[8] },
                0, 7);

        scan = new Scan(ROW);
        scan.addColumn(FAMILY, QUALIFIER);
        scan.setMaxVersions();
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7],
                        STAMPS[8] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7],
                        VALUES[8] },
                0, 7);

        get = new Get(ROW);
        get.setMaxVersions();
        result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7],
                        STAMPS[8] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7],
                        VALUES[8] },
                0, 7);

        scan = new Scan(ROW);
        scan.setMaxVersions();
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7],
                        STAMPS[8] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7],
                        VALUES[8] },
                0, 7);

        // Verify we can get each one properly
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);

        // Verify we don't accidentally get others
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);

        // Ensure maxVersions of table is respected

        TEST_UTIL.flush();

        // Insert 4 more versions of same column and a dupe
        put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
        put.add(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
        put.add(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
        put.add(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
        ht.put(put);

        get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIER);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
                        STAMPS[11], STAMPS[13], STAMPS[15] },
                new byte[][] { VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
                        VALUES[11], VALUES[13], VALUES[15] },
                0, 9);

        scan = new Scan(ROW);
        scan.addColumn(FAMILY, QUALIFIER);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
                        STAMPS[11], STAMPS[13], STAMPS[15] },
                new byte[][] { VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
                        VALUES[11], VALUES[13], VALUES[15] },
                0, 9);

        // Delete a version in the memstore and a version in a storefile
        Delete delete = new Delete(ROW);
        delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[11]);
        delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[7]);
        ht.delete(delete);

        // Test that it's gone
        get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIER);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9],
                        STAMPS[13], STAMPS[15] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8],
                        VALUES[9], VALUES[13], VALUES[15] },
                0, 9);

        scan = new Scan(ROW);
        scan.addColumn(FAMILY, QUALIFIER);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9],
                        STAMPS[13], STAMPS[15] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8],
                        VALUES[9], VALUES[13], VALUES[15] },
                0, 9);

    }

    @Test
    public void testVersionLimits() throws Exception {
        byte[] TABLE = Bytes.toBytes("testVersionLimits");
        byte[][] FAMILIES = makeNAscii(FAMILY, 3);
        int[] LIMITS = { 1, 3, 5 };
        long[] STAMPS = makeStamps(10);
        byte[][] VALUES = makeNAscii(VALUE, 10);
        HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, LIMITS);

        // Insert limit + 1 on each family
        Put put = new Put(ROW);
        put.add(FAMILIES[0], QUALIFIER, STAMPS[0], VALUES[0]);
        put.add(FAMILIES[0], QUALIFIER, STAMPS[1], VALUES[1]);
        put.add(FAMILIES[1], QUALIFIER, STAMPS[0], VALUES[0]);
        put.add(FAMILIES[1], QUALIFIER, STAMPS[1], VALUES[1]);
        put.add(FAMILIES[1], QUALIFIER, STAMPS[2], VALUES[2]);
        put.add(FAMILIES[1], QUALIFIER, STAMPS[3], VALUES[3]);
        put.add(FAMILIES[2], QUALIFIER, STAMPS[0], VALUES[0]);
        put.add(FAMILIES[2], QUALIFIER, STAMPS[1], VALUES[1]);
        put.add(FAMILIES[2], QUALIFIER, STAMPS[2], VALUES[2]);
        put.add(FAMILIES[2], QUALIFIER, STAMPS[3], VALUES[3]);
        put.add(FAMILIES[2], QUALIFIER, STAMPS[4], VALUES[4]);
        put.add(FAMILIES[2], QUALIFIER, STAMPS[5], VALUES[5]);
        put.add(FAMILIES[2], QUALIFIER, STAMPS[6], VALUES[6]);
        ht.put(put);

        // Verify we only get the right number out of each

        // Family0

        Get get = new Get(ROW);
        get.addColumn(FAMILIES[0], QUALIFIER);
        get.setMaxVersions(Integer.MAX_VALUE);
        Result result = ht.get(get);
        assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { STAMPS[1] }, new byte[][] { VALUES[1] }, 0,
                0);

        get = new Get(ROW);
        get.addFamily(FAMILIES[0]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { STAMPS[1] }, new byte[][] { VALUES[1] }, 0,
                0);

        Scan scan = new Scan(ROW);
        scan.addColumn(FAMILIES[0], QUALIFIER);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { STAMPS[1] }, new byte[][] { VALUES[1] }, 0,
                0);

        scan = new Scan(ROW);
        scan.addFamily(FAMILIES[0]);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { STAMPS[1] }, new byte[][] { VALUES[1] }, 0,
                0);

        // Family1

        get = new Get(ROW);
        get.addColumn(FAMILIES[1], QUALIFIER);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILIES[1], QUALIFIER, new long[] { STAMPS[1], STAMPS[2], STAMPS[3] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);

        get = new Get(ROW);
        get.addFamily(FAMILIES[1]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILIES[1], QUALIFIER, new long[] { STAMPS[1], STAMPS[2], STAMPS[3] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);

        scan = new Scan(ROW);
        scan.addColumn(FAMILIES[1], QUALIFIER);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILIES[1], QUALIFIER, new long[] { STAMPS[1], STAMPS[2], STAMPS[3] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);

        scan = new Scan(ROW);
        scan.addFamily(FAMILIES[1]);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILIES[1], QUALIFIER, new long[] { STAMPS[1], STAMPS[2], STAMPS[3] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);

        // Family2

        get = new Get(ROW);
        get.addColumn(FAMILIES[2], QUALIFIER);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
                new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6] },
                new byte[][] { VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6] }, 0, 4);

        get = new Get(ROW);
        get.addFamily(FAMILIES[2]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
                new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6] },
                new byte[][] { VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6] }, 0, 4);

        scan = new Scan(ROW);
        scan.addColumn(FAMILIES[2], QUALIFIER);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
                new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6] },
                new byte[][] { VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6] }, 0, 4);

        scan = new Scan(ROW);
        scan.addFamily(FAMILIES[2]);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
                new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6] },
                new byte[][] { VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6] }, 0, 4);

        // Try all families

        get = new Get(ROW);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertTrue("Expected 9 keys but received " + result.size(), result.size() == 9);

        get = new Get(ROW);
        get.addFamily(FAMILIES[0]);
        get.addFamily(FAMILIES[1]);
        get.addFamily(FAMILIES[2]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertTrue("Expected 9 keys but received " + result.size(), result.size() == 9);

        get = new Get(ROW);
        get.addColumn(FAMILIES[0], QUALIFIER);
        get.addColumn(FAMILIES[1], QUALIFIER);
        get.addColumn(FAMILIES[2], QUALIFIER);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertTrue("Expected 9 keys but received " + result.size(), result.size() == 9);

        scan = new Scan(ROW);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertTrue("Expected 9 keys but received " + result.size(), result.size() == 9);

        scan = new Scan(ROW);
        scan.setMaxVersions(Integer.MAX_VALUE);
        scan.addFamily(FAMILIES[0]);
        scan.addFamily(FAMILIES[1]);
        scan.addFamily(FAMILIES[2]);
        result = getSingleScanResult(ht, scan);
        assertTrue("Expected 9 keys but received " + result.size(), result.size() == 9);

        scan = new Scan(ROW);
        scan.setMaxVersions(Integer.MAX_VALUE);
        scan.addColumn(FAMILIES[0], QUALIFIER);
        scan.addColumn(FAMILIES[1], QUALIFIER);
        scan.addColumn(FAMILIES[2], QUALIFIER);
        result = getSingleScanResult(ht, scan);
        assertTrue("Expected 9 keys but received " + result.size(), result.size() == 9);

    }

    @Test
    public void testDeleteFamilyVersion() throws Exception {
        HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
        byte[] TABLE = Bytes.toBytes("testDeleteFamilyVersion");

        byte[][] QUALIFIERS = makeNAscii(QUALIFIER, 1);
        byte[][] VALUES = makeN(VALUE, 5);
        long[] ts = { 1000, 2000, 3000, 4000, 5000 };

        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 5);

        Put put = new Put(ROW);
        for (int q = 0; q < 1; q++)
            for (int t = 0; t < 5; t++)
                put.add(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
        ht.put(put);
        admin.flush(TABLE);

        Delete delete = new Delete(ROW);
        delete.deleteFamilyVersion(FAMILY, ts[1]); // delete version '2000'
        delete.deleteFamilyVersion(FAMILY, ts[3]); // delete version '4000'
        ht.delete(delete);
        admin.flush(TABLE);

        for (int i = 0; i < 1; i++) {
            Get get = new Get(ROW);
            get.addColumn(FAMILY, QUALIFIERS[i]);
            get.setMaxVersions(Integer.MAX_VALUE);
            Result result = ht.get(get);
            // verify version '1000'/'3000'/'5000' remains for all columns
            assertNResult(result, ROW, FAMILY, QUALIFIERS[i], new long[] { ts[0], ts[2], ts[4] },
                    new byte[][] { VALUES[0], VALUES[2], VALUES[4] }, 0, 2);
        }
        ht.close();
        admin.close();
    }

    @Test
    public void testDeleteFamilyVersionWithOtherDeletes() throws Exception {
        byte[] TABLE = Bytes.toBytes("testDeleteFamilyVersionWithOtherDeletes");

        byte[][] QUALIFIERS = makeNAscii(QUALIFIER, 5);
        byte[][] VALUES = makeN(VALUE, 5);
        long[] ts = { 1000, 2000, 3000, 4000, 5000 };

        HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 5);
        Put put = null;
        Result result = null;
        Get get = null;
        Delete delete = null;

        // 1. put on ROW
        put = new Put(ROW);
        for (int q = 0; q < 5; q++)
            for (int t = 0; t < 5; t++)
                put.add(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
        ht.put(put);
        admin.flush(TABLE);

        // 2. put on ROWS[0]
        byte[] ROW2 = Bytes.toBytes("myRowForTest");
        put = new Put(ROW2);
        for (int q = 0; q < 5; q++)
            for (int t = 0; t < 5; t++)
                put.add(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
        ht.put(put);
        admin.flush(TABLE);

        // 3. delete on ROW
        delete = new Delete(ROW);
        // delete version <= 2000 of all columns
        // note: deleteFamily must be the first since it will mask
        // the subsequent other type deletes!
        delete.deleteFamily(FAMILY, ts[1]);
        // delete version '4000' of all columns
        delete.deleteFamilyVersion(FAMILY, ts[3]);
        // delete version <= 3000 of column 0
        delete.deleteColumns(FAMILY, QUALIFIERS[0], ts[2]);
        // delete version <= 5000 of column 2
        delete.deleteColumns(FAMILY, QUALIFIERS[2], ts[4]);
        // delete version 5000 of column 4
        delete.deleteColumn(FAMILY, QUALIFIERS[4], ts[4]);
        ht.delete(delete);
        admin.flush(TABLE);

        // 4. delete on ROWS[0]
        delete = new Delete(ROW2);
        delete.deleteFamilyVersion(FAMILY, ts[1]); // delete version '2000'
        delete.deleteFamilyVersion(FAMILY, ts[3]); // delete version '4000'
        ht.delete(delete);
        admin.flush(TABLE);

        // 5. check ROW
        get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIERS[0]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIERS[0], new long[] { ts[4] }, new byte[][] { VALUES[4] }, 0, 0);

        get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIERS[1]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIERS[1], new long[] { ts[2], ts[4] },
                new byte[][] { VALUES[2], VALUES[4] }, 0, 1);

        get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIERS[2]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertEquals(0, result.size());

        get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIERS[3]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIERS[3], new long[] { ts[2], ts[4] },
                new byte[][] { VALUES[2], VALUES[4] }, 0, 1);

        get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIERS[4]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIERS[4], new long[] { ts[2] }, new byte[][] { VALUES[2] }, 0, 0);

        // 6. check ROWS[0]
        for (int i = 0; i < 5; i++) {
            get = new Get(ROW2);
            get.addColumn(FAMILY, QUALIFIERS[i]);
            get.setMaxVersions(Integer.MAX_VALUE);
            result = ht.get(get);
            // verify version '1000'/'3000'/'5000' remains for all columns
            assertNResult(result, ROW2, FAMILY, QUALIFIERS[i], new long[] { ts[0], ts[2], ts[4] },
                    new byte[][] { VALUES[0], VALUES[2], VALUES[4] }, 0, 2);
        }
        ht.close();
        admin.close();
    }

    @Test
    public void testDeletes() throws Exception {
        byte[] TABLE = Bytes.toBytes("testDeletes");

        byte[][] ROWS = makeNAscii(ROW, 6);
        byte[][] FAMILIES = makeNAscii(FAMILY, 3);
        byte[][] VALUES = makeN(VALUE, 5);
        long[] ts = { 1000, 2000, 3000, 4000, 5000 };

        HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, 3);

        Put put = new Put(ROW);
        put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
        put.add(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
        ht.put(put);

        Delete delete = new Delete(ROW);
        delete.deleteFamily(FAMILIES[0], ts[0]);
        ht.delete(delete);

        Get get = new Get(ROW);
        get.addFamily(FAMILIES[0]);
        get.setMaxVersions(Integer.MAX_VALUE);
        Result result = ht.get(get);
        assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1] }, new byte[][] { VALUES[1] }, 0, 0);

        Scan scan = new Scan(ROW);
        scan.addFamily(FAMILIES[0]);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1] }, new byte[][] { VALUES[1] }, 0, 0);

        // Test delete latest version
        put = new Put(ROW);
        put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
        put.add(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
        put.add(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
        put.add(FAMILIES[0], null, ts[4], VALUES[4]);
        put.add(FAMILIES[0], null, ts[2], VALUES[2]);
        put.add(FAMILIES[0], null, ts[3], VALUES[3]);
        ht.put(put);

        delete = new Delete(ROW);
        delete.deleteColumn(FAMILIES[0], QUALIFIER); // ts[4]
        ht.delete(delete);

        get = new Get(ROW);
        get.addColumn(FAMILIES[0], QUALIFIER);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1], ts[2], ts[3] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);

        scan = new Scan(ROW);
        scan.addColumn(FAMILIES[0], QUALIFIER);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1], ts[2], ts[3] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);

        // Test for HBASE-1847
        delete = new Delete(ROW);
        delete.deleteColumn(FAMILIES[0], null);
        ht.delete(delete);

        // Cleanup null qualifier
        delete = new Delete(ROW);
        delete.deleteColumns(FAMILIES[0], null);
        ht.delete(delete);

        // Expected client behavior might be that you can re-put deleted values
        // But alas, this is not to be.  We can't put them back in either case.

        put = new Put(ROW);
        put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000
        put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000
        ht.put(put);

        // It used to be due to the internal implementation of Get, that
        // the Get() call would return ts[4] UNLIKE the Scan below. With
        // the switch to using Scan for Get this is no longer the case.
        get = new Get(ROW);
        get.addFamily(FAMILIES[0]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1], ts[2], ts[3] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);

        // The Scanner returns the previous values, the expected-naive-unexpected behavior

        scan = new Scan(ROW);
        scan.addFamily(FAMILIES[0]);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1], ts[2], ts[3] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);

        // Test deleting an entire family from one row but not the other various ways

        put = new Put(ROWS[0]);
        put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
        put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
        put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
        put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
        ht.put(put);

        put = new Put(ROWS[1]);
        put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
        put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
        put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
        put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
        ht.put(put);

        put = new Put(ROWS[2]);
        put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
        put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
        put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
        put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
        ht.put(put);

        // Assert that above went in.
        get = new Get(ROWS[2]);
        get.addFamily(FAMILIES[1]);
        get.addFamily(FAMILIES[2]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertTrue("Expected 4 key but received " + result.size() + ": " + result, result.size() == 4);

        delete = new Delete(ROWS[0]);
        delete.deleteFamily(FAMILIES[2]);
        ht.delete(delete);

        delete = new Delete(ROWS[1]);
        delete.deleteColumns(FAMILIES[1], QUALIFIER);
        ht.delete(delete);

        delete = new Delete(ROWS[2]);
        delete.deleteColumn(FAMILIES[1], QUALIFIER);
        delete.deleteColumn(FAMILIES[1], QUALIFIER);
        delete.deleteColumn(FAMILIES[2], QUALIFIER);
        ht.delete(delete);

        get = new Get(ROWS[0]);
        get.addFamily(FAMILIES[1]);
        get.addFamily(FAMILIES[2]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertTrue("Expected 2 keys but received " + result.size(), result.size() == 2);
        assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, new long[] { ts[0], ts[1] },
                new byte[][] { VALUES[0], VALUES[1] }, 0, 1);

        scan = new Scan(ROWS[0]);
        scan.addFamily(FAMILIES[1]);
        scan.addFamily(FAMILIES[2]);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertTrue("Expected 2 keys but received " + result.size(), result.size() == 2);
        assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, new long[] { ts[0], ts[1] },
                new byte[][] { VALUES[0], VALUES[1] }, 0, 1);

        get = new Get(ROWS[1]);
        get.addFamily(FAMILIES[1]);
        get.addFamily(FAMILIES[2]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertTrue("Expected 2 keys but received " + result.size(), result.size() == 2);

        scan = new Scan(ROWS[1]);
        scan.addFamily(FAMILIES[1]);
        scan.addFamily(FAMILIES[2]);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertTrue("Expected 2 keys but received " + result.size(), result.size() == 2);

        get = new Get(ROWS[2]);
        get.addFamily(FAMILIES[1]);
        get.addFamily(FAMILIES[2]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertEquals(1, result.size());
        assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER, new long[] { ts[2] }, new byte[][] { VALUES[2] }, 0,
                0);

        scan = new Scan(ROWS[2]);
        scan.addFamily(FAMILIES[1]);
        scan.addFamily(FAMILIES[2]);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertEquals(1, result.size());
        assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER, new long[] { ts[2] }, new byte[][] { VALUES[2] }, 0,
                0);

        // Test if we delete the family first in one row (HBASE-1541)

        delete = new Delete(ROWS[3]);
        delete.deleteFamily(FAMILIES[1]);
        ht.delete(delete);

        put = new Put(ROWS[3]);
        put.add(FAMILIES[2], QUALIFIER, VALUES[0]);
        ht.put(put);

        put = new Put(ROWS[4]);
        put.add(FAMILIES[1], QUALIFIER, VALUES[1]);
        put.add(FAMILIES[2], QUALIFIER, VALUES[2]);
        ht.put(put);

        get = new Get(ROWS[3]);
        get.addFamily(FAMILIES[1]);
        get.addFamily(FAMILIES[2]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertTrue("Expected 1 key but received " + result.size(), result.size() == 1);

        get = new Get(ROWS[4]);
        get.addFamily(FAMILIES[1]);
        get.addFamily(FAMILIES[2]);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertTrue("Expected 2 keys but received " + result.size(), result.size() == 2);

        scan = new Scan(ROWS[3]);
        scan.addFamily(FAMILIES[1]);
        scan.addFamily(FAMILIES[2]);
        scan.setMaxVersions(Integer.MAX_VALUE);
        ResultScanner scanner = ht.getScanner(scan);
        result = scanner.next();
        assertTrue("Expected 1 key but received " + result.size(), result.size() == 1);
        assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[3]));
        assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[0]));
        result = scanner.next();
        assertTrue("Expected 2 keys but received " + result.size(), result.size() == 2);
        assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[4]));
        assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[1]), ROWS[4]));
        assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[1]));
        assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[1]), VALUES[2]));
        scanner.close();

        // Add test of bulk deleting.
        for (int i = 0; i < 10; i++) {
            byte[] bytes = Bytes.toBytes(i);
            put = new Put(bytes);
            put.setDurability(Durability.SKIP_WAL);
            put.add(FAMILIES[0], QUALIFIER, bytes);
            ht.put(put);
        }
        for (int i = 0; i < 10; i++) {
            byte[] bytes = Bytes.toBytes(i);
            get = new Get(bytes);
            get.addFamily(FAMILIES[0]);
            result = ht.get(get);
            assertTrue(result.size() == 1);
        }
        ArrayList<Delete> deletes = new ArrayList<Delete>();
        for (int i = 0; i < 10; i++) {
            byte[] bytes = Bytes.toBytes(i);
            delete = new Delete(bytes);
            delete.deleteFamily(FAMILIES[0]);
            deletes.add(delete);
        }
        ht.delete(deletes);
        for (int i = 0; i < 10; i++) {
            byte[] bytes = Bytes.toBytes(i);
            get = new Get(bytes);
            get.addFamily(FAMILIES[0]);
            result = ht.get(get);
            assertTrue(result.size() == 0);
        }
    }

    /*
     * Baseline "scalability" test.
     *
     * Tests one hundred families, one million columns, one million versions
     */
    @Ignore
    @Test
    public void testMillions() throws Exception {

        // 100 families

        // millions of columns

        // millions of versions

    }

    @Ignore
    @Test
    public void testMultipleRegionsAndBatchPuts() throws Exception {
        // Two family table

        // Insert lots of rows

        // Insert to the same row with batched puts

        // Insert to multiple rows with batched puts

        // Split the table

        // Get row from first region

        // Get row from second region

        // Scan all rows

        // Insert to multiple regions with batched puts

        // Get row from first region

        // Get row from second region

        // Scan all rows

    }

    @Ignore
    @Test
    public void testMultipleRowMultipleFamily() throws Exception {

    }

    //
    // JIRA Testers
    //

    /**
     * HBASE-867
     *    If millions of columns in a column family, hbase scanner won't come up
     *
     *    Test will create numRows rows, each with numColsPerRow columns
     *    (1 version each), and attempt to scan them all.
     *
     *    To test at scale, up numColsPerRow to the millions
     *    (have not gotten that to work running as junit though)
     */
    @Test
    public void testJiraTest867() throws Exception {
        int numRows = 10;
        int numColsPerRow = 2000;

        byte[] TABLE = Bytes.toBytes("testJiraTest867");

        byte[][] ROWS = makeN(ROW, numRows);
        byte[][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);

        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);

        // Insert rows

        for (int i = 0; i < numRows; i++) {
            Put put = new Put(ROWS[i]);
            put.setDurability(Durability.SKIP_WAL);
            for (int j = 0; j < numColsPerRow; j++) {
                put.add(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
            }
            assertTrue("Put expected to contain " + numColsPerRow + " columns but " + "only contains " + put.size(),
                    put.size() == numColsPerRow);
            ht.put(put);
        }

        // Get a row
        Get get = new Get(ROWS[numRows - 1]);
        Result result = ht.get(get);
        assertNumKeys(result, numColsPerRow);
        Cell[] keys = result.rawCells();
        for (int i = 0; i < result.size(); i++) {
            assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
        }

        // Scan the rows
        Scan scan = new Scan();
        ResultScanner scanner = ht.getScanner(scan);
        int rowCount = 0;
        while ((result = scanner.next()) != null) {
            assertNumKeys(result, numColsPerRow);
            Cell[] kvs = result.rawCells();
            for (int i = 0; i < numColsPerRow; i++) {
                assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
            }
            rowCount++;
        }
        scanner.close();
        assertTrue("Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows",
                rowCount == numRows);

        // flush and try again

        TEST_UTIL.flush();

        // Get a row
        get = new Get(ROWS[numRows - 1]);
        result = ht.get(get);
        assertNumKeys(result, numColsPerRow);
        keys = result.rawCells();
        for (int i = 0; i < result.size(); i++) {
            assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
        }

        // Scan the rows
        scan = new Scan();
        scanner = ht.getScanner(scan);
        rowCount = 0;
        while ((result = scanner.next()) != null) {
            assertNumKeys(result, numColsPerRow);
            Cell[] kvs = result.rawCells();
            for (int i = 0; i < numColsPerRow; i++) {
                assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
            }
            rowCount++;
        }
        scanner.close();
        assertTrue("Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows",
                rowCount == numRows);

    }

    /**
     * HBASE-861
     *    get with timestamp will return a value if there is a version with an
     *    earlier timestamp
     */
    @Test
    public void testJiraTest861() throws Exception {

        byte[] TABLE = Bytes.toBytes("testJiraTest861");
        byte[][] VALUES = makeNAscii(VALUE, 7);
        long[] STAMPS = makeStamps(7);

        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);

        // Insert three versions

        Put put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
        put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        ht.put(put);

        // Get the middle value
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);

        // Try to get one version before (expect fail)
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);

        // Try to get one version after (expect fail)
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);

        // Try same from storefile
        TEST_UTIL.flush();
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);

        // Insert two more versions surrounding others, into memstore
        put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
        put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
        ht.put(put);

        // Check we can get everything we should and can't get what we shouldn't
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);

        // Try same from two storefiles
        TEST_UTIL.flush();
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);

    }

    /**
     * HBASE-33
     *    Add a HTable get/obtainScanner method that retrieves all versions of a
     *    particular column and row between two timestamps
     */
    @Test
    public void testJiraTest33() throws Exception {

        byte[] TABLE = Bytes.toBytes("testJiraTest33");
        byte[][] VALUES = makeNAscii(VALUE, 7);
        long[] STAMPS = makeStamps(7);

        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);

        // Insert lots versions

        Put put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
        put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
        put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
        ht.put(put);

        getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
        getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
        getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
        getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);

        scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
        scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
        scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
        scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);

        // Try same from storefile
        TEST_UTIL.flush();

        getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
        getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
        getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
        getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);

        scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
        scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
        scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
        scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);

    }

    /**
     * HBASE-1014
     *    commit(BatchUpdate) method should return timestamp
     */
    @Test
    public void testJiraTest1014() throws Exception {

        byte[] TABLE = Bytes.toBytes("testJiraTest1014");

        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);

        long manualStamp = 12345;

        // Insert lots versions

        Put put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, manualStamp, VALUE);
        ht.put(put);

        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp - 1);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp + 1);

    }

    /**
     * HBASE-1182
     *    Scan for columns > some timestamp
     */
    @Test
    public void testJiraTest1182() throws Exception {

        byte[] TABLE = Bytes.toBytes("testJiraTest1182");
        byte[][] VALUES = makeNAscii(VALUE, 7);
        long[] STAMPS = makeStamps(7);

        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);

        // Insert lots versions

        Put put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
        put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
        put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
        ht.put(put);

        getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
        getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
        getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);

        scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
        scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
        scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);

        // Try same from storefile
        TEST_UTIL.flush();

        getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
        getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
        getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);

        scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
        scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
        scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
    }

    /**
     * HBASE-52
     *    Add a means of scanning over all versions
     */
    @Test
    public void testJiraTest52() throws Exception {
        byte[] TABLE = Bytes.toBytes("testJiraTest52");
        byte[][] VALUES = makeNAscii(VALUE, 7);
        long[] STAMPS = makeStamps(7);

        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);

        // Insert lots versions

        Put put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
        put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
        put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
        ht.put(put);

        getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);

        scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);

        // Try same from storefile
        TEST_UTIL.flush();

        getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);

        scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
    }

    //
    // Bulk Testers
    //

    private void getVersionRangeAndVerifyGreaterThan(HTable ht, byte[] row, byte[] family, byte[] qualifier,
            long[] stamps, byte[][] values, int start, int end) throws IOException {
        Get get = new Get(row);
        get.addColumn(family, qualifier);
        get.setMaxVersions(Integer.MAX_VALUE);
        get.setTimeRange(stamps[start + 1], Long.MAX_VALUE);
        Result result = ht.get(get);
        assertNResult(result, row, family, qualifier, stamps, values, start + 1, end);
    }

    private void getVersionRangeAndVerify(HTable ht, byte[] row, byte[] family, byte[] qualifier, long[] stamps,
            byte[][] values, int start, int end) throws IOException {
        Get get = new Get(row);
        get.addColumn(family, qualifier);
        get.setMaxVersions(Integer.MAX_VALUE);
        get.setTimeRange(stamps[start], stamps[end] + 1);
        Result result = ht.get(get);
        assertNResult(result, row, family, qualifier, stamps, values, start, end);
    }

    private void getAllVersionsAndVerify(HTable ht, byte[] row, byte[] family, byte[] qualifier, long[] stamps,
            byte[][] values, int start, int end) throws IOException {
        Get get = new Get(row);
        get.addColumn(family, qualifier);
        get.setMaxVersions(Integer.MAX_VALUE);
        Result result = ht.get(get);
        assertNResult(result, row, family, qualifier, stamps, values, start, end);
    }

    private void scanVersionRangeAndVerifyGreaterThan(HTable ht, byte[] row, byte[] family, byte[] qualifier,
            long[] stamps, byte[][] values, int start, int end) throws IOException {
        Scan scan = new Scan(row);
        scan.addColumn(family, qualifier);
        scan.setMaxVersions(Integer.MAX_VALUE);
        scan.setTimeRange(stamps[start + 1], Long.MAX_VALUE);
        Result result = getSingleScanResult(ht, scan);
        assertNResult(result, row, family, qualifier, stamps, values, start + 1, end);
    }

    private void scanVersionRangeAndVerify(HTable ht, byte[] row, byte[] family, byte[] qualifier, long[] stamps,
            byte[][] values, int start, int end) throws IOException {
        Scan scan = new Scan(row);
        scan.addColumn(family, qualifier);
        scan.setMaxVersions(Integer.MAX_VALUE);
        scan.setTimeRange(stamps[start], stamps[end] + 1);
        Result result = getSingleScanResult(ht, scan);
        assertNResult(result, row, family, qualifier, stamps, values, start, end);
    }

    private void scanAllVersionsAndVerify(HTable ht, byte[] row, byte[] family, byte[] qualifier, long[] stamps,
            byte[][] values, int start, int end) throws IOException {
        Scan scan = new Scan(row);
        scan.addColumn(family, qualifier);
        scan.setMaxVersions(Integer.MAX_VALUE);
        Result result = getSingleScanResult(ht, scan);
        assertNResult(result, row, family, qualifier, stamps, values, start, end);
    }

    private void getVersionAndVerify(HTable ht, byte[] row, byte[] family, byte[] qualifier, long stamp,
            byte[] value) throws Exception {
        Get get = new Get(row);
        get.addColumn(family, qualifier);
        get.setTimeStamp(stamp);
        get.setMaxVersions(Integer.MAX_VALUE);
        Result result = ht.get(get);
        assertSingleResult(result, row, family, qualifier, stamp, value);
    }

    private void getVersionAndVerifyMissing(HTable ht, byte[] row, byte[] family, byte[] qualifier, long stamp)
            throws Exception {
        Get get = new Get(row);
        get.addColumn(family, qualifier);
        get.setTimeStamp(stamp);
        get.setMaxVersions(Integer.MAX_VALUE);
        Result result = ht.get(get);
        assertEmptyResult(result);
    }

    private void scanVersionAndVerify(HTable ht, byte[] row, byte[] family, byte[] qualifier, long stamp,
            byte[] value) throws Exception {
        Scan scan = new Scan(row);
        scan.addColumn(family, qualifier);
        scan.setTimeStamp(stamp);
        scan.setMaxVersions(Integer.MAX_VALUE);
        Result result = getSingleScanResult(ht, scan);
        assertSingleResult(result, row, family, qualifier, stamp, value);
    }

    private void scanVersionAndVerifyMissing(HTable ht, byte[] row, byte[] family, byte[] qualifier, long stamp)
            throws Exception {
        Scan scan = new Scan(row);
        scan.addColumn(family, qualifier);
        scan.setTimeStamp(stamp);
        scan.setMaxVersions(Integer.MAX_VALUE);
        Result result = getSingleScanResult(ht, scan);
        assertNullResult(result);
    }

    private void getTestNull(HTable ht, byte[] row, byte[] family, byte[] value) throws Exception {

        Get get = new Get(row);
        get.addColumn(family, null);
        Result result = ht.get(get);
        assertSingleResult(result, row, family, null, value);

        get = new Get(row);
        get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
        result = ht.get(get);
        assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);

        get = new Get(row);
        get.addFamily(family);
        result = ht.get(get);
        assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);

        get = new Get(row);
        result = ht.get(get);
        assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);

    }

    private void scanTestNull(HTable ht, byte[] row, byte[] family, byte[] value) throws Exception {
        scanTestNull(ht, row, family, value, false);
    }

    private void scanTestNull(HTable ht, byte[] row, byte[] family, byte[] value, boolean isReversedScan)
            throws Exception {

        Scan scan = new Scan();
        scan.setReversed(isReversedScan);
        scan.addColumn(family, null);
        Result result = getSingleScanResult(ht, scan);
        assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);

        scan = new Scan();
        scan.setReversed(isReversedScan);
        scan.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);

        scan = new Scan();
        scan.setReversed(isReversedScan);
        scan.addFamily(family);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);

        scan = new Scan();
        scan.setReversed(isReversedScan);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);

    }

    private void singleRowGetTest(HTable ht, byte[][] ROWS, byte[][] FAMILIES, byte[][] QUALIFIERS, byte[][] VALUES)
            throws Exception {

        // Single column from memstore
        Get get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[4], QUALIFIERS[0]);
        Result result = ht.get(get);
        assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);

        // Single column from storefile
        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[2], QUALIFIERS[2]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);

        // Single column from storefile, family match
        get = new Get(ROWS[0]);
        get.addFamily(FAMILIES[7]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);

        // Two columns, one from memstore one from storefile, same family,
        // wildcard match
        get = new Get(ROWS[0]);
        get.addFamily(FAMILIES[4]);
        result = ht.get(get);
        assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0], FAMILIES[4], QUALIFIERS[4],
                VALUES[4]);

        // Two columns, one from memstore one from storefile, same family,
        // explicit match
        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[4], QUALIFIERS[0]);
        get.addColumn(FAMILIES[4], QUALIFIERS[4]);
        result = ht.get(get);
        assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0], FAMILIES[4], QUALIFIERS[4],
                VALUES[4]);

        // Three column, one from memstore two from storefile, different families,
        // wildcard match
        get = new Get(ROWS[0]);
        get.addFamily(FAMILIES[4]);
        get.addFamily(FAMILIES[7]);
        result = ht.get(get);
        assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
                new int[][] { { 4, 0, 0 }, { 4, 4, 4 }, { 7, 7, 7 } });

        // Multiple columns from everywhere storefile, many family, wildcard
        get = new Get(ROWS[0]);
        get.addFamily(FAMILIES[2]);
        get.addFamily(FAMILIES[4]);
        get.addFamily(FAMILIES[6]);
        get.addFamily(FAMILIES[7]);
        result = ht.get(get);
        assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, new int[][] { { 2, 2, 2 }, { 2, 4, 4 },
                { 4, 0, 0 }, { 4, 4, 4 }, { 6, 6, 6 }, { 6, 7, 7 }, { 7, 7, 7 } });

        // Multiple columns from everywhere storefile, many family, wildcard
        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[2], QUALIFIERS[2]);
        get.addColumn(FAMILIES[2], QUALIFIERS[4]);
        get.addColumn(FAMILIES[4], QUALIFIERS[0]);
        get.addColumn(FAMILIES[4], QUALIFIERS[4]);
        get.addColumn(FAMILIES[6], QUALIFIERS[6]);
        get.addColumn(FAMILIES[6], QUALIFIERS[7]);
        get.addColumn(FAMILIES[7], QUALIFIERS[7]);
        get.addColumn(FAMILIES[7], QUALIFIERS[8]);
        result = ht.get(get);
        assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, new int[][] { { 2, 2, 2 }, { 2, 4, 4 },
                { 4, 0, 0 }, { 4, 4, 4 }, { 6, 6, 6 }, { 6, 7, 7 }, { 7, 7, 7 } });

        // Everything
        get = new Get(ROWS[0]);
        result = ht.get(get);
        assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, new int[][] { { 2, 2, 2 }, { 2, 4, 4 },
                { 4, 0, 0 }, { 4, 4, 4 }, { 6, 6, 6 }, { 6, 7, 7 }, { 7, 7, 7 }, { 9, 0, 0 } });

        // Get around inserted columns

        get = new Get(ROWS[1]);
        result = ht.get(get);
        assertEmptyResult(result);

        get = new Get(ROWS[0]);
        get.addColumn(FAMILIES[4], QUALIFIERS[3]);
        get.addColumn(FAMILIES[2], QUALIFIERS[3]);
        result = ht.get(get);
        assertEmptyResult(result);

    }

    private void singleRowScanTest(HTable ht, byte[][] ROWS, byte[][] FAMILIES, byte[][] QUALIFIERS,
            byte[][] VALUES) throws Exception {

        // Single column from memstore
        Scan scan = new Scan();
        scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
        Result result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);

        // Single column from storefile
        scan = new Scan();
        scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);

        // Single column from storefile, family match
        scan = new Scan();
        scan.addFamily(FAMILIES[7]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);

        // Two columns, one from memstore one from storefile, same family,
        // wildcard match
        scan = new Scan();
        scan.addFamily(FAMILIES[4]);
        result = getSingleScanResult(ht, scan);
        assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0], FAMILIES[4], QUALIFIERS[4],
                VALUES[4]);

        // Two columns, one from memstore one from storefile, same family,
        // explicit match
        scan = new Scan();
        scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
        scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
        result = getSingleScanResult(ht, scan);
        assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0], FAMILIES[4], QUALIFIERS[4],
                VALUES[4]);

        // Three column, one from memstore two from storefile, different families,
        // wildcard match
        scan = new Scan();
        scan.addFamily(FAMILIES[4]);
        scan.addFamily(FAMILIES[7]);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
                new int[][] { { 4, 0, 0 }, { 4, 4, 4 }, { 7, 7, 7 } });

        // Multiple columns from everywhere storefile, many family, wildcard
        scan = new Scan();
        scan.addFamily(FAMILIES[2]);
        scan.addFamily(FAMILIES[4]);
        scan.addFamily(FAMILIES[6]);
        scan.addFamily(FAMILIES[7]);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, new int[][] { { 2, 2, 2 }, { 2, 4, 4 },
                { 4, 0, 0 }, { 4, 4, 4 }, { 6, 6, 6 }, { 6, 7, 7 }, { 7, 7, 7 } });

        // Multiple columns from everywhere storefile, many family, wildcard
        scan = new Scan();
        scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
        scan.addColumn(FAMILIES[2], QUALIFIERS[4]);
        scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
        scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
        scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
        scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
        scan.addColumn(FAMILIES[7], QUALIFIERS[7]);
        scan.addColumn(FAMILIES[7], QUALIFIERS[8]);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, new int[][] { { 2, 2, 2 }, { 2, 4, 4 },
                { 4, 0, 0 }, { 4, 4, 4 }, { 6, 6, 6 }, { 6, 7, 7 }, { 7, 7, 7 } });

        // Everything
        scan = new Scan();
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, new int[][] { { 2, 2, 2 }, { 2, 4, 4 },
                { 4, 0, 0 }, { 4, 4, 4 }, { 6, 6, 6 }, { 6, 7, 7 }, { 7, 7, 7 }, { 9, 0, 0 } });

        // Scan around inserted columns

        scan = new Scan(ROWS[1]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        scan = new Scan();
        scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
        scan.addColumn(FAMILIES[2], QUALIFIERS[3]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);
    }

    /**
     * Verify a single column using gets.
     * Expects family and qualifier arrays to be valid for at least
     * the range:  idx-2 < idx < idx+2
     */
    private void getVerifySingleColumn(HTable ht, byte[][] ROWS, int ROWIDX, byte[][] FAMILIES, int FAMILYIDX,
            byte[][] QUALIFIERS, int QUALIFIERIDX, byte[][] VALUES, int VALUEIDX) throws Exception {

        Get get = new Get(ROWS[ROWIDX]);
        Result result = ht.get(get);
        assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);

        get = new Get(ROWS[ROWIDX]);
        get.addFamily(FAMILIES[FAMILYIDX]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);

        get = new Get(ROWS[ROWIDX]);
        get.addFamily(FAMILIES[FAMILYIDX - 2]);
        get.addFamily(FAMILIES[FAMILYIDX]);
        get.addFamily(FAMILIES[FAMILYIDX + 2]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);

        get = new Get(ROWS[ROWIDX]);
        get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[0]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);

        get = new Get(ROWS[ROWIDX]);
        get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[1]);
        get.addFamily(FAMILIES[FAMILYIDX]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);

        get = new Get(ROWS[ROWIDX]);
        get.addFamily(FAMILIES[FAMILYIDX]);
        get.addColumn(FAMILIES[FAMILYIDX + 1], QUALIFIERS[1]);
        get.addColumn(FAMILIES[FAMILYIDX - 2], QUALIFIERS[1]);
        get.addFamily(FAMILIES[FAMILYIDX - 1]);
        get.addFamily(FAMILIES[FAMILYIDX + 2]);
        result = ht.get(get);
        assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);

    }

    /**
     * Verify a single column using scanners.
     * Expects family and qualifier arrays to be valid for at least
     * the range:  idx-2 to idx+2
     * Expects row array to be valid for at least idx to idx+2
     */
    private void scanVerifySingleColumn(HTable ht, byte[][] ROWS, int ROWIDX, byte[][] FAMILIES, int FAMILYIDX,
            byte[][] QUALIFIERS, int QUALIFIERIDX, byte[][] VALUES, int VALUEIDX) throws Exception {

        Scan scan = new Scan();
        Result result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);

        scan = new Scan(ROWS[ROWIDX]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);

        scan = new Scan(ROWS[ROWIDX], ROWS[ROWIDX + 1]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);

        scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX + 1]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);

        scan = new Scan();
        scan.addFamily(FAMILIES[FAMILYIDX]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);

        scan = new Scan();
        scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);

        scan = new Scan();
        scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX + 1]);
        scan.addFamily(FAMILIES[FAMILYIDX]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);

        scan = new Scan();
        scan.addColumn(FAMILIES[FAMILYIDX - 1], QUALIFIERS[QUALIFIERIDX + 1]);
        scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
        scan.addFamily(FAMILIES[FAMILYIDX + 1]);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);

    }

    /**
     * Verify we do not read any values by accident around a single column
     * Same requirements as getVerifySingleColumn
     */
    private void getVerifySingleEmpty(HTable ht, byte[][] ROWS, int ROWIDX, byte[][] FAMILIES, int FAMILYIDX,
            byte[][] QUALIFIERS, int QUALIFIERIDX) throws Exception {

        Get get = new Get(ROWS[ROWIDX]);
        get.addFamily(FAMILIES[4]);
        get.addColumn(FAMILIES[4], QUALIFIERS[1]);
        Result result = ht.get(get);
        assertEmptyResult(result);

        get = new Get(ROWS[ROWIDX]);
        get.addFamily(FAMILIES[4]);
        get.addColumn(FAMILIES[4], QUALIFIERS[2]);
        result = ht.get(get);
        assertEmptyResult(result);

        get = new Get(ROWS[ROWIDX]);
        get.addFamily(FAMILIES[3]);
        get.addColumn(FAMILIES[4], QUALIFIERS[2]);
        get.addFamily(FAMILIES[5]);
        result = ht.get(get);
        assertEmptyResult(result);

        get = new Get(ROWS[ROWIDX + 1]);
        result = ht.get(get);
        assertEmptyResult(result);

    }

    private void scanVerifySingleEmpty(HTable ht, byte[][] ROWS, int ROWIDX, byte[][] FAMILIES, int FAMILYIDX,
            byte[][] QUALIFIERS, int QUALIFIERIDX) throws Exception {

        Scan scan = new Scan(ROWS[ROWIDX + 1]);
        Result result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        scan = new Scan(ROWS[ROWIDX + 1], ROWS[ROWIDX + 2]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        scan = new Scan();
        scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX + 1]);
        scan.addFamily(FAMILIES[FAMILYIDX - 1]);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

    }

    //
    // Verifiers
    //

    private void assertKey(Cell key, byte[] row, byte[] family, byte[] qualifier, byte[] value) throws Exception {
        assertTrue("Expected row [" + Bytes.toString(row) + "] " + "Got row ["
                + Bytes.toString(CellUtil.cloneRow(key)) + "]", equals(row, CellUtil.cloneRow(key)));
        assertTrue(
                "Expected family [" + Bytes.toString(family) + "] " + "Got family ["
                        + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
                equals(family, CellUtil.cloneFamily(key)));
        assertTrue(
                "Expected qualifier [" + Bytes.toString(qualifier) + "] " + "Got qualifier ["
                        + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
                equals(qualifier, CellUtil.cloneQualifier(key)));
        assertTrue("Expected value [" + Bytes.toString(value) + "] " + "Got value ["
                + Bytes.toString(CellUtil.cloneValue(key)) + "]", equals(value, CellUtil.cloneValue(key)));
    }

    private void assertIncrementKey(Cell key, byte[] row, byte[] family, byte[] qualifier, long value)
            throws Exception {
        assertTrue("Expected row [" + Bytes.toString(row) + "] " + "Got row ["
                + Bytes.toString(CellUtil.cloneRow(key)) + "]", equals(row, CellUtil.cloneRow(key)));
        assertTrue(
                "Expected family [" + Bytes.toString(family) + "] " + "Got family ["
                        + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
                equals(family, CellUtil.cloneFamily(key)));
        assertTrue(
                "Expected qualifier [" + Bytes.toString(qualifier) + "] " + "Got qualifier ["
                        + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
                equals(qualifier, CellUtil.cloneQualifier(key)));
        assertTrue("Expected value [" + value + "] " + "Got value [" + Bytes.toLong(CellUtil.cloneValue(key)) + "]",
                Bytes.toLong(CellUtil.cloneValue(key)) == value);
    }

    private void assertNumKeys(Result result, int n) throws Exception {
        assertTrue("Expected " + n + " keys but got " + result.size(), result.size() == n);
    }

    private void assertNResult(Result result, byte[] row, byte[][] families, byte[][] qualifiers, byte[][] values,
            int[][] idxs) throws Exception {
        assertTrue(
                "Expected row [" + Bytes.toString(row) + "] " + "Got row [" + Bytes.toString(result.getRow()) + "]",
                equals(row, result.getRow()));
        assertTrue("Expected " + idxs.length + " keys but result contains " + result.size(),
                result.size() == idxs.length);

        Cell[] keys = result.rawCells();

        for (int i = 0; i < keys.length; i++) {
            byte[] family = families[idxs[i][0]];
            byte[] qualifier = qualifiers[idxs[i][1]];
            byte[] value = values[idxs[i][2]];
            Cell key = keys[i];

            byte[] famb = CellUtil.cloneFamily(key);
            byte[] qualb = CellUtil.cloneQualifier(key);
            byte[] valb = CellUtil.cloneValue(key);
            assertTrue("(" + i + ") Expected family [" + Bytes.toString(family) + "] " + "Got family ["
                    + Bytes.toString(famb) + "]", equals(family, famb));
            assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier) + "] " + "Got qualifier ["
                    + Bytes.toString(qualb) + "]", equals(qualifier, qualb));
            assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] " + "Got value ["
                    + Bytes.toString(valb) + "]", equals(value, valb));
        }
    }

    private void assertNResult(Result result, byte[] row, byte[] family, byte[] qualifier, long[] stamps,
            byte[][] values, int start, int end) throws IOException {
        assertTrue(
                "Expected row [" + Bytes.toString(row) + "] " + "Got row [" + Bytes.toString(result.getRow()) + "]",
                equals(row, result.getRow()));
        int expectedResults = end - start + 1;
        assertEquals(expectedResults, result.size());

        Cell[] keys = result.rawCells();

        for (int i = 0; i < keys.length; i++) {
            byte[] value = values[end - i];
            long ts = stamps[end - i];
            Cell key = keys[i];

            assertTrue(
                    "(" + i + ") Expected family [" + Bytes.toString(family) + "] " + "Got family ["
                            + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
                    CellUtil.matchingFamily(key, family));
            assertTrue(
                    "(" + i + ") Expected qualifier [" + Bytes.toString(qualifier) + "] " + "Got qualifier ["
                            + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
                    CellUtil.matchingQualifier(key, qualifier));
            assertTrue("Expected ts [" + ts + "] " + "Got ts [" + key.getTimestamp() + "]",
                    ts == key.getTimestamp());
            assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] " + "Got value ["
                    + Bytes.toString(CellUtil.cloneValue(key)) + "]", CellUtil.matchingValue(key, value));
        }
    }

    /**
     * Validate that result contains two specified keys, exactly.
     * It is assumed key A sorts before key B.
     */
    private void assertDoubleResult(Result result, byte[] row, byte[] familyA, byte[] qualifierA, byte[] valueA,
            byte[] familyB, byte[] qualifierB, byte[] valueB) throws Exception {
        assertTrue(
                "Expected row [" + Bytes.toString(row) + "] " + "Got row [" + Bytes.toString(result.getRow()) + "]",
                equals(row, result.getRow()));
        assertTrue("Expected two keys but result contains " + result.size(), result.size() == 2);
        Cell[] kv = result.rawCells();
        Cell kvA = kv[0];
        assertTrue(
                "(A) Expected family [" + Bytes.toString(familyA) + "] " + "Got family ["
                        + Bytes.toString(CellUtil.cloneFamily(kvA)) + "]",
                equals(familyA, CellUtil.cloneFamily(kvA)));
        assertTrue(
                "(A) Expected qualifier [" + Bytes.toString(qualifierA) + "] " + "Got qualifier ["
                        + Bytes.toString(CellUtil.cloneQualifier(kvA)) + "]",
                equals(qualifierA, CellUtil.cloneQualifier(kvA)));
        assertTrue("(A) Expected value [" + Bytes.toString(valueA) + "] " + "Got value ["
                + Bytes.toString(CellUtil.cloneValue(kvA)) + "]", equals(valueA, CellUtil.cloneValue(kvA)));
        Cell kvB = kv[1];
        assertTrue(
                "(B) Expected family [" + Bytes.toString(familyB) + "] " + "Got family ["
                        + Bytes.toString(CellUtil.cloneFamily(kvB)) + "]",
                equals(familyB, CellUtil.cloneFamily(kvB)));
        assertTrue(
                "(B) Expected qualifier [" + Bytes.toString(qualifierB) + "] " + "Got qualifier ["
                        + Bytes.toString(CellUtil.cloneQualifier(kvB)) + "]",
                equals(qualifierB, CellUtil.cloneQualifier(kvB)));
        assertTrue("(B) Expected value [" + Bytes.toString(valueB) + "] " + "Got value ["
                + Bytes.toString(CellUtil.cloneValue(kvB)) + "]", equals(valueB, CellUtil.cloneValue(kvB)));
    }

    private void assertSingleResult(Result result, byte[] row, byte[] family, byte[] qualifier, byte[] value)
            throws Exception {
        assertTrue(
                "Expected row [" + Bytes.toString(row) + "] " + "Got row [" + Bytes.toString(result.getRow()) + "]",
                equals(row, result.getRow()));
        assertTrue("Expected a single key but result contains " + result.size(), result.size() == 1);
        Cell kv = result.rawCells()[0];
        assertTrue("Expected family [" + Bytes.toString(family) + "] " + "Got family ["
                + Bytes.toString(CellUtil.cloneFamily(kv)) + "]", equals(family, CellUtil.cloneFamily(kv)));
        assertTrue(
                "Expected qualifier [" + Bytes.toString(qualifier) + "] " + "Got qualifier ["
                        + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
                equals(qualifier, CellUtil.cloneQualifier(kv)));
        assertTrue("Expected value [" + Bytes.toString(value) + "] " + "Got value ["
                + Bytes.toString(CellUtil.cloneValue(kv)) + "]", equals(value, CellUtil.cloneValue(kv)));
    }

    private void assertSingleResult(Result result, byte[] row, byte[] family, byte[] qualifier, long ts,
            byte[] value) throws Exception {
        assertTrue(
                "Expected row [" + Bytes.toString(row) + "] " + "Got row [" + Bytes.toString(result.getRow()) + "]",
                equals(row, result.getRow()));
        assertTrue("Expected a single key but result contains " + result.size(), result.size() == 1);
        Cell kv = result.rawCells()[0];
        assertTrue("Expected family [" + Bytes.toString(family) + "] " + "Got family ["
                + Bytes.toString(CellUtil.cloneFamily(kv)) + "]", equals(family, CellUtil.cloneFamily(kv)));
        assertTrue(
                "Expected qualifier [" + Bytes.toString(qualifier) + "] " + "Got qualifier ["
                        + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
                equals(qualifier, CellUtil.cloneQualifier(kv)));
        assertTrue("Expected ts [" + ts + "] " + "Got ts [" + kv.getTimestamp() + "]", ts == kv.getTimestamp());
        assertTrue("Expected value [" + Bytes.toString(value) + "] " + "Got value ["
                + Bytes.toString(CellUtil.cloneValue(kv)) + "]", equals(value, CellUtil.cloneValue(kv)));
    }

    private void assertEmptyResult(Result result) throws Exception {
        assertTrue("expected an empty result but result contains " + result.size() + " keys", result.isEmpty());
    }

    private void assertNullResult(Result result) throws Exception {
        assertTrue("expected null result but received a non-null result", result == null);
    }

    //
    // Helpers
    //

    private Result getSingleScanResult(HTable ht, Scan scan) throws IOException {
        ResultScanner scanner = ht.getScanner(scan);
        Result result = scanner.next();
        scanner.close();
        return result;
    }

    private byte[][] makeNAscii(byte[] base, int n) {
        if (n > 256) {
            return makeNBig(base, n);
        }
        byte[][] ret = new byte[n][];
        for (int i = 0; i < n; i++) {
            byte[] tail = Bytes.toBytes(Integer.toString(i));
            ret[i] = Bytes.add(base, tail);
        }
        return ret;
    }

    private byte[][] makeN(byte[] base, int n) {
        if (n > 256) {
            return makeNBig(base, n);
        }
        byte[][] ret = new byte[n][];
        for (int i = 0; i < n; i++) {
            ret[i] = Bytes.add(base, new byte[] { (byte) i });
        }
        return ret;
    }

    private byte[][] makeNBig(byte[] base, int n) {
        byte[][] ret = new byte[n][];
        for (int i = 0; i < n; i++) {
            int byteA = (i % 256);
            int byteB = (i >> 8);
            ret[i] = Bytes.add(base, new byte[] { (byte) byteB, (byte) byteA });
        }
        return ret;
    }

    private long[] makeStamps(int n) {
        long[] stamps = new long[n];
        for (int i = 0; i < n; i++)
            stamps[i] = i + 1;
        return stamps;
    }

    private boolean equals(byte[] left, byte[] right) {
        if (left == null && right == null)
            return true;
        if (left == null && right.length == 0)
            return true;
        if (right == null && left.length == 0)
            return true;
        return Bytes.equals(left, right);
    }

    @Test
    public void testDuplicateVersions() throws Exception {
        byte[] TABLE = Bytes.toBytes("testDuplicateVersions");

        long[] STAMPS = makeStamps(20);
        byte[][] VALUES = makeNAscii(VALUE, 20);

        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);

        // Insert 4 versions of same column
        Put put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
        ht.put(put);

        // Verify we can get each one properly
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);

        // Verify we don't accidentally get others
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);

        // Ensure maxVersions in query is respected
        Get get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIER);
        get.setMaxVersions(2);
        Result result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
                new byte[][] { VALUES[4], VALUES[5] }, 0, 1);

        Scan scan = new Scan(ROW);
        scan.addColumn(FAMILY, QUALIFIER);
        scan.setMaxVersions(2);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
                new byte[][] { VALUES[4], VALUES[5] }, 0, 1);

        // Flush and redo

        TEST_UTIL.flush();

        // Verify we can get each one properly
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);

        // Verify we don't accidentally get others
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);

        // Ensure maxVersions in query is respected
        get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIER);
        get.setMaxVersions(2);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
                new byte[][] { VALUES[4], VALUES[5] }, 0, 1);

        scan = new Scan(ROW);
        scan.addColumn(FAMILY, QUALIFIER);
        scan.setMaxVersions(2);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
                new byte[][] { VALUES[4], VALUES[5] }, 0, 1);

        // Add some memstore and retest

        // Insert 4 more versions of same column and a dupe
        put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
        put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
        put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
        put.add(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
        put.add(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
        ht.put(put);

        // Ensure maxVersions in query is respected
        get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIER);
        get.setMaxVersions(7);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
                new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8] }, 0,
                6);

        scan = new Scan(ROW);
        scan.addColumn(FAMILY, QUALIFIER);
        scan.setMaxVersions(7);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
                new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8] }, 0,
                6);

        get = new Get(ROW);
        get.setMaxVersions(7);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
                new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8] }, 0,
                6);

        scan = new Scan(ROW);
        scan.setMaxVersions(7);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
                new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8] }, 0,
                6);

        // Verify we can get each one properly
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
        getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
        scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);

        // Verify we don't accidentally get others
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
        getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
        scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);

        // Ensure maxVersions of table is respected

        TEST_UTIL.flush();

        // Insert 4 more versions of same column and a dupe
        put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
        put.add(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
        put.add(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
        put.add(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
        ht.put(put);

        get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIER);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
                        STAMPS[11], STAMPS[13], STAMPS[15] },
                new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
                        VALUES[11], VALUES[13], VALUES[15] },
                0, 9);

        scan = new Scan(ROW);
        scan.addColumn(FAMILY, QUALIFIER);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
                        STAMPS[11], STAMPS[13], STAMPS[15] },
                new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
                        VALUES[11], VALUES[13], VALUES[15] },
                0, 9);

        // Delete a version in the memstore and a version in a storefile
        Delete delete = new Delete(ROW);
        delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[11]);
        delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[7]);
        ht.delete(delete);

        // Test that it's gone
        get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIER);
        get.setMaxVersions(Integer.MAX_VALUE);
        result = ht.get(get);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9],
                        STAMPS[13], STAMPS[15] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
                        VALUES[9], VALUES[13], VALUES[15] },
                0, 9);

        scan = new Scan(ROW);
        scan.addColumn(FAMILY, QUALIFIER);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILY, QUALIFIER,
                new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9],
                        STAMPS[13], STAMPS[15] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
                        VALUES[9], VALUES[13], VALUES[15] },
                0, 9);
    }

    @Test
    public void testUpdates() throws Exception {

        byte[] TABLE = Bytes.toBytes("testUpdates");
        HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);

        // Write a column with values at timestamp 1, 2 and 3
        byte[] row = Bytes.toBytes("row1");
        byte[] qualifier = Bytes.toBytes("myCol");
        Put put = new Put(row);
        put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
        hTable.put(put);

        put = new Put(row);
        put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
        hTable.put(put);

        put = new Put(row);
        put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
        hTable.put(put);

        Get get = new Get(row);
        get.addColumn(FAMILY, qualifier);
        get.setMaxVersions();

        // Check that the column indeed has the right values at timestamps 1 and
        // 2
        Result result = hTable.get(get);
        NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
        assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
        assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));

        // Update the value at timestamp 1
        put = new Put(row);
        put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
        hTable.put(put);

        // Update the value at timestamp 2
        put = new Put(row);
        put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
        hTable.put(put);

        // Check that the values at timestamp 2 and 1 got updated
        result = hTable.get(get);
        navigableMap = result.getMap().get(FAMILY).get(qualifier);
        assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
        assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
    }

    @Test
    public void testUpdatesWithMajorCompaction() throws Exception {

        String tableName = "testUpdatesWithMajorCompaction";
        byte[] TABLE = Bytes.toBytes(tableName);
        HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
        HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());

        // Write a column with values at timestamp 1, 2 and 3
        byte[] row = Bytes.toBytes("row2");
        byte[] qualifier = Bytes.toBytes("myCol");
        Put put = new Put(row);
        put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
        hTable.put(put);

        put = new Put(row);
        put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
        hTable.put(put);

        put = new Put(row);
        put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
        hTable.put(put);

        Get get = new Get(row);
        get.addColumn(FAMILY, qualifier);
        get.setMaxVersions();

        // Check that the column indeed has the right values at timestamps 1 and
        // 2
        Result result = hTable.get(get);
        NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
        assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
        assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));

        // Trigger a major compaction
        admin.flush(tableName);
        admin.majorCompact(tableName);
        Thread.sleep(6000);

        // Update the value at timestamp 1
        put = new Put(row);
        put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
        hTable.put(put);

        // Update the value at timestamp 2
        put = new Put(row);
        put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
        hTable.put(put);

        // Trigger a major compaction
        admin.flush(tableName);
        admin.majorCompact(tableName);
        Thread.sleep(6000);

        // Check that the values at timestamp 2 and 1 got updated
        result = hTable.get(get);
        navigableMap = result.getMap().get(FAMILY).get(qualifier);
        assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
        assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
    }

    @Test
    public void testMajorCompactionBetweenTwoUpdates() throws Exception {

        String tableName = "testMajorCompactionBetweenTwoUpdates";
        byte[] TABLE = Bytes.toBytes(tableName);
        HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
        HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());

        // Write a column with values at timestamp 1, 2 and 3
        byte[] row = Bytes.toBytes("row3");
        byte[] qualifier = Bytes.toBytes("myCol");
        Put put = new Put(row);
        put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
        hTable.put(put);

        put = new Put(row);
        put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
        hTable.put(put);

        put = new Put(row);
        put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
        hTable.put(put);

        Get get = new Get(row);
        get.addColumn(FAMILY, qualifier);
        get.setMaxVersions();

        // Check that the column indeed has the right values at timestamps 1 and
        // 2
        Result result = hTable.get(get);
        NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
        assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
        assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));

        // Trigger a major compaction
        admin.flush(tableName);
        admin.majorCompact(tableName);
        Thread.sleep(6000);

        // Update the value at timestamp 1
        put = new Put(row);
        put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
        hTable.put(put);

        // Trigger a major compaction
        admin.flush(tableName);
        admin.majorCompact(tableName);
        Thread.sleep(6000);

        // Update the value at timestamp 2
        put = new Put(row);
        put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
        hTable.put(put);

        // Trigger a major compaction
        admin.flush(tableName);
        admin.majorCompact(tableName);
        Thread.sleep(6000);

        // Check that the values at timestamp 2 and 1 got updated
        result = hTable.get(get);
        navigableMap = result.getMap().get(FAMILY).get(qualifier);

        assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
        assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
    }

    @Test
    public void testGet_EmptyTable() throws IOException {
        HTable table = TEST_UTIL.createTable(Bytes.toBytes("testGet_EmptyTable"), FAMILY);
        Get get = new Get(ROW);
        get.addFamily(FAMILY);
        Result r = table.get(get);
        assertTrue(r.isEmpty());
    }

    @Test
    public void testGet_NullQualifier() throws IOException {
        HTable table = TEST_UTIL.createTable(Bytes.toBytes("testGet_NullQualifier"), FAMILY);
        Put put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, VALUE);
        table.put(put);

        put = new Put(ROW);
        put.add(FAMILY, null, VALUE);
        table.put(put);
        LOG.info("Row put");

        Get get = new Get(ROW);
        get.addColumn(FAMILY, null);
        Result r = table.get(get);
        assertEquals(1, r.size());

        get = new Get(ROW);
        get.addFamily(FAMILY);
        r = table.get(get);
        assertEquals(2, r.size());
    }

    @Test
    public void testGet_NonExistentRow() throws IOException {
        HTable table = TEST_UTIL.createTable(Bytes.toBytes("testGet_NonExistentRow"), FAMILY);
        Put put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, VALUE);
        table.put(put);
        LOG.info("Row put");

        Get get = new Get(ROW);
        get.addFamily(FAMILY);
        Result r = table.get(get);
        assertFalse(r.isEmpty());
        System.out.println("Row retrieved successfully");

        byte[] missingrow = Bytes.toBytes("missingrow");
        get = new Get(missingrow);
        get.addFamily(FAMILY);
        r = table.get(get);
        assertTrue(r.isEmpty());
        LOG.info("Row missing as it should be");
    }

    @Test
    public void testPut() throws IOException {
        final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
        final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
        final byte[] row1 = Bytes.toBytes("row1");
        final byte[] row2 = Bytes.toBytes("row2");
        final byte[] value = Bytes.toBytes("abcd");
        HTable table = TEST_UTIL.createTable(Bytes.toBytes("testPut"),
                new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY });
        Put put = new Put(row1);
        put.add(CONTENTS_FAMILY, null, value);
        table.put(put);

        put = new Put(row2);
        put.add(CONTENTS_FAMILY, null, value);

        assertEquals(put.size(), 1);
        assertEquals(put.getFamilyCellMap().get(CONTENTS_FAMILY).size(), 1);

        // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
        KeyValue kv = (KeyValue) put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);

        assertTrue(Bytes.equals(kv.getFamily(), CONTENTS_FAMILY));
        // will it return null or an empty byte array?
        assertTrue(Bytes.equals(kv.getQualifier(), new byte[0]));

        assertTrue(Bytes.equals(kv.getValue(), value));

        table.put(put);

        Scan scan = new Scan();
        scan.addColumn(CONTENTS_FAMILY, null);
        ResultScanner scanner = table.getScanner(scan);
        for (Result r : scanner) {
            for (Cell key : r.rawCells()) {
                System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
            }
        }
    }

    @Test
    public void testPutNoCF() throws IOException {
        final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
        final byte[] VAL = Bytes.toBytes(100);
        HTable table = TEST_UTIL.createTable(Bytes.toBytes("testPutNoCF"), new byte[][] { FAMILY });

        boolean caughtNSCFE = false;

        try {
            Put p = new Put(ROW);
            p.add(BAD_FAM, QUALIFIER, VAL);
            table.put(p);
        } catch (RetriesExhaustedWithDetailsException e) {
            caughtNSCFE = e.getCause(0) instanceof NoSuchColumnFamilyException;
        }
        assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);

    }

    @Test
    public void testRowsPut() throws IOException {
        final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
        final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
        final int NB_BATCH_ROWS = 10;
        final byte[] value = Bytes.toBytes("abcd");
        HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPut"),
                new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY });
        ArrayList<Put> rowsUpdate = new ArrayList<Put>();
        for (int i = 0; i < NB_BATCH_ROWS; i++) {
            byte[] row = Bytes.toBytes("row" + i);
            Put put = new Put(row);
            put.setDurability(Durability.SKIP_WAL);
            put.add(CONTENTS_FAMILY, null, value);
            rowsUpdate.add(put);
        }
        table.put(rowsUpdate);
        Scan scan = new Scan();
        scan.addFamily(CONTENTS_FAMILY);
        ResultScanner scanner = table.getScanner(scan);
        int nbRows = 0;
        for (@SuppressWarnings("unused")
        Result row : scanner)
            nbRows++;
        assertEquals(NB_BATCH_ROWS, nbRows);
    }

    @Test
    public void testRowsPutBufferedOneFlush() throws IOException {
        final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
        final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
        final byte[] value = Bytes.toBytes("abcd");
        final int NB_BATCH_ROWS = 10;
        HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedOneFlush"),
                new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY });
        table.setAutoFlush(false, true);
        ArrayList<Put> rowsUpdate = new ArrayList<Put>();
        for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
            byte[] row = Bytes.toBytes("row" + i);
            Put put = new Put(row);
            put.setDurability(Durability.SKIP_WAL);
            put.add(CONTENTS_FAMILY, null, value);
            rowsUpdate.add(put);
        }
        table.put(rowsUpdate);

        Scan scan = new Scan();
        scan.addFamily(CONTENTS_FAMILY);
        ResultScanner scanner = table.getScanner(scan);
        int nbRows = 0;
        for (@SuppressWarnings("unused")
        Result row : scanner)
            nbRows++;
        assertEquals(0, nbRows);
        scanner.close();

        table.flushCommits();

        scan = new Scan();
        scan.addFamily(CONTENTS_FAMILY);
        scanner = table.getScanner(scan);
        nbRows = 0;
        for (@SuppressWarnings("unused")
        Result row : scanner)
            nbRows++;
        assertEquals(NB_BATCH_ROWS * 10, nbRows);
    }

    @Test
    public void testRowsPutBufferedManyManyFlushes() throws IOException {
        final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
        final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
        final byte[] value = Bytes.toBytes("abcd");
        final int NB_BATCH_ROWS = 10;
        HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedManyManyFlushes"),
                new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY });
        table.setAutoFlush(false, true);
        table.setWriteBufferSize(10);
        ArrayList<Put> rowsUpdate = new ArrayList<Put>();
        for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
            byte[] row = Bytes.toBytes("row" + i);
            Put put = new Put(row);
            put.setDurability(Durability.SKIP_WAL);
            put.add(CONTENTS_FAMILY, null, value);
            rowsUpdate.add(put);
        }
        table.put(rowsUpdate);

        table.flushCommits();

        Scan scan = new Scan();
        scan.addFamily(CONTENTS_FAMILY);
        ResultScanner scanner = table.getScanner(scan);
        int nbRows = 0;
        for (@SuppressWarnings("unused")
        Result row : scanner)
            nbRows++;
        assertEquals(NB_BATCH_ROWS * 10, nbRows);
    }

    @Test
    public void testAddKeyValue() throws IOException {
        final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
        final byte[] value = Bytes.toBytes("abcd");
        final byte[] row1 = Bytes.toBytes("row1");
        final byte[] row2 = Bytes.toBytes("row2");
        byte[] qualifier = Bytes.toBytes("qf1");
        Put put = new Put(row1);

        // Adding KeyValue with the same row
        KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
        boolean ok = true;
        try {
            put.add(kv);
        } catch (IOException e) {
            ok = false;
        }
        assertEquals(true, ok);

        // Adding KeyValue with the different row
        kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
        ok = false;
        try {
            put.add(kv);
        } catch (IOException e) {
            ok = true;
        }
        assertEquals(true, ok);
    }

    /**
     * test for HBASE-737
     * @throws IOException
     */
    @Test
    public void testHBase737() throws IOException {
        final byte[] FAM1 = Bytes.toBytes("fam1");
        final byte[] FAM2 = Bytes.toBytes("fam2");
        // Open table
        HTable table = TEST_UTIL.createTable(Bytes.toBytes("testHBase737"), new byte[][] { FAM1, FAM2 });
        // Insert some values
        Put put = new Put(ROW);
        put.add(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
        table.put(put);
        try {
            Thread.sleep(1000);
        } catch (InterruptedException i) {
            //ignore
        }

        put = new Put(ROW);
        put.add(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
        table.put(put);

        try {
            Thread.sleep(1000);
        } catch (InterruptedException i) {
            //ignore
        }

        put = new Put(ROW);
        put.add(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
        table.put(put);

        long times[] = new long[3];

        // First scan the memstore

        Scan scan = new Scan();
        scan.addFamily(FAM1);
        scan.addFamily(FAM2);
        ResultScanner s = table.getScanner(scan);
        try {
            int index = 0;
            Result r = null;
            while ((r = s.next()) != null) {
                for (Cell key : r.rawCells()) {
                    times[index++] = key.getTimestamp();
                }
            }
        } finally {
            s.close();
        }
        for (int i = 0; i < times.length - 1; i++) {
            for (int j = i + 1; j < times.length; j++) {
                assertTrue(times[j] > times[i]);
            }
        }

        // Flush data to disk and try again
        TEST_UTIL.flush();

        // Reset times
        for (int i = 0; i < times.length; i++) {
            times[i] = 0;
        }

        try {
            Thread.sleep(1000);
        } catch (InterruptedException i) {
            //ignore
        }
        scan = new Scan();
        scan.addFamily(FAM1);
        scan.addFamily(FAM2);
        s = table.getScanner(scan);
        try {
            int index = 0;
            Result r = null;
            while ((r = s.next()) != null) {
                for (Cell key : r.rawCells()) {
                    times[index++] = key.getTimestamp();
                }
            }
        } finally {
            s.close();
        }
        for (int i = 0; i < times.length - 1; i++) {
            for (int j = i + 1; j < times.length; j++) {
                assertTrue(times[j] > times[i]);
            }
        }
    }

    @Test
    public void testListTables() throws IOException, InterruptedException {
        byte[] t1 = Bytes.toBytes("testListTables1");
        byte[] t2 = Bytes.toBytes("testListTables2");
        byte[] t3 = Bytes.toBytes("testListTables3");
        byte[][] tables = new byte[][] { t1, t2, t3 };
        for (int i = 0; i < tables.length; i++) {
            TEST_UTIL.createTable(tables[i], FAMILY);
        }
        HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
        HTableDescriptor[] ts = admin.listTables();
        HashSet<HTableDescriptor> result = new HashSet<HTableDescriptor>(ts.length);
        for (int i = 0; i < ts.length; i++) {
            result.add(ts[i]);
        }
        int size = result.size();
        assertTrue(size >= tables.length);
        for (int i = 0; i < tables.length && i < size; i++) {
            boolean found = false;
            for (int j = 0; j < ts.length; j++) {
                if (Bytes.equals(ts[j].getTableName().getName(), tables[i])) {
                    found = true;
                    break;
                }
            }
            assertTrue("Not found: " + Bytes.toString(tables[i]), found);
        }
    }

    /**
     * creates an HTable for tableName using an unmanaged HConnection.
     *
     * @param tableName - table to create
     * @return the created HTable object
     * @throws IOException
     */
    HTable createUnmangedHConnectionHTable(final byte[] tableName) throws IOException {
        TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
        HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
        return (HTable) conn.getTable(tableName);
    }

    /**
     * simple test that just executes parts of the client
     * API that accept a pre-created HConnection instance
     *
     * @throws IOException
     */
    @Test
    public void testUnmanagedHConnection() throws IOException {
        final byte[] tableName = Bytes.toBytes("testUnmanagedHConnection");
        HTable t = createUnmangedHConnectionHTable(tableName);
        HBaseAdmin ha = new HBaseAdmin(t.getConnection());
        assertTrue(ha.tableExists(tableName));
        assertTrue(t.get(new Get(ROW)).isEmpty());
    }

    /**
     * test of that unmanaged HConnections are able to reconnect
     * properly (see HBASE-5058)
     *
     * @throws Exception
     */
    @Test
    public void testUnmanagedHConnectionReconnect() throws Exception {
        final byte[] tableName = Bytes.toBytes("testUnmanagedHConnectionReconnect");
        HTable t = createUnmangedHConnectionHTable(tableName);
        HConnection conn = t.getConnection();
        HBaseAdmin ha = new HBaseAdmin(conn);
        assertTrue(ha.tableExists(tableName));
        assertTrue(t.get(new Get(ROW)).isEmpty());

        // stop the master
        MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
        cluster.stopMaster(0, false);
        cluster.waitOnMaster(0);

        // start up a new master
        cluster.startMaster();
        assertTrue(cluster.waitForActiveAndReadyMaster());

        // test that the same unmanaged connection works with a new
        // HBaseAdmin and can connect to the new master;
        HBaseAdmin newAdmin = new HBaseAdmin(conn);
        assertTrue(newAdmin.tableExists(tableName));
        assertTrue(newAdmin.getClusterStatus().getServersSize() == SLAVES + 1);
    }

    @Test
    public void testMiscHTableStuff() throws IOException {
        final byte[] tableAname = Bytes.toBytes("testMiscHTableStuffA");
        final byte[] tableBname = Bytes.toBytes("testMiscHTableStuffB");
        final byte[] attrName = Bytes.toBytes("TESTATTR");
        final byte[] attrValue = Bytes.toBytes("somevalue");
        byte[] value = Bytes.toBytes("value");

        HTable a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
        HTable b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY);
        Put put = new Put(ROW);
        put.add(HConstants.CATALOG_FAMILY, null, value);
        a.put(put);

        // open a new connection to A and a connection to b
        HTable newA = new HTable(TEST_UTIL.getConfiguration(), tableAname);

        // copy data from A to B
        Scan scan = new Scan();
        scan.addFamily(HConstants.CATALOG_FAMILY);
        ResultScanner s = newA.getScanner(scan);
        try {
            for (Result r : s) {
                put = new Put(r.getRow());
                put.setDurability(Durability.SKIP_WAL);
                for (Cell kv : r.rawCells()) {
                    put.add(kv);
                }
                b.put(put);
            }
        } finally {
            s.close();
        }

        // Opening a new connection to A will cause the tables to be reloaded
        HTable anotherA = new HTable(TEST_UTIL.getConfiguration(), tableAname);
        Get get = new Get(ROW);
        get.addFamily(HConstants.CATALOG_FAMILY);
        anotherA.get(get);

        // We can still access A through newA because it has the table information
        // cached. And if it needs to recalibrate, that will cause the information
        // to be reloaded.

        // Test user metadata
        HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
        // make a modifiable descriptor
        HTableDescriptor desc = new HTableDescriptor(a.getTableDescriptor());
        // offline the table
        admin.disableTable(tableAname);
        // add a user attribute to HTD
        desc.setValue(attrName, attrValue);
        // add a user attribute to HCD
        for (HColumnDescriptor c : desc.getFamilies())
            c.setValue(attrName, attrValue);
        // update metadata for all regions of this table
        admin.modifyTable(tableAname, desc);
        // enable the table
        admin.enableTable(tableAname);

        // Test that attribute changes were applied
        desc = a.getTableDescriptor();
        assertTrue("wrong table descriptor returned",
                Bytes.compareTo(desc.getTableName().getName(), tableAname) == 0);
        // check HTD attribute
        value = desc.getValue(attrName);
        assertFalse("missing HTD attribute value", value == null);
        assertFalse("HTD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
        // check HCD attribute
        for (HColumnDescriptor c : desc.getFamilies()) {
            value = c.getValue(attrName);
            assertFalse("missing HCD attribute value", value == null);
            assertFalse("HCD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
        }
    }

    @Test
    public void testGetClosestRowBefore() throws IOException, InterruptedException {
        final byte[] tableAname = Bytes.toBytes("testGetClosestRowBefore");
        final byte[] firstRow = Bytes.toBytes("row111");
        final byte[] secondRow = Bytes.toBytes("row222");
        final byte[] thirdRow = Bytes.toBytes("row333");
        final byte[] forthRow = Bytes.toBytes("row444");
        final byte[] beforeFirstRow = Bytes.toBytes("row");
        final byte[] beforeSecondRow = Bytes.toBytes("row22");
        final byte[] beforeThirdRow = Bytes.toBytes("row33");
        final byte[] beforeForthRow = Bytes.toBytes("row44");

        HTable table = TEST_UTIL.createTable(tableAname,
                new byte[][] { HConstants.CATALOG_FAMILY, Bytes.toBytes("info2") }, 1, 1024);
        // set block size to 64 to making 2 kvs into one block, bypassing the walkForwardInSingleRow
        // in Store.rowAtOrBeforeFromStoreFile
        table.setAutoFlush(true);
        String regionName = table.getRegionLocations().firstKey().getEncodedName();
        HRegion region = TEST_UTIL.getRSForFirstRegionInTable(tableAname).getFromOnlineRegions(regionName);
        Put put1 = new Put(firstRow);
        Put put2 = new Put(secondRow);
        Put put3 = new Put(thirdRow);
        Put put4 = new Put(forthRow);
        byte[] one = new byte[] { 1 };
        byte[] two = new byte[] { 2 };
        byte[] three = new byte[] { 3 };
        byte[] four = new byte[] { 4 };

        put1.add(HConstants.CATALOG_FAMILY, null, one);
        put2.add(HConstants.CATALOG_FAMILY, null, two);
        put3.add(HConstants.CATALOG_FAMILY, null, three);
        put4.add(HConstants.CATALOG_FAMILY, null, four);
        table.put(put1);
        table.put(put2);
        table.put(put3);
        table.put(put4);
        region.flushcache();
        Result result = null;

        // Test before first that null is returned
        result = table.getRowOrBefore(beforeFirstRow, HConstants.CATALOG_FAMILY);
        assertTrue(result == null);

        // Test at first that first is returned
        result = table.getRowOrBefore(firstRow, HConstants.CATALOG_FAMILY);
        assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
        assertTrue(Bytes.equals(result.getRow(), firstRow));
        assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));

        // Test in between first and second that first is returned
        result = table.getRowOrBefore(beforeSecondRow, HConstants.CATALOG_FAMILY);
        assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
        assertTrue(Bytes.equals(result.getRow(), firstRow));
        assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));

        // Test at second make sure second is returned
        result = table.getRowOrBefore(secondRow, HConstants.CATALOG_FAMILY);
        assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
        assertTrue(Bytes.equals(result.getRow(), secondRow));
        assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));

        // Test in second and third, make sure second is returned
        result = table.getRowOrBefore(beforeThirdRow, HConstants.CATALOG_FAMILY);
        assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
        assertTrue(Bytes.equals(result.getRow(), secondRow));
        assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));

        // Test at third make sure third is returned
        result = table.getRowOrBefore(thirdRow, HConstants.CATALOG_FAMILY);
        assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
        assertTrue(Bytes.equals(result.getRow(), thirdRow));
        assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));

        // Test in third and forth, make sure third is returned
        result = table.getRowOrBefore(beforeForthRow, HConstants.CATALOG_FAMILY);
        assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
        assertTrue(Bytes.equals(result.getRow(), thirdRow));
        assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));

        // Test at forth make sure forth is returned
        result = table.getRowOrBefore(forthRow, HConstants.CATALOG_FAMILY);
        assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
        assertTrue(Bytes.equals(result.getRow(), forthRow));
        assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));

        // Test after forth make sure forth is returned
        result = table.getRowOrBefore(Bytes.add(forthRow, one), HConstants.CATALOG_FAMILY);
        assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
        assertTrue(Bytes.equals(result.getRow(), forthRow));
        assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
    }

    /**
     * For HBASE-2156
     * @throws Exception
     */
    @Test
    public void testScanVariableReuse() throws Exception {
        Scan scan = new Scan();
        scan.addFamily(FAMILY);
        scan.addColumn(FAMILY, ROW);

        assertTrue(scan.getFamilyMap().get(FAMILY).size() == 1);

        scan = new Scan();
        scan.addFamily(FAMILY);

        assertTrue(scan.getFamilyMap().get(FAMILY) == null);
        assertTrue(scan.getFamilyMap().containsKey(FAMILY));
    }

    @Test
    public void testMultiRowMutation() throws Exception {
        LOG.info("Starting testMultiRowMutation");
        final byte[] TABLENAME = Bytes.toBytes("testMultiRowMutation");
        final byte[] ROW1 = Bytes.toBytes("testRow1");

        HTable t = TEST_UTIL.createTable(TABLENAME, FAMILY);
        Put p = new Put(ROW);
        p.add(FAMILY, QUALIFIER, VALUE);
        MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, p);

        p = new Put(ROW1);
        p.add(FAMILY, QUALIFIER, VALUE);
        MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, p);

        MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
        mrmBuilder.addMutationRequest(m1);
        mrmBuilder.addMutationRequest(m2);
        MutateRowsRequest mrm = mrmBuilder.build();
        CoprocessorRpcChannel channel = t.coprocessorService(ROW);
        MultiRowMutationService.BlockingInterface service = MultiRowMutationService.newBlockingStub(channel);
        service.mutateRows(null, mrm);
        Get g = new Get(ROW);
        Result r = t.get(g);
        assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
        g = new Get(ROW1);
        r = t.get(g);
        assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
    }

    @Test
    public void testRowMutation() throws Exception {
        LOG.info("Starting testRowMutation");
        final byte[] TABLENAME = Bytes.toBytes("testRowMutation");
        HTable t = TEST_UTIL.createTable(TABLENAME, FAMILY);
        byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b") };
        RowMutations arm = new RowMutations(ROW);
        Put p = new Put(ROW);
        p.add(FAMILY, QUALIFIERS[0], VALUE);
        arm.add(p);
        t.mutateRow(arm);

        Get g = new Get(ROW);
        Result r = t.get(g);
        assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0])));

        arm = new RowMutations(ROW);
        p = new Put(ROW);
        p.add(FAMILY, QUALIFIERS[1], VALUE);
        arm.add(p);
        Delete d = new Delete(ROW);
        d.deleteColumns(FAMILY, QUALIFIERS[0]);
        arm.add(d);
        // TODO: Trying mutateRow again.  The batch was failing with a one try only.
        t.mutateRow(arm);
        r = t.get(g);
        assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1])));
        assertNull(r.getValue(FAMILY, QUALIFIERS[0]));
    }

    @Test
    public void testAppend() throws Exception {
        LOG.info("Starting testAppend");
        final byte[] TABLENAME = Bytes.toBytes("testAppend");
        HTable t = TEST_UTIL.createTable(TABLENAME, FAMILY);
        byte[] v1 = Bytes.toBytes("42");
        byte[] v2 = Bytes.toBytes("23");
        byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("c") };
        Append a = new Append(ROW);
        a.add(FAMILY, QUALIFIERS[0], v1);
        a.add(FAMILY, QUALIFIERS[1], v2);
        a.setReturnResults(false);
        assertNullResult(t.append(a));

        a = new Append(ROW);
        a.add(FAMILY, QUALIFIERS[0], v2);
        a.add(FAMILY, QUALIFIERS[1], v1);
        a.add(FAMILY, QUALIFIERS[2], v2);
        Result r = t.append(a);
        assertEquals(0, Bytes.compareTo(Bytes.add(v1, v2), r.getValue(FAMILY, QUALIFIERS[0])));
        assertEquals(0, Bytes.compareTo(Bytes.add(v2, v1), r.getValue(FAMILY, QUALIFIERS[1])));
        // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct
        assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2])));
        assertEquals(r.getColumnLatest(FAMILY, QUALIFIERS[0]).getTimestamp(),
                r.getColumnLatest(FAMILY, QUALIFIERS[2]).getTimestamp());
    }

    @Test
    public void testIncrementWithDeletes() throws Exception {
        LOG.info("Starting testIncrementWithDeletes");
        final TableName TABLENAME = TableName.valueOf("testIncrementWithDeletes");
        HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
        final byte[] COLUMN = Bytes.toBytes("column");

        ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
        TEST_UTIL.flush(TABLENAME);

        Delete del = new Delete(ROW);
        ht.delete(del);

        ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);

        Get get = new Get(ROW);
        Result r = ht.get(get);
        assertEquals(1, r.size());
        assertEquals(5, Bytes.toLong(r.getValue(FAMILY, COLUMN)));
    }

    @Test
    public void testIncrementingInvalidValue() throws Exception {
        LOG.info("Starting testIncrementingInvalidValue");
        final byte[] TABLENAME = Bytes.toBytes("testIncrementingInvalidValue");
        HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
        final byte[] COLUMN = Bytes.toBytes("column");
        Put p = new Put(ROW);
        // write an integer here (not a Long)
        p.add(FAMILY, COLUMN, Bytes.toBytes(5));
        ht.put(p);
        try {
            ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
            fail("Should have thrown DoNotRetryIOException");
        } catch (DoNotRetryIOException iox) {
            // success
        }
        Increment inc = new Increment(ROW);
        inc.addColumn(FAMILY, COLUMN, 5);
        try {
            ht.increment(inc);
            fail("Should have thrown DoNotRetryIOException");
        } catch (DoNotRetryIOException iox) {
            // success
        }
    }

    @Test
    public void testIncrementInvalidArguments() throws Exception {
        LOG.info("Starting testIncrementInvalidArguments");
        final byte[] TABLENAME = Bytes.toBytes("testIncrementInvalidArguments");
        HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
        final byte[] COLUMN = Bytes.toBytes("column");
        try {
            // try null row
            ht.incrementColumnValue(null, FAMILY, COLUMN, 5);
            fail("Should have thrown IOException");
        } catch (IOException iox) {
            // success
        }
        try {
            // try null family
            ht.incrementColumnValue(ROW, null, COLUMN, 5);
            fail("Should have thrown IOException");
        } catch (IOException iox) {
            // success
        }
        try {
            // try null qualifier
            ht.incrementColumnValue(ROW, FAMILY, null, 5);
            fail("Should have thrown IOException");
        } catch (IOException iox) {
            // success
        }
        // try null row
        try {
            Increment incNoRow = new Increment(null);
            incNoRow.addColumn(FAMILY, COLUMN, 5);
            fail("Should have thrown IllegalArgumentException");
        } catch (IllegalArgumentException iax) {
            // success
        } catch (NullPointerException npe) {
            // success
        }
        // try null family
        try {
            Increment incNoFamily = new Increment(ROW);
            incNoFamily.addColumn(null, COLUMN, 5);
            fail("Should have thrown IllegalArgumentException");
        } catch (IllegalArgumentException iax) {
            // success
        }
        // try null qualifier
        try {
            Increment incNoQualifier = new Increment(ROW);
            incNoQualifier.addColumn(FAMILY, null, 5);
            fail("Should have thrown IllegalArgumentException");
        } catch (IllegalArgumentException iax) {
            // success
        }
    }

    @Test
    public void testIncrementOutOfOrder() throws Exception {
        LOG.info("Starting testIncrementOutOfOrder");
        final byte[] TABLENAME = Bytes.toBytes("testIncrementOutOfOrder");
        HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);

        byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("B"), Bytes.toBytes("A"), Bytes.toBytes("C") };

        Increment inc = new Increment(ROW);
        for (int i = 0; i < QUALIFIERS.length; i++) {
            inc.addColumn(FAMILY, QUALIFIERS[i], 1);
        }
        ht.increment(inc);

        // Verify expected results
        Result r = ht.get(new Get(ROW));
        Cell[] kvs = r.rawCells();
        assertEquals(3, kvs.length);
        assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 1);
        assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 1);
        assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1);

        // Now try multiple columns again
        inc = new Increment(ROW);
        for (int i = 0; i < QUALIFIERS.length; i++) {
            inc.addColumn(FAMILY, QUALIFIERS[i], 1);
        }
        ht.increment(inc);

        // Verify
        r = ht.get(new Get(ROW));
        kvs = r.rawCells();
        assertEquals(3, kvs.length);
        assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 2);
        assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 2);
        assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2);
    }

    @Test
    public void testIncrement() throws Exception {
        LOG.info("Starting testIncrement");
        final byte[] TABLENAME = Bytes.toBytes("testIncrement");
        HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);

        byte[][] ROWS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
                Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"), Bytes.toBytes("g"), Bytes.toBytes("h"),
                Bytes.toBytes("i") };
        byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
                Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"), Bytes.toBytes("g"), Bytes.toBytes("h"),
                Bytes.toBytes("i") };

        // Do some simple single-column increments

        // First with old API
        ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[0], 1);
        ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[1], 2);
        ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[2], 3);
        ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[3], 4);

        // Now increment things incremented with old and do some new
        Increment inc = new Increment(ROW);
        inc.addColumn(FAMILY, QUALIFIERS[1], 1);
        inc.addColumn(FAMILY, QUALIFIERS[3], 1);
        inc.addColumn(FAMILY, QUALIFIERS[4], 1);
        ht.increment(inc);

        // Verify expected results
        Result r = ht.get(new Get(ROW));
        Cell[] kvs = r.rawCells();
        assertEquals(5, kvs.length);
        assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1);
        assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 3);
        assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 3);
        assertIncrementKey(kvs[3], ROW, FAMILY, QUALIFIERS[3], 5);
        assertIncrementKey(kvs[4], ROW, FAMILY, QUALIFIERS[4], 1);

        // Now try multiple columns by different amounts
        inc = new Increment(ROWS[0]);
        for (int i = 0; i < QUALIFIERS.length; i++) {
            inc.addColumn(FAMILY, QUALIFIERS[i], i + 1);
        }
        ht.increment(inc);
        // Verify
        r = ht.get(new Get(ROWS[0]));
        kvs = r.rawCells();
        assertEquals(QUALIFIERS.length, kvs.length);
        for (int i = 0; i < QUALIFIERS.length; i++) {
            assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], i + 1);
        }

        // Re-increment them
        inc = new Increment(ROWS[0]);
        for (int i = 0; i < QUALIFIERS.length; i++) {
            inc.addColumn(FAMILY, QUALIFIERS[i], i + 1);
        }
        ht.increment(inc);
        // Verify
        r = ht.get(new Get(ROWS[0]));
        kvs = r.rawCells();
        assertEquals(QUALIFIERS.length, kvs.length);
        for (int i = 0; i < QUALIFIERS.length; i++) {
            assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2 * (i + 1));
        }
    }

    @Test
    public void testClientPoolRoundRobin() throws IOException {
        final byte[] tableName = Bytes.toBytes("testClientPoolRoundRobin");

        int poolSize = 3;
        int numVersions = poolSize * 2;
        Configuration conf = TEST_UTIL.getConfiguration();
        conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "round-robin");
        conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);

        HTable table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, conf, Integer.MAX_VALUE);

        final long ts = EnvironmentEdgeManager.currentTimeMillis();
        Get get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIER);
        get.setMaxVersions();

        for (int versions = 1; versions <= numVersions; versions++) {
            Put put = new Put(ROW);
            put.add(FAMILY, QUALIFIER, ts + versions, VALUE);
            table.put(put);

            Result result = table.get(get);
            NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(QUALIFIER);

            assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER + " did not match " + versions,
                    versions, navigableMap.size());
            for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
                assertTrue("The value at time " + entry.getKey() + " did not match what was put",
                        Bytes.equals(VALUE, entry.getValue()));
            }
        }
    }

    @Ignore("Flakey: HBASE-8989")
    @Test
    public void testClientPoolThreadLocal() throws IOException {
        final byte[] tableName = Bytes.toBytes("testClientPoolThreadLocal");

        int poolSize = Integer.MAX_VALUE;
        int numVersions = 3;
        Configuration conf = TEST_UTIL.getConfiguration();
        conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "thread-local");
        conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);

        final HTable table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, conf, 3);

        final long ts = EnvironmentEdgeManager.currentTimeMillis();
        final Get get = new Get(ROW);
        get.addColumn(FAMILY, QUALIFIER);
        get.setMaxVersions();

        for (int versions = 1; versions <= numVersions; versions++) {
            Put put = new Put(ROW);
            put.add(FAMILY, QUALIFIER, ts + versions, VALUE);
            table.put(put);

            Result result = table.get(get);
            NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(QUALIFIER);

            assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER + " did not match " + versions
                    + "; " + put.toString() + ", " + get.toString(), versions, navigableMap.size());
            for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
                assertTrue("The value at time " + entry.getKey() + " did not match what was put",
                        Bytes.equals(VALUE, entry.getValue()));
            }
        }

        final Object waitLock = new Object();
        ExecutorService executorService = Executors.newFixedThreadPool(numVersions);
        final AtomicReference<AssertionError> error = new AtomicReference<AssertionError>(null);
        for (int versions = numVersions; versions < numVersions * 2; versions++) {
            final int versionsCopy = versions;
            executorService.submit(new Callable<Void>() {
                @Override
                public Void call() {
                    try {
                        Put put = new Put(ROW);
                        put.add(FAMILY, QUALIFIER, ts + versionsCopy, VALUE);
                        table.put(put);

                        Result result = table.get(get);
                        NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(QUALIFIER);

                        assertEquals(
                                "The number of versions of '" + Bytes.toString(FAMILY) + ":"
                                        + Bytes.toString(QUALIFIER) + " did not match " + versionsCopy,
                                versionsCopy, navigableMap.size());
                        for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
                            assertTrue("The value at time " + entry.getKey() + " did not match what was put",
                                    Bytes.equals(VALUE, entry.getValue()));
                        }
                        synchronized (waitLock) {
                            waitLock.wait();
                        }
                    } catch (Exception e) {
                    } catch (AssertionError e) {
                        // the error happens in a thread, it won't fail the test,
                        // need to pass it to the caller for proper handling.
                        error.set(e);
                        LOG.error(e);
                    }

                    return null;
                }
            });
        }
        synchronized (waitLock) {
            waitLock.notifyAll();
        }
        executorService.shutdownNow();
        assertNull(error.get());
    }

    @Test
    public void testCheckAndPut() throws IOException {
        final byte[] anotherrow = Bytes.toBytes("anotherrow");
        final byte[] value2 = Bytes.toBytes("abcd");

        HTable table = TEST_UTIL.createTable(Bytes.toBytes("testCheckAndPut"), new byte[][] { FAMILY });
        Put put1 = new Put(ROW);
        put1.add(FAMILY, QUALIFIER, VALUE);

        // row doesn't exist, so using non-null value should be considered "not match".
        boolean ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put1);
        assertEquals(ok, false);

        // row doesn't exist, so using "null" to check for existence should be considered "match".
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1);
        assertEquals(ok, true);

        // row now exists, so using "null" to check for existence should be considered "not match".
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1);
        assertEquals(ok, false);

        Put put2 = new Put(ROW);
        put2.add(FAMILY, QUALIFIER, value2);

        // row now exists, use the matching value to check
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put2);
        assertEquals(ok, true);

        Put put3 = new Put(anotherrow);
        put3.add(FAMILY, QUALIFIER, VALUE);

        // try to do CheckAndPut on different rows
        try {
            ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, value2, put3);
            fail("trying to check and modify different rows should have failed.");
        } catch (Exception e) {
        }

    }

    @Test
    public void testCheckAndPutWithCompareOp() throws IOException {
        final byte[] value1 = Bytes.toBytes("aaaa");
        final byte[] value2 = Bytes.toBytes("bbbb");
        final byte[] value3 = Bytes.toBytes("cccc");
        final byte[] value4 = Bytes.toBytes("dddd");

        HTable table = TEST_UTIL.createTable(Bytes.toBytes("testCheckAndPutWithCompareOp"),
                new byte[][] { FAMILY });

        Put put2 = new Put(ROW);
        put2.add(FAMILY, QUALIFIER, value2);

        Put put3 = new Put(ROW);
        put3.add(FAMILY, QUALIFIER, value3);

        // row doesn't exist, so using "null" to check for existence should be considered "match".
        boolean ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put2);
        assertEquals(ok, true);

        // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
        // turns out "match"
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value1, put2);
        assertEquals(ok, false);
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value1, put2);
        assertEquals(ok, false);
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value1, put2);
        assertEquals(ok, false);
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value1, put2);
        assertEquals(ok, true);
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value1, put2);
        assertEquals(ok, true);
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value1, put3);
        assertEquals(ok, true);

        // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
        // turns out "match"
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value4, put3);
        assertEquals(ok, false);
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value4, put3);
        assertEquals(ok, false);
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value4, put3);
        assertEquals(ok, false);
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value4, put3);
        assertEquals(ok, true);
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value4, put3);
        assertEquals(ok, true);
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value4, put2);
        assertEquals(ok, true);

        // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
        // turns out "match"
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value2, put2);
        assertEquals(ok, false);
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value2, put2);
        assertEquals(ok, false);
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value2, put2);
        assertEquals(ok, false);
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value2, put2);
        assertEquals(ok, true);
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value2, put2);
        assertEquals(ok, true);
        ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value2, put3);
        assertEquals(ok, true);
    }

    @Test
    public void testCheckAndDeleteWithCompareOp() throws IOException {
        final byte[] value1 = Bytes.toBytes("aaaa");
        final byte[] value2 = Bytes.toBytes("bbbb");
        final byte[] value3 = Bytes.toBytes("cccc");
        final byte[] value4 = Bytes.toBytes("dddd");

        HTable table = TEST_UTIL.createTable(Bytes.toBytes("testCheckAndDeleteWithCompareOp"),
                new byte[][] { FAMILY });

        Put put2 = new Put(ROW);
        put2.add(FAMILY, QUALIFIER, value2);
        table.put(put2);

        Put put3 = new Put(ROW);
        put3.add(FAMILY, QUALIFIER, value3);

        Delete delete = new Delete(ROW);
        delete.deleteColumns(FAMILY, QUALIFIER);

        // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
        // turns out "match"
        boolean ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value1, delete);
        assertEquals(ok, false);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value1, delete);
        assertEquals(ok, false);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value1, delete);
        assertEquals(ok, false);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value1, delete);
        assertEquals(ok, true);
        table.put(put2);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value1, delete);
        assertEquals(ok, true);
        table.put(put2);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value1, delete);
        assertEquals(ok, true);

        // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
        // turns out "match"
        table.put(put3);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value4, delete);
        assertEquals(ok, false);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value4, delete);
        assertEquals(ok, false);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value4, delete);
        assertEquals(ok, false);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value4, delete);
        assertEquals(ok, true);
        table.put(put3);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value4, delete);
        assertEquals(ok, true);
        table.put(put3);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value4, delete);
        assertEquals(ok, true);

        // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
        // turns out "match"
        table.put(put2);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value2, delete);
        assertEquals(ok, false);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value2, delete);
        assertEquals(ok, false);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value2, delete);
        assertEquals(ok, false);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value2, delete);
        assertEquals(ok, true);
        table.put(put2);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value2, delete);
        assertEquals(ok, true);
        table.put(put2);
        ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value2, delete);
        assertEquals(ok, true);
    }

    /**
    * Test ScanMetrics
    * @throws Exception
    */
    @Test
    @SuppressWarnings("unused")
    public void testScanMetrics() throws Exception {
        byte[] TABLENAME = Bytes.toBytes("testScanMetrics");

        Configuration conf = TEST_UTIL.getConfiguration();
        TEST_UTIL.createTable(TABLENAME, FAMILY);

        // Set up test table:
        // Create table:
        HTable ht = new HTable(conf, TABLENAME);

        // Create multiple regions for this table
        int numOfRegions = TEST_UTIL.createMultiRegions(ht, FAMILY);
        // Create 3 rows in the table, with rowkeys starting with "z*" so that
        // scan are forced to hit all the regions.
        Put put1 = new Put(Bytes.toBytes("z1"));
        put1.add(FAMILY, QUALIFIER, VALUE);
        Put put2 = new Put(Bytes.toBytes("z2"));
        put2.add(FAMILY, QUALIFIER, VALUE);
        Put put3 = new Put(Bytes.toBytes("z3"));
        put3.add(FAMILY, QUALIFIER, VALUE);
        ht.put(Arrays.asList(put1, put2, put3));

        Scan scan1 = new Scan();
        int numRecords = 0;
        for (Result result : ht.getScanner(scan1)) {
            numRecords++;
        }
        LOG.info("test data has " + numRecords + " records.");

        // by default, scan metrics collection is turned off
        assertEquals(null, scan1.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA));

        // turn on scan metrics
        Scan scan = new Scan();
        scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
        scan.setCaching(numRecords + 1);
        ResultScanner scanner = ht.getScanner(scan);
        for (Result result : scanner.next(numRecords - 1)) {
        }
        scanner.close();
        // closing the scanner will set the metrics.
        assertNotNull(scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA));

        // set caching to 1, becasue metrics are collected in each roundtrip only
        scan = new Scan();
        scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
        scan.setCaching(1);
        scanner = ht.getScanner(scan);
        // per HBASE-5717, this should still collect even if you don't run all the way to
        // the end of the scanner. So this is asking for 2 of the 3 rows we inserted.
        for (Result result : scanner.next(numRecords - 1)) {
        }
        scanner.close();

        ScanMetrics scanMetrics = getScanMetrics(scan);
        assertEquals("Did not access all the regions in the table", numOfRegions, scanMetrics.countOfRegions.get());

        // now, test that the metrics are still collected even if you don't call close, but do
        // run past the end of all the records
        Scan scanWithoutClose = new Scan();
        scanWithoutClose.setCaching(1);
        scanWithoutClose.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
        ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose);
        for (Result result : scannerWithoutClose.next(numRecords + 1)) {
        }
        ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose);
        assertEquals("Did not access all the regions in the table", numOfRegions,
                scanMetricsWithoutClose.countOfRegions.get());

        // finally, test that the metrics are collected correctly if you both run past all the records,
        // AND close the scanner
        Scan scanWithClose = new Scan();
        // make sure we can set caching up to the number of a scanned values
        scanWithClose.setCaching(numRecords);
        scanWithClose.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
        ResultScanner scannerWithClose = ht.getScanner(scanWithClose);
        for (Result result : scannerWithClose.next(numRecords + 1)) {
        }
        scannerWithClose.close();
        ScanMetrics scanMetricsWithClose = getScanMetrics(scanWithClose);
        assertEquals("Did not access all the regions in the table", numOfRegions,
                scanMetricsWithClose.countOfRegions.get());
    }

    private ScanMetrics getScanMetrics(Scan scan) throws Exception {
        byte[] serializedMetrics = scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA);
        assertTrue("Serialized metrics were not found.", serializedMetrics != null);

        ScanMetrics scanMetrics = ProtobufUtil.toScanMetrics(serializedMetrics);

        return scanMetrics;
    }

    /**
     * Tests that cache on write works all the way up from the client-side.
     *
     * Performs inserts, flushes, and compactions, verifying changes in the block
     * cache along the way.
     *
     * @throws Exception
     */
    @Test
    public void testCacheOnWriteEvictOnClose() throws Exception {
        byte[] tableName = Bytes.toBytes("testCOWEOCfromClient");
        byte[] data = Bytes.toBytes("data");
        HTable table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY });
        // get the block cache and region
        String regionName = table.getRegionLocations().firstKey().getEncodedName();
        HRegion region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getFromOnlineRegions(regionName);
        Store store = region.getStores().values().iterator().next();
        CacheConfig cacheConf = store.getCacheConfig();
        cacheConf.setCacheDataOnWrite(true);
        cacheConf.setEvictOnClose(true);
        BlockCache cache = cacheConf.getBlockCache();

        // establish baseline stats
        long startBlockCount = cache.getBlockCount();
        long startBlockHits = cache.getStats().getHitCount();
        long startBlockMiss = cache.getStats().getMissCount();

        // wait till baseline is stable, (minimal 500 ms)
        for (int i = 0; i < 5; i++) {
            Thread.sleep(100);
            if (startBlockCount != cache.getBlockCount() || startBlockHits != cache.getStats().getHitCount()
                    || startBlockMiss != cache.getStats().getMissCount()) {
                startBlockCount = cache.getBlockCount();
                startBlockHits = cache.getStats().getHitCount();
                startBlockMiss = cache.getStats().getMissCount();
                i = -1;
            }
        }

        // insert data
        Put put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, data);
        table.put(put);
        assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
        // data was in memstore so don't expect any changes
        assertEquals(startBlockCount, cache.getBlockCount());
        assertEquals(startBlockHits, cache.getStats().getHitCount());
        assertEquals(startBlockMiss, cache.getStats().getMissCount());
        // flush the data
        System.out.println("Flushing cache");
        region.flushcache();
        // expect one more block in cache, no change in hits/misses
        long expectedBlockCount = startBlockCount + 1;
        long expectedBlockHits = startBlockHits;
        long expectedBlockMiss = startBlockMiss;
        assertEquals(expectedBlockCount, cache.getBlockCount());
        assertEquals(expectedBlockHits, cache.getStats().getHitCount());
        assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
        // read the data and expect same blocks, one new hit, no misses
        assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
        assertEquals(expectedBlockCount, cache.getBlockCount());
        assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
        assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
        // insert a second column, read the row, no new blocks, one new hit
        byte[] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER);
        byte[] data2 = Bytes.add(data, data);
        put = new Put(ROW);
        put.add(FAMILY, QUALIFIER2, data2);
        table.put(put);
        Result r = table.get(new Get(ROW));
        assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
        assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
        assertEquals(expectedBlockCount, cache.getBlockCount());
        assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
        assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
        // flush, one new block
        System.out.println("Flushing cache");
        region.flushcache();
        assertEquals(++expectedBlockCount, cache.getBlockCount());
        assertEquals(expectedBlockHits, cache.getStats().getHitCount());
        assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
        // compact, net minus two blocks, two hits, no misses
        System.out.println("Compacting");
        assertEquals(2, store.getStorefilesCount());
        store.triggerMajorCompaction();
        region.compactStores();
        waitForStoreFileCount(store, 1, 10000); // wait 10 seconds max
        assertEquals(1, store.getStorefilesCount());
        expectedBlockCount -= 2; // evicted two blocks, cached none
        assertEquals(expectedBlockCount, cache.getBlockCount());
        expectedBlockHits += 2;
        assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
        assertEquals(expectedBlockHits, cache.getStats().getHitCount());
        // read the row, this should be a cache miss because we don't cache data
        // blocks on compaction
        r = table.get(new Get(ROW));
        assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
        assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
        expectedBlockCount += 1; // cached one data block
        assertEquals(expectedBlockCount, cache.getBlockCount());
        assertEquals(expectedBlockHits, cache.getStats().getHitCount());
        assertEquals(++expectedBlockMiss, cache.getStats().getMissCount());
    }

    private void waitForStoreFileCount(Store store, int count, int timeout) throws InterruptedException {
        long start = System.currentTimeMillis();
        while (start + timeout > System.currentTimeMillis() && store.getStorefilesCount() != count) {
            Thread.sleep(100);
        }
        System.out.println(
                "start=" + start + ", now=" + System.currentTimeMillis() + ", cur=" + store.getStorefilesCount());
        assertEquals(count, store.getStorefilesCount());
    }

    @Test
    /**
     * Tests the non cached version of getRegionLocation by moving a region.
     */
    public void testNonCachedGetRegionLocation() throws Exception {
        // Test Initialization.
        String tableName = "testNonCachedGetRegionLocation";
        byte[] TABLE = Bytes.toBytes(tableName);
        byte[] family1 = Bytes.toBytes("f1");
        byte[] family2 = Bytes.toBytes("f2");
        HTable table = TEST_UTIL.createTable(TABLE, new byte[][] { family1, family2 }, 10);
        HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
        Map<HRegionInfo, ServerName> regionsMap = table.getRegionLocations();
        assertEquals(1, regionsMap.size());
        HRegionInfo regionInfo = regionsMap.keySet().iterator().next();
        ServerName addrBefore = regionsMap.get(regionInfo);
        // Verify region location before move.
        HRegionLocation addrCache = table.getRegionLocation(regionInfo.getStartKey(), false);
        HRegionLocation addrNoCache = table.getRegionLocation(regionInfo.getStartKey(), true);

        assertEquals(addrBefore.getPort(), addrCache.getPort());
        assertEquals(addrBefore.getPort(), addrNoCache.getPort());

        ServerName addrAfter = null;
        // Now move the region to a different server.
        for (int i = 0; i < SLAVES; i++) {
            HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
            ServerName addr = regionServer.getServerName();
            if (addr.getPort() != addrBefore.getPort()) {
                admin.move(regionInfo.getEncodedNameAsBytes(), Bytes.toBytes(addr.toString()));
                // Wait for the region to move.
                Thread.sleep(5000);
                addrAfter = addr;
                break;
            }
        }

        // Verify the region was moved.
        addrCache = table.getRegionLocation(regionInfo.getStartKey(), false);
        addrNoCache = table.getRegionLocation(regionInfo.getStartKey(), true);
        assertNotNull(addrAfter);
        assertTrue(addrAfter.getPort() != addrCache.getPort());
        assertEquals(addrAfter.getPort(), addrNoCache.getPort());
    }

    @Test
    /**
     * Tests getRegionsInRange by creating some regions over which a range of
     * keys spans; then changing the key range.
     */
    public void testGetRegionsInRange() throws Exception {
        // Test Initialization.
        byte[] startKey = Bytes.toBytes("ddc");
        byte[] endKey = Bytes.toBytes("mmm");
        byte[] TABLE = Bytes.toBytes("testGetRegionsInRange");
        HTable table = TEST_UTIL.createTable(TABLE, new byte[][] { FAMILY }, 10);
        int numOfRegions = TEST_UTIL.createMultiRegions(table, FAMILY);
        assertEquals(25, numOfRegions);

        // Get the regions in this range
        List<HRegionLocation> regionsList = table.getRegionsInRange(startKey, endKey);
        assertEquals(10, regionsList.size());

        // Change the start key
        startKey = Bytes.toBytes("fff");
        regionsList = table.getRegionsInRange(startKey, endKey);
        assertEquals(7, regionsList.size());

        // Change the end key
        endKey = Bytes.toBytes("nnn");
        regionsList = table.getRegionsInRange(startKey, endKey);
        assertEquals(8, regionsList.size());

        // Empty start key
        regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW, endKey);
        assertEquals(13, regionsList.size());

        // Empty end key
        regionsList = table.getRegionsInRange(startKey, HConstants.EMPTY_END_ROW);
        assertEquals(20, regionsList.size());

        // Both start and end keys empty
        regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
        assertEquals(25, regionsList.size());

        // Change the end key to somewhere in the last block
        endKey = Bytes.toBytes("yyz");
        regionsList = table.getRegionsInRange(startKey, endKey);
        assertEquals(20, regionsList.size());

        // Change the start key to somewhere in the first block
        startKey = Bytes.toBytes("aac");
        regionsList = table.getRegionsInRange(startKey, endKey);
        assertEquals(25, regionsList.size());

        // Make start and end key the same
        startKey = endKey = Bytes.toBytes("ccc");
        regionsList = table.getRegionsInRange(startKey, endKey);
        assertEquals(1, regionsList.size());
    }

    @Test
    public void testJira6912() throws Exception {
        byte[] TABLE = Bytes.toBytes("testJira6912");
        HTable foo = TEST_UTIL.createTable(TABLE, new byte[][] { FAMILY }, 10);

        List<Put> puts = new ArrayList<Put>();
        for (int i = 0; i != 100; i++) {
            Put put = new Put(Bytes.toBytes(i));
            put.add(FAMILY, FAMILY, Bytes.toBytes(i));
            puts.add(put);
        }
        foo.put(puts);
        // If i comment this out it works
        TEST_UTIL.flush();

        Scan scan = new Scan();
        scan.setStartRow(Bytes.toBytes(1));
        scan.setStopRow(Bytes.toBytes(3));
        scan.addColumn(FAMILY, FAMILY);
        scan.setFilter(new RowFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator(Bytes.toBytes(1))));

        ResultScanner scanner = foo.getScanner(scan);
        Result[] bar = scanner.next(100);
        assertEquals(1, bar.length);
    }

    @Test
    public void testScan_NullQualifier() throws IOException {
        HTable table = TEST_UTIL.createTable(Bytes.toBytes("testScan_NullQualifier"), FAMILY);
        Put put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, VALUE);
        table.put(put);

        put = new Put(ROW);
        put.add(FAMILY, null, VALUE);
        table.put(put);
        LOG.info("Row put");

        Scan scan = new Scan();
        scan.addColumn(FAMILY, null);

        ResultScanner scanner = table.getScanner(scan);
        Result[] bar = scanner.next(100);
        assertEquals(1, bar.length);
        assertEquals(1, bar[0].size());

        scan = new Scan();
        scan.addFamily(FAMILY);

        scanner = table.getScanner(scan);
        bar = scanner.next(100);
        assertEquals(1, bar.length);
        assertEquals(2, bar[0].size());
    }

    @Test
    public void testNegativeTimestamp() throws IOException {
        HTable table = TEST_UTIL.createTable(Bytes.toBytes("testNegativeTimestamp"), FAMILY);

        try {
            Put put = new Put(ROW, -1);
            put.add(FAMILY, QUALIFIER, VALUE);
            table.put(put);
            fail("Negative timestamps should not have been allowed");
        } catch (IllegalArgumentException ex) {
            assertTrue(ex.getMessage().contains("negative"));
        }

        try {
            Put put = new Put(ROW);
            put.add(FAMILY, QUALIFIER, -1, VALUE);
            table.put(put);
            fail("Negative timestamps should not have been allowed");
        } catch (IllegalArgumentException ex) {
            assertTrue(ex.getMessage().contains("negative"));
        }

        try {
            Delete delete = new Delete(ROW, -1);
            table.delete(delete);
            fail("Negative timestamps should not have been allowed");
        } catch (IllegalArgumentException ex) {
            assertTrue(ex.getMessage().contains("negative"));
        }

        try {
            Delete delete = new Delete(ROW);
            delete.deleteFamily(FAMILY, -1);
            table.delete(delete);
            fail("Negative timestamps should not have been allowed");
        } catch (IllegalArgumentException ex) {
            assertTrue(ex.getMessage().contains("negative"));
        }

        try {
            Scan scan = new Scan();
            scan.setTimeRange(-1, 1);
            table.getScanner(scan);
            fail("Negative timestamps should not have been allowed");
        } catch (IllegalArgumentException ex) {
            assertTrue(ex.getMessage().contains("negative"));
        }

        // KeyValue should allow negative timestamps for backwards compat. Otherwise, if the user
        // already has negative timestamps in cluster data, HBase won't be able to handle that
        try {
            new KeyValue(Bytes.toBytes(42), Bytes.toBytes(42), Bytes.toBytes(42), -1, Bytes.toBytes(42));
        } catch (IllegalArgumentException ex) {
            fail("KeyValue SHOULD allow negative timestamps");
        }

        table.close();
    }

    @Test
    public void testIllegalTableDescriptor() throws Exception {
        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testIllegalTableDescriptor"));
        HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);

        // create table with 0 families
        checkTableIsIllegal(htd);
        htd.addFamily(hcd);
        checkTableIsLegal(htd);

        htd.setMaxFileSize(1024); // 1K
        checkTableIsIllegal(htd);
        htd.setMaxFileSize(0);
        checkTableIsIllegal(htd);
        htd.setMaxFileSize(1024 * 1024 * 1024); // 1G
        checkTableIsLegal(htd);

        htd.setMemStoreFlushSize(1024);
        checkTableIsIllegal(htd);
        htd.setMemStoreFlushSize(0);
        checkTableIsIllegal(htd);
        htd.setMemStoreFlushSize(128 * 1024 * 1024); // 128M
        checkTableIsLegal(htd);

        htd.setRegionSplitPolicyClassName("nonexisting.foo.class");
        checkTableIsIllegal(htd);
        htd.setRegionSplitPolicyClassName(null);
        checkTableIsLegal(htd);

        hcd.setBlocksize(0);
        checkTableIsIllegal(htd);
        hcd.setBlocksize(1024 * 1024 * 128); // 128M
        checkTableIsIllegal(htd);
        hcd.setBlocksize(1024);
        checkTableIsLegal(htd);

        hcd.setTimeToLive(0);
        checkTableIsIllegal(htd);
        hcd.setTimeToLive(-1);
        checkTableIsIllegal(htd);
        hcd.setTimeToLive(1);
        checkTableIsLegal(htd);

        hcd.setMinVersions(-1);
        checkTableIsIllegal(htd);
        hcd.setMinVersions(3);
        try {
            hcd.setMaxVersions(2);
            fail();
        } catch (IllegalArgumentException ex) {
            // expected
            hcd.setMaxVersions(10);
        }
        checkTableIsLegal(htd);

        hcd.setScope(-1);
        checkTableIsIllegal(htd);
        hcd.setScope(0);
        checkTableIsLegal(htd);

        // check the conf settings to disable sanity checks
        htd.setMemStoreFlushSize(0);
        htd.setConfiguration("hbase.table.sanity.checks", Boolean.FALSE.toString());
        checkTableIsLegal(htd);
    }

    private void checkTableIsLegal(HTableDescriptor htd) throws IOException {
        HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
        admin.createTable(htd);
        assertTrue(admin.tableExists(htd.getTableName()));
        admin.disableTable(htd.getTableName());
        admin.deleteTable(htd.getTableName());
    }

    private void checkTableIsIllegal(HTableDescriptor htd) throws IOException {
        HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
        try {
            admin.createTable(htd);
            fail();
        } catch (Exception ex) {
            // should throw ex
        }
        assertFalse(admin.tableExists(htd.getTableName()));
    }

    @Test
    public void testRawScanRespectsVersions() throws Exception {
        byte[] TABLE = Bytes.toBytes("testRawScan");
        HTable table = TEST_UTIL.createTable(TABLE, new byte[][] { FAMILY });
        byte[] row = Bytes.toBytes("row");

        // put the same row 4 times, with different values
        Put p = new Put(row);
        p.add(FAMILY, QUALIFIER, 10, VALUE);
        table.put(p);
        table.flushCommits();

        p = new Put(row);
        p.add(FAMILY, QUALIFIER, 11, ArrayUtils.add(VALUE, (byte) 2));
        table.put(p);
        table.flushCommits();

        p = new Put(row);
        p.add(FAMILY, QUALIFIER, 12, ArrayUtils.add(VALUE, (byte) 3));
        table.put(p);
        table.flushCommits();

        p = new Put(row);
        p.add(FAMILY, QUALIFIER, 13, ArrayUtils.add(VALUE, (byte) 4));
        table.put(p);
        table.flushCommits();

        int versions = 4;
        Scan s = new Scan(row);
        // get all the possible versions
        s.setMaxVersions();
        s.setRaw(true);

        ResultScanner scanner = table.getScanner(s);
        int count = 0;
        for (Result r : scanner) {
            assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
            count++;
        }
        assertEquals("Found more than a single row when raw scanning the table with a single row!", 1, count);
        scanner.close();

        // then if we decrease the number of versions, but keep the scan raw, we should see exactly that
        // number of versions
        versions = 2;
        s.setMaxVersions(versions);
        scanner = table.getScanner(s);
        count = 0;
        for (Result r : scanner) {
            assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
            count++;
        }
        assertEquals("Found more than a single row when raw scanning the table with a single row!", 1, count);
        scanner.close();

        // finally, if we turn off raw scanning, but max out the number of versions, we should go back
        // to seeing just three
        versions = 3;
        s.setMaxVersions(versions);
        scanner = table.getScanner(s);
        count = 0;
        for (Result r : scanner) {
            assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
            count++;
        }
        assertEquals("Found more than a single row when raw scanning the table with a single row!", 1, count);
        scanner.close();

        table.close();
        TEST_UTIL.deleteTable(TABLE);
    }

    @Test
    public void testSmallScan() throws Exception {
        // Test Initialization.
        byte[] TABLE = Bytes.toBytes("testSmallScan");
        HTable table = TEST_UTIL.createTable(TABLE, FAMILY);

        // Insert one row each region
        int insertNum = 10;
        for (int i = 0; i < 10; i++) {
            Put put = new Put(Bytes.toBytes("row" + String.format("%03d", i)));
            put.add(FAMILY, QUALIFIER, VALUE);
            table.put(put);
        }

        // nomal scan
        ResultScanner scanner = table.getScanner(new Scan());
        int count = 0;
        for (Result r : scanner) {
            assertTrue(!r.isEmpty());
            count++;
        }
        assertEquals(insertNum, count);

        // small scan
        Scan scan = new Scan(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
        scan.setSmall(true);
        scan.setCaching(2);
        scanner = table.getScanner(scan);
        count = 0;
        for (Result r : scanner) {
            assertTrue(!r.isEmpty());
            count++;
        }
        assertEquals(insertNum, count);

    }

    @Test
    public void testSuperSimpleWithReverseScan() throws Exception {
        byte[] TABLE = Bytes.toBytes("testSuperSimpleWithReverseScan");
        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
        Put put = new Put(Bytes.toBytes("0-b11111-0000000000000000000"));
        put.add(FAMILY, QUALIFIER, VALUE);
        ht.put(put);
        put = new Put(Bytes.toBytes("0-b11111-0000000000000000002"));
        put.add(FAMILY, QUALIFIER, VALUE);
        ht.put(put);
        put = new Put(Bytes.toBytes("0-b11111-0000000000000000004"));
        put.add(FAMILY, QUALIFIER, VALUE);
        ht.put(put);
        put = new Put(Bytes.toBytes("0-b11111-0000000000000000006"));
        put.add(FAMILY, QUALIFIER, VALUE);
        ht.put(put);
        put = new Put(Bytes.toBytes("0-b11111-0000000000000000008"));
        put.add(FAMILY, QUALIFIER, VALUE);
        ht.put(put);
        put = new Put(Bytes.toBytes("0-b22222-0000000000000000001"));
        put.add(FAMILY, QUALIFIER, VALUE);
        ht.put(put);
        put = new Put(Bytes.toBytes("0-b22222-0000000000000000003"));
        put.add(FAMILY, QUALIFIER, VALUE);
        ht.put(put);
        put = new Put(Bytes.toBytes("0-b22222-0000000000000000005"));
        put.add(FAMILY, QUALIFIER, VALUE);
        ht.put(put);
        put = new Put(Bytes.toBytes("0-b22222-0000000000000000007"));
        put.add(FAMILY, QUALIFIER, VALUE);
        ht.put(put);
        put = new Put(Bytes.toBytes("0-b22222-0000000000000000009"));
        put.add(FAMILY, QUALIFIER, VALUE);
        ht.put(put);
        ht.flushCommits();
        Scan scan = new Scan(Bytes.toBytes("0-b11111-9223372036854775807"),
                Bytes.toBytes("0-b11111-0000000000000000000"));
        scan.setReversed(true);
        ResultScanner scanner = ht.getScanner(scan);
        Result result = scanner.next();
        assertTrue(Bytes.equals(result.getRow(), Bytes.toBytes("0-b11111-0000000000000000008")));
        scanner.close();
        ht.close();
    }

    @Test
    public void testFiltersWithReverseScan() throws Exception {
        byte[] TABLE = Bytes.toBytes("testFiltersWithReverseScan");
        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
        byte[][] ROWS = makeN(ROW, 10);
        byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
                Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
                Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
                Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
                Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>") };
        for (int i = 0; i < 10; i++) {
            Put put = new Put(ROWS[i]);
            put.add(FAMILY, QUALIFIERS[i], VALUE);
            ht.put(put);
        }
        Scan scan = new Scan();
        scan.setReversed(true);
        scan.addFamily(FAMILY);
        Filter filter = new QualifierFilter(CompareOp.EQUAL, new RegexStringComparator("col[1-5]"));
        scan.setFilter(filter);
        ResultScanner scanner = ht.getScanner(scan);
        int expectedIndex = 5;
        for (Result result : scanner) {
            assertEquals(result.size(), 1);
            assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[expectedIndex]));
            assertTrue(Bytes.equals(result.raw()[0].getQualifier(), QUALIFIERS[expectedIndex]));
            expectedIndex--;
        }
        assertEquals(expectedIndex, 0);
        scanner.close();
        ht.close();
    }

    @Test
    public void testKeyOnlyFilterWithReverseScan() throws Exception {
        byte[] TABLE = Bytes.toBytes("testKeyOnlyFilterWithReverseScan");
        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
        byte[][] ROWS = makeN(ROW, 10);
        byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
                Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
                Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
                Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
                Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>") };
        for (int i = 0; i < 10; i++) {
            Put put = new Put(ROWS[i]);
            put.add(FAMILY, QUALIFIERS[i], VALUE);
            ht.put(put);
        }
        Scan scan = new Scan();
        scan.setReversed(true);
        scan.addFamily(FAMILY);
        Filter filter = new KeyOnlyFilter(true);
        scan.setFilter(filter);
        ResultScanner scanner = ht.getScanner(scan);
        int count = 0;
        for (Result result : ht.getScanner(scan)) {
            assertEquals(result.size(), 1);
            assertEquals(result.raw()[0].getValueLength(), Bytes.SIZEOF_INT);
            assertEquals(Bytes.toInt(result.raw()[0].getValue()), VALUE.length);
            count++;
        }
        assertEquals(count, 10);
        scanner.close();
        ht.close();
    }

    /**
     * Test simple table and non-existent row cases.
     */
    @Test
    public void testSimpleMissingWithReverseScan() throws Exception {
        byte[] TABLE = Bytes.toBytes("testSimpleMissingWithReverseScan");
        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
        byte[][] ROWS = makeN(ROW, 4);

        // Try to get a row on an empty table
        Scan scan = new Scan();
        scan.setReversed(true);
        Result result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        scan = new Scan(ROWS[0]);
        scan.setReversed(true);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        scan = new Scan(ROWS[0], ROWS[1]);
        scan.setReversed(true);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        scan = new Scan();
        scan.setReversed(true);
        scan.addFamily(FAMILY);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        scan = new Scan();
        scan.setReversed(true);
        scan.addColumn(FAMILY, QUALIFIER);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);

        // Insert a row

        Put put = new Put(ROWS[2]);
        put.add(FAMILY, QUALIFIER, VALUE);
        ht.put(put);

        // Make sure we can scan the row
        scan = new Scan();
        scan.setReversed(true);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);

        scan = new Scan(ROWS[3], ROWS[0]);
        scan.setReversed(true);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);

        scan = new Scan(ROWS[2], ROWS[1]);
        scan.setReversed(true);
        result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);

        // Try to scan empty rows around it
        // Introduced MemStore#shouldSeekForReverseScan to fix the following
        scan = new Scan(ROWS[1]);
        scan.setReversed(true);
        result = getSingleScanResult(ht, scan);
        assertNullResult(result);
        ht.close();
    }

    @Test
    public void testNullWithReverseScan() throws Exception {
        byte[] TABLE = Bytes.toBytes("testNullWithReverseScan");
        HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
        // Null qualifier (should work)
        Put put = new Put(ROW);
        put.add(FAMILY, null, VALUE);
        ht.put(put);
        scanTestNull(ht, ROW, FAMILY, VALUE, true);
        Delete delete = new Delete(ROW);
        delete.deleteColumns(FAMILY, null);
        ht.delete(delete);
        // Use a new table
        byte[] TABLE2 = Bytes.toBytes("testNull2WithReverseScan");
        ht = TEST_UTIL.createTable(TABLE2, FAMILY);
        // Empty qualifier, byte[0] instead of null (should work)
        put = new Put(ROW);
        put.add(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
        ht.put(put);
        scanTestNull(ht, ROW, FAMILY, VALUE, true);
        TEST_UTIL.flush();
        scanTestNull(ht, ROW, FAMILY, VALUE, true);
        delete = new Delete(ROW);
        delete.deleteColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
        ht.delete(delete);
        // Null value
        put = new Put(ROW);
        put.add(FAMILY, QUALIFIER, null);
        ht.put(put);
        Scan scan = new Scan();
        scan.setReversed(true);
        scan.addColumn(FAMILY, QUALIFIER);
        Result result = getSingleScanResult(ht, scan);
        assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
        ht.close();
    }

    @Test
    public void testDeletesWithReverseScan() throws Exception {
        byte[] TABLE = Bytes.toBytes("testDeletesWithReverseScan");
        byte[][] ROWS = makeNAscii(ROW, 6);
        byte[][] FAMILIES = makeNAscii(FAMILY, 3);
        byte[][] VALUES = makeN(VALUE, 5);
        long[] ts = { 1000, 2000, 3000, 4000, 5000 };
        HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, TEST_UTIL.getConfiguration(), 3);

        Put put = new Put(ROW);
        put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
        put.add(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
        ht.put(put);

        Delete delete = new Delete(ROW);
        delete.deleteFamily(FAMILIES[0], ts[0]);
        ht.delete(delete);

        Scan scan = new Scan(ROW);
        scan.setReversed(true);
        scan.addFamily(FAMILIES[0]);
        scan.setMaxVersions(Integer.MAX_VALUE);
        Result result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1] }, new byte[][] { VALUES[1] }, 0, 0);

        // Test delete latest version
        put = new Put(ROW);
        put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
        put.add(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
        put.add(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
        put.add(FAMILIES[0], null, ts[4], VALUES[4]);
        put.add(FAMILIES[0], null, ts[2], VALUES[2]);
        put.add(FAMILIES[0], null, ts[3], VALUES[3]);
        ht.put(put);

        delete = new Delete(ROW);
        delete.deleteColumn(FAMILIES[0], QUALIFIER); // ts[4]
        ht.delete(delete);

        scan = new Scan(ROW);
        scan.setReversed(true);
        scan.addColumn(FAMILIES[0], QUALIFIER);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1], ts[2], ts[3] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);

        // Test for HBASE-1847
        delete = new Delete(ROW);
        delete.deleteColumn(FAMILIES[0], null);
        ht.delete(delete);

        // Cleanup null qualifier
        delete = new Delete(ROW);
        delete.deleteColumns(FAMILIES[0], null);
        ht.delete(delete);

        // Expected client behavior might be that you can re-put deleted values
        // But alas, this is not to be. We can't put them back in either case.

        put = new Put(ROW);
        put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000
        put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000
        ht.put(put);

        // The Scanner returns the previous values, the expected-naive-unexpected
        // behavior

        scan = new Scan(ROW);
        scan.setReversed(true);
        scan.addFamily(FAMILIES[0]);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1], ts[2], ts[3] },
                new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);

        // Test deleting an entire family from one row but not the other various
        // ways

        put = new Put(ROWS[0]);
        put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
        put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
        put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
        put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
        ht.put(put);

        put = new Put(ROWS[1]);
        put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
        put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
        put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
        put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
        ht.put(put);

        put = new Put(ROWS[2]);
        put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
        put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
        put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
        put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
        ht.put(put);

        delete = new Delete(ROWS[0]);
        delete.deleteFamily(FAMILIES[2]);
        ht.delete(delete);

        delete = new Delete(ROWS[1]);
        delete.deleteColumns(FAMILIES[1], QUALIFIER);
        ht.delete(delete);

        delete = new Delete(ROWS[2]);
        delete.deleteColumn(FAMILIES[1], QUALIFIER);
        delete.deleteColumn(FAMILIES[1], QUALIFIER);
        delete.deleteColumn(FAMILIES[2], QUALIFIER);
        ht.delete(delete);

        scan = new Scan(ROWS[0]);
        scan.setReversed(true);
        scan.addFamily(FAMILIES[1]);
        scan.addFamily(FAMILIES[2]);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertTrue("Expected 2 keys but received " + result.size(), result.size() == 2);
        assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, new long[] { ts[0], ts[1] },
                new byte[][] { VALUES[0], VALUES[1] }, 0, 1);

        scan = new Scan(ROWS[1]);
        scan.setReversed(true);
        scan.addFamily(FAMILIES[1]);
        scan.addFamily(FAMILIES[2]);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertTrue("Expected 2 keys but received " + result.size(), result.size() == 2);

        scan = new Scan(ROWS[2]);
        scan.setReversed(true);
        scan.addFamily(FAMILIES[1]);
        scan.addFamily(FAMILIES[2]);
        scan.setMaxVersions(Integer.MAX_VALUE);
        result = getSingleScanResult(ht, scan);
        assertEquals(1, result.size());
        assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER, new long[] { ts[2] }, new byte[][] { VALUES[2] }, 0,
                0);

        // Test if we delete the family first in one row (HBASE-1541)

        delete = new Delete(ROWS[3]);
        delete.deleteFamily(FAMILIES[1]);
        ht.delete(delete);

        put = new Put(ROWS[3]);
        put.add(FAMILIES[2], QUALIFIER, VALUES[0]);
        ht.put(put);

        put = new Put(ROWS[4]);
        put.add(FAMILIES[1], QUALIFIER, VALUES[1]);
        put.add(FAMILIES[2], QUALIFIER, VALUES[2]);
        ht.put(put);

        scan = new Scan(ROWS[4]);
        scan.setReversed(true);
        scan.addFamily(FAMILIES[1]);
        scan.addFamily(FAMILIES[2]);
        scan.setMaxVersions(Integer.MAX_VALUE);
        ResultScanner scanner = ht.getScanner(scan);
        result = scanner.next();
        assertTrue("Expected 2 keys but received " + result.size(), result.size() == 2);
        assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[4]));
        assertTrue(Bytes.equals(result.raw()[1].getRow(), ROWS[4]));
        assertTrue(Bytes.equals(result.raw()[0].getValue(), VALUES[1]));
        assertTrue(Bytes.equals(result.raw()[1].getValue(), VALUES[2]));
        result = scanner.next();
        assertTrue("Expected 1 key but received " + result.size(), result.size() == 1);
        assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[3]));
        assertTrue(Bytes.equals(result.raw()[0].getValue(), VALUES[0]));
        scanner.close();
        ht.close();
    }

    /**
     * Tests reversed scan under multi regions
     */
    @Test
    public void testReversedScanUnderMultiRegions() throws Exception {
        // Test Initialization.
        byte[] TABLE = Bytes.toBytes("testReversedScanUnderMultiRegions");
        byte[] maxByteArray = ReversedClientScanner.MAX_BYTE_ARRAY;
        byte[][] splitRows = new byte[][] { Bytes.toBytes("005"),
                Bytes.add(Bytes.toBytes("005"), Bytes.multiple(maxByteArray, 16)), Bytes.toBytes("006"),
                Bytes.add(Bytes.toBytes("006"), Bytes.multiple(maxByteArray, 8)), Bytes.toBytes("007"),
                Bytes.add(Bytes.toBytes("007"), Bytes.multiple(maxByteArray, 4)), Bytes.toBytes("008"),
                Bytes.multiple(maxByteArray, 2) };
        HTable table = TEST_UTIL.createTable(TABLE, FAMILY, splitRows);
        TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());

        assertEquals(splitRows.length + 1, table.getRegionLocations().size());
        // Insert one row each region
        int insertNum = splitRows.length;
        for (int i = 0; i < insertNum; i++) {
            Put put = new Put(splitRows[i]);
            put.add(FAMILY, QUALIFIER, VALUE);
            table.put(put);
        }

        // scan forward
        ResultScanner scanner = table.getScanner(new Scan());
        int count = 0;
        for (Result r : scanner) {
            assertTrue(!r.isEmpty());
            count++;
        }
        assertEquals(insertNum, count);

        // scan backward
        Scan scan = new Scan();
        scan.setReversed(true);
        scanner = table.getScanner(scan);
        count = 0;
        byte[] lastRow = null;
        for (Result r : scanner) {
            assertTrue(!r.isEmpty());
            count++;
            byte[] thisRow = r.getRow();
            if (lastRow != null) {
                assertTrue("Error scan order, last row= " + Bytes.toString(lastRow) + ",this row="
                        + Bytes.toString(thisRow), Bytes.compareTo(thisRow, lastRow) < 0);
            }
            lastRow = thisRow;
        }
        assertEquals(insertNum, count);
        table.close();
    }

    /**
     * Tests reversed scan under multi regions
     */
    @Test
    public void testSmallReversedScanUnderMultiRegions() throws Exception {
        // Test Initialization.
        byte[] TABLE = Bytes.toBytes("testSmallReversedScanUnderMultiRegions");
        byte[][] splitRows = new byte[][] { Bytes.toBytes("000"), Bytes.toBytes("002"), Bytes.toBytes("004"),
                Bytes.toBytes("006"), Bytes.toBytes("008"), Bytes.toBytes("010") };
        HTable table = TEST_UTIL.createTable(TABLE, FAMILY, splitRows);
        TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());

        assertEquals(splitRows.length + 1, table.getRegionLocations().size());
        for (byte[] splitRow : splitRows) {
            Put put = new Put(splitRow);
            put.add(FAMILY, QUALIFIER, VALUE);
            table.put(put);

            byte[] nextRow = Bytes.copy(splitRow);
            nextRow[nextRow.length - 1]++;

            put = new Put(nextRow);
            put.add(FAMILY, QUALIFIER, VALUE);
            table.put(put);
        }

        // scan forward
        ResultScanner scanner = table.getScanner(new Scan());
        int count = 0;
        for (Result r : scanner) {
            assertTrue(!r.isEmpty());
            count++;
        }
        assertEquals(12, count);

        reverseScanTest(table, false);
        reverseScanTest(table, true);

        table.close();
    }

    private void reverseScanTest(HTable table, boolean small) throws IOException {
        // scan backward
        Scan scan = new Scan();
        scan.setReversed(true);
        ResultScanner scanner = table.getScanner(scan);
        int count = 0;
        byte[] lastRow = null;
        for (Result r : scanner) {
            assertTrue(!r.isEmpty());
            count++;
            byte[] thisRow = r.getRow();
            if (lastRow != null) {
                assertTrue("Error scan order, last row= " + Bytes.toString(lastRow) + ",this row="
                        + Bytes.toString(thisRow), Bytes.compareTo(thisRow, lastRow) < 0);
            }
            lastRow = thisRow;
        }
        assertEquals(12, count);

        scan = new Scan();
        scan.setSmall(small);
        scan.setReversed(true);
        scan.setStartRow(Bytes.toBytes("002"));
        scanner = table.getScanner(scan);
        count = 0;
        lastRow = null;
        for (Result r : scanner) {
            assertTrue(!r.isEmpty());
            count++;
            byte[] thisRow = r.getRow();
            if (lastRow != null) {
                assertTrue("Error scan order, last row= " + Bytes.toString(lastRow) + ",this row="
                        + Bytes.toString(thisRow), Bytes.compareTo(thisRow, lastRow) < 0);
            }
            lastRow = thisRow;
        }
        assertEquals(3, count); // 000 001 002

        scan = new Scan();
        scan.setSmall(small);
        scan.setReversed(true);
        scan.setStartRow(Bytes.toBytes("002"));
        scan.setStopRow(Bytes.toBytes("000"));
        scanner = table.getScanner(scan);
        count = 0;
        lastRow = null;
        for (Result r : scanner) {
            assertTrue(!r.isEmpty());
            count++;
            byte[] thisRow = r.getRow();
            if (lastRow != null) {
                assertTrue("Error scan order, last row= " + Bytes.toString(lastRow) + ",this row="
                        + Bytes.toString(thisRow), Bytes.compareTo(thisRow, lastRow) < 0);
            }
            lastRow = thisRow;
        }
        assertEquals(2, count); // 001 002

        scan = new Scan();
        scan.setSmall(small);
        scan.setReversed(true);
        scan.setStartRow(Bytes.toBytes("001"));
        scanner = table.getScanner(scan);
        count = 0;
        lastRow = null;
        for (Result r : scanner) {
            assertTrue(!r.isEmpty());
            count++;
            byte[] thisRow = r.getRow();
            if (lastRow != null) {
                assertTrue("Error scan order, last row= " + Bytes.toString(lastRow) + ",this row="
                        + Bytes.toString(thisRow), Bytes.compareTo(thisRow, lastRow) < 0);
            }
            lastRow = thisRow;
        }
        assertEquals(2, count); // 000 001

        scan = new Scan();
        scan.setSmall(small);
        scan.setReversed(true);
        scan.setStartRow(Bytes.toBytes("000"));
        scanner = table.getScanner(scan);
        count = 0;
        lastRow = null;
        for (Result r : scanner) {
            assertTrue(!r.isEmpty());
            count++;
            byte[] thisRow = r.getRow();
            if (lastRow != null) {
                assertTrue("Error scan order, last row= " + Bytes.toString(lastRow) + ",this row="
                        + Bytes.toString(thisRow), Bytes.compareTo(thisRow, lastRow) < 0);
            }
            lastRow = thisRow;
        }
        assertEquals(1, count); // 000

        scan = new Scan();
        scan.setSmall(small);
        scan.setReversed(true);
        scan.setStartRow(Bytes.toBytes("006"));
        scan.setStopRow(Bytes.toBytes("002"));
        scanner = table.getScanner(scan);
        count = 0;
        lastRow = null;
        for (Result r : scanner) {
            assertTrue(!r.isEmpty());
            count++;
            byte[] thisRow = r.getRow();
            if (lastRow != null) {
                assertTrue("Error scan order, last row= " + Bytes.toString(lastRow) + ",this row="
                        + Bytes.toString(thisRow), Bytes.compareTo(thisRow, lastRow) < 0);
            }
            lastRow = thisRow;
        }
        assertEquals(4, count); // 003 004 005 006
    }
}