org.apache.hadoop.hbase.regionserver.TestHStore.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hbase.regionserver.TestHStore.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.regionserver;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import java.io.IOException;
import java.lang.ref.SoftReference;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.NavigableSet;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellBuilder;
import org.apache.hadoop.hbase.CellBuilderFactory;
import org.apache.hadoop.hbase.CellBuilderType;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MemoryCompactionPolicy;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterBase;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher;
import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.mockito.Mockito;

import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;

/**
 * Test class for the HStore
 */
@Category({ RegionServerTests.class, MediumTests.class })
public class TestHStore {
    private static final Log LOG = LogFactory.getLog(TestHStore.class);
    @Rule
    public TestName name = new TestName();

    HRegion region;
    HStore store;
    byte[] table = Bytes.toBytes("table");
    byte[] family = Bytes.toBytes("family");

    byte[] row = Bytes.toBytes("row");
    byte[] row2 = Bytes.toBytes("row2");
    byte[] qf1 = Bytes.toBytes("qf1");
    byte[] qf2 = Bytes.toBytes("qf2");
    byte[] qf3 = Bytes.toBytes("qf3");
    byte[] qf4 = Bytes.toBytes("qf4");
    byte[] qf5 = Bytes.toBytes("qf5");
    byte[] qf6 = Bytes.toBytes("qf6");

    NavigableSet<byte[]> qualifiers = new ConcurrentSkipListSet<>(Bytes.BYTES_COMPARATOR);

    List<Cell> expected = new ArrayList<>();
    List<Cell> result = new ArrayList<>();

    long id = System.currentTimeMillis();
    Get get = new Get(row);

    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
    private static final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();

    /**
     * Setup
     * @throws IOException
     */
    @Before
    public void setUp() throws IOException {
        qualifiers.add(qf1);
        qualifiers.add(qf3);
        qualifiers.add(qf5);

        Iterator<byte[]> iter = qualifiers.iterator();
        while (iter.hasNext()) {
            byte[] next = iter.next();
            expected.add(new KeyValue(row, family, next, 1, (byte[]) null));
            get.addColumn(family, next);
        }
    }

    private void init(String methodName) throws IOException {
        init(methodName, TEST_UTIL.getConfiguration());
    }

    private HStore init(String methodName, Configuration conf) throws IOException {
        // some of the tests write 4 versions and then flush
        // (with HBASE-4241, lower versions are collected on flush)
        return init(methodName, conf, ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(4).build());
    }

    private HStore init(String methodName, Configuration conf, ColumnFamilyDescriptor hcd) throws IOException {
        return init(methodName, conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)), hcd);
    }

    private HStore init(String methodName, Configuration conf, TableDescriptorBuilder builder,
            ColumnFamilyDescriptor hcd) throws IOException {
        return init(methodName, conf, builder, hcd, null);
    }

    private HStore init(String methodName, Configuration conf, TableDescriptorBuilder builder,
            ColumnFamilyDescriptor hcd, MyStoreHook hook) throws IOException {
        return init(methodName, conf, builder, hcd, hook, false);
    }

    private void initHRegion(String methodName, Configuration conf, TableDescriptorBuilder builder,
            ColumnFamilyDescriptor hcd, MyStoreHook hook, boolean switchToPread) throws IOException {
        TableDescriptor htd = builder.addColumnFamily(hcd).build();
        Path basedir = new Path(DIR + methodName);
        Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
        final Path logdir = new Path(basedir, AbstractFSWALProvider.getWALDirectoryName(methodName));

        FileSystem fs = FileSystem.get(conf);

        fs.delete(logdir, true);
        ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, MemStoreLABImpl.CHUNK_SIZE_DEFAULT, 1, 0,
                null);
        HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
        Configuration walConf = new Configuration(conf);
        FSUtils.setRootDir(walConf, basedir);
        WALFactory wals = new WALFactory(walConf, null, methodName);
        region = new HRegion(new HRegionFileSystem(conf, fs, tableDir, info),
                wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()), conf, htd, null);
    }

    private HStore init(String methodName, Configuration conf, TableDescriptorBuilder builder,
            ColumnFamilyDescriptor hcd, MyStoreHook hook, boolean switchToPread) throws IOException {
        initHRegion(methodName, conf, builder, hcd, hook, switchToPread);
        if (hook == null) {
            store = new HStore(region, hcd, conf);
        } else {
            store = new MyStore(region, hcd, conf, hook, switchToPread);
        }
        return store;
    }

    /**
     * Test we do not lose data if we fail a flush and then close.
     * Part of HBase-10466
     * @throws Exception
     */
    @Test
    public void testFlushSizeAccounting() throws Exception {
        LOG.info("Setting up a faulty file system that cannot write in " + this.name.getMethodName());
        final Configuration conf = HBaseConfiguration.create();
        // Only retry once.
        conf.setInt("hbase.hstore.flush.retries.number", 1);
        User user = User.createUserForTesting(conf, this.name.getMethodName(), new String[] { "foo" });
        // Inject our faulty LocalFileSystem
        conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
        user.runAs(new PrivilegedExceptionAction<Object>() {
            @Override
            public Object run() throws Exception {
                // Make sure it worked (above is sensitive to caching details in hadoop core)
                FileSystem fs = FileSystem.get(conf);
                assertEquals(FaultyFileSystem.class, fs.getClass());
                FaultyFileSystem ffs = (FaultyFileSystem) fs;

                // Initialize region
                init(name.getMethodName(), conf);

                MemStoreSize size = store.memstore.getFlushableSize();
                assertEquals(0, size.getDataSize());
                LOG.info("Adding some data");
                MemStoreSize kvSize = new MemStoreSize();
                store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), kvSize);
                // add the heap size of active (mutable) segment
                kvSize.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD);
                size = store.memstore.getFlushableSize();
                assertEquals(kvSize, size);
                // Flush.  Bug #1 from HBASE-10466.  Make sure size calculation on failed flush is right.
                try {
                    LOG.info("Flushing");
                    flushStore(store, id++);
                    fail("Didn't bubble up IOE!");
                } catch (IOException ioe) {
                    assertTrue(ioe.getMessage().contains("Fault injected"));
                }
                // due to snapshot, change mutable to immutable segment
                kvSize.incMemStoreSize(0, CSLMImmutableSegment.DEEP_OVERHEAD_CSLM - MutableSegment.DEEP_OVERHEAD);
                size = store.memstore.getFlushableSize();
                assertEquals(kvSize, size);
                MemStoreSize kvSize2 = new MemStoreSize();
                store.add(new KeyValue(row, family, qf2, 2, (byte[]) null), kvSize2);
                kvSize2.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD);
                // Even though we add a new kv, we expect the flushable size to be 'same' since we have
                // not yet cleared the snapshot -- the above flush failed.
                assertEquals(kvSize, size);
                ffs.fault.set(false);
                flushStore(store, id++);
                size = store.memstore.getFlushableSize();
                // Size should be the foreground kv size.
                assertEquals(kvSize2, size);
                flushStore(store, id++);
                size = store.memstore.getFlushableSize();
                assertEquals(0, size.getDataSize());
                assertEquals(MutableSegment.DEEP_OVERHEAD, size.getHeapSize());
                return null;
            }
        });
    }

    /**
     * Verify that compression and data block encoding are respected by the
     * Store.createWriterInTmp() method, used on store flush.
     */
    @Test
    public void testCreateWriter() throws Exception {
        Configuration conf = HBaseConfiguration.create();
        FileSystem fs = FileSystem.get(conf);

        ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(family)
                .setCompressionType(Compression.Algorithm.GZ).setDataBlockEncoding(DataBlockEncoding.DIFF).build();
        init(name.getMethodName(), conf, hcd);

        // Test createWriterInTmp()
        StoreFileWriter writer = store.createWriterInTmp(4, hcd.getCompressionType(), false, true, false, false);
        Path path = writer.getPath();
        writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
        writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
        writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3)));
        writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4)));
        writer.close();

        // Verify that compression and encoding settings are respected
        HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), true, conf);
        assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm());
        assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding());
        reader.close();
    }

    @Test
    public void testDeleteExpiredStoreFiles() throws Exception {
        testDeleteExpiredStoreFiles(0);
        testDeleteExpiredStoreFiles(1);
    }

    /*
     * @param minVersions the MIN_VERSIONS for the column family
     */
    public void testDeleteExpiredStoreFiles(int minVersions) throws Exception {
        int storeFileNum = 4;
        int ttl = 4;
        IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
        EnvironmentEdgeManagerTestHelper.injectEdge(edge);

        Configuration conf = HBaseConfiguration.create();
        // Enable the expired store file deletion
        conf.setBoolean("hbase.store.delete.expired.storefile", true);
        // Set the compaction threshold higher to avoid normal compactions.
        conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 5);

        init(name.getMethodName() + "-" + minVersions, conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
                .setMinVersions(minVersions).setTimeToLive(ttl).build());

        long storeTtl = this.store.getScanInfo().getTtl();
        long sleepTime = storeTtl / storeFileNum;
        long timeStamp;
        // There are 4 store files and the max time stamp difference among these
        // store files will be (this.store.ttl / storeFileNum)
        for (int i = 1; i <= storeFileNum; i++) {
            LOG.info("Adding some data for the store file #" + i);
            timeStamp = EnvironmentEdgeManager.currentTime();
            this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null), null);
            this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null), null);
            this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null), null);
            flush(i);
            edge.incrementTime(sleepTime);
        }

        // Verify the total number of store files
        assertEquals(storeFileNum, this.store.getStorefiles().size());

        // Each call will find one expired store file and delete it before compaction happens.
        // There will be no compaction due to threshold above. Last file will not be replaced.
        for (int i = 1; i <= storeFileNum - 1; i++) {
            // verify the expired store file.
            assertFalse(this.store.requestCompaction().isPresent());
            Collection<HStoreFile> sfs = this.store.getStorefiles();
            // Ensure i files are gone.
            if (minVersions == 0) {
                assertEquals(storeFileNum - i, sfs.size());
                // Ensure only non-expired files remain.
                for (HStoreFile sf : sfs) {
                    assertTrue(sf.getReader().getMaxTimestamp() >= (edge.currentTime() - storeTtl));
                }
            } else {
                assertEquals(storeFileNum, sfs.size());
            }
            // Let the next store file expired.
            edge.incrementTime(sleepTime);
        }
        assertFalse(this.store.requestCompaction().isPresent());

        Collection<HStoreFile> sfs = this.store.getStorefiles();
        // Assert the last expired file is not removed.
        if (minVersions == 0) {
            assertEquals(1, sfs.size());
        }
        long ts = sfs.iterator().next().getReader().getMaxTimestamp();
        assertTrue(ts < (edge.currentTime() - storeTtl));

        for (HStoreFile sf : sfs) {
            sf.closeStoreFile(true);
        }
    }

    @Test
    public void testLowestModificationTime() throws Exception {
        Configuration conf = HBaseConfiguration.create();
        FileSystem fs = FileSystem.get(conf);
        // Initialize region
        init(name.getMethodName(), conf);

        int storeFileNum = 4;
        for (int i = 1; i <= storeFileNum; i++) {
            LOG.info("Adding some data for the store file #" + i);
            this.store.add(new KeyValue(row, family, qf1, i, (byte[]) null), null);
            this.store.add(new KeyValue(row, family, qf2, i, (byte[]) null), null);
            this.store.add(new KeyValue(row, family, qf3, i, (byte[]) null), null);
            flush(i);
        }
        // after flush; check the lowest time stamp
        long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
        long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
        assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);

        // after compact; check the lowest time stamp
        store.compact(store.requestCompaction().get(), NoLimitThroughputController.INSTANCE, null);
        lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
        lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
        assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);
    }

    private static long getLowestTimeStampFromFS(FileSystem fs, final Collection<HStoreFile> candidates)
            throws IOException {
        long minTs = Long.MAX_VALUE;
        if (candidates.isEmpty()) {
            return minTs;
        }
        Path[] p = new Path[candidates.size()];
        int i = 0;
        for (HStoreFile sf : candidates) {
            p[i] = sf.getPath();
            ++i;
        }

        FileStatus[] stats = fs.listStatus(p);
        if (stats == null || stats.length == 0) {
            return minTs;
        }
        for (FileStatus s : stats) {
            minTs = Math.min(minTs, s.getModificationTime());
        }
        return minTs;
    }

    //////////////////////////////////////////////////////////////////////////////
    // Get tests
    //////////////////////////////////////////////////////////////////////////////

    private static final int BLOCKSIZE_SMALL = 8192;

    /**
     * Test for hbase-1686.
     * @throws IOException
     */
    @Test
    public void testEmptyStoreFile() throws IOException {
        init(this.name.getMethodName());
        // Write a store file.
        this.store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), null);
        this.store.add(new KeyValue(row, family, qf2, 1, (byte[]) null), null);
        flush(1);
        // Now put in place an empty store file.  Its a little tricky.  Have to
        // do manually with hacked in sequence id.
        HStoreFile f = this.store.getStorefiles().iterator().next();
        Path storedir = f.getPath().getParent();
        long seqid = f.getMaxSequenceId();
        Configuration c = HBaseConfiguration.create();
        FileSystem fs = FileSystem.get(c);
        HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
        StoreFileWriter w = new StoreFileWriter.Builder(c, new CacheConfig(c), fs).withOutputDir(storedir)
                .withFileContext(meta).build();
        w.appendMetadata(seqid + 1, false);
        w.close();
        this.store.close();
        // Reopen it... should pick up two files
        this.store = new HStore(this.store.getHRegion(), this.store.getColumnFamilyDescriptor(), c);
        assertEquals(2, this.store.getStorefilesCount());

        result = HBaseTestingUtility.getFromStoreFile(store, get.getRow(), qualifiers);
        assertEquals(1, result.size());
    }

    /**
     * Getting data from memstore only
     * @throws IOException
     */
    @Test
    public void testGet_FromMemStoreOnly() throws IOException {
        init(this.name.getMethodName());

        //Put data in memstore
        this.store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), null);
        this.store.add(new KeyValue(row, family, qf2, 1, (byte[]) null), null);
        this.store.add(new KeyValue(row, family, qf3, 1, (byte[]) null), null);
        this.store.add(new KeyValue(row, family, qf4, 1, (byte[]) null), null);
        this.store.add(new KeyValue(row, family, qf5, 1, (byte[]) null), null);
        this.store.add(new KeyValue(row, family, qf6, 1, (byte[]) null), null);

        //Get
        result = HBaseTestingUtility.getFromStoreFile(store, get.getRow(), qualifiers);

        //Compare
        assertCheck();
    }

    @Test
    public void testTimeRangeIfSomeCellsAreDroppedInFlush() throws IOException {
        testTimeRangeIfSomeCellsAreDroppedInFlush(1);
        testTimeRangeIfSomeCellsAreDroppedInFlush(3);
        testTimeRangeIfSomeCellsAreDroppedInFlush(5);
    }

    private void testTimeRangeIfSomeCellsAreDroppedInFlush(int maxVersion) throws IOException {
        init(this.name.getMethodName(), TEST_UTIL.getConfiguration(),
                ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(maxVersion).build());
        long currentTs = 100;
        long minTs = currentTs;
        // the extra cell won't be flushed to disk,
        // so the min of timerange will be different between memStore and hfile.
        for (int i = 0; i != (maxVersion + 1); ++i) {
            this.store.add(new KeyValue(row, family, qf1, ++currentTs, (byte[]) null), null);
            if (i == 1) {
                minTs = currentTs;
            }
        }
        flushStore(store, id++);

        Collection<HStoreFile> files = store.getStorefiles();
        assertEquals(1, files.size());
        HStoreFile f = files.iterator().next();
        f.initReader();
        StoreFileReader reader = f.getReader();
        assertEquals(minTs, reader.timeRange.getMin());
        assertEquals(currentTs, reader.timeRange.getMax());
    }

    /**
     * Getting data from files only
     * @throws IOException
     */
    @Test
    public void testGet_FromFilesOnly() throws IOException {
        init(this.name.getMethodName());

        //Put data in memstore
        this.store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), null);
        this.store.add(new KeyValue(row, family, qf2, 1, (byte[]) null), null);
        //flush
        flush(1);

        //Add more data
        this.store.add(new KeyValue(row, family, qf3, 1, (byte[]) null), null);
        this.store.add(new KeyValue(row, family, qf4, 1, (byte[]) null), null);
        //flush
        flush(2);

        //Add more data
        this.store.add(new KeyValue(row, family, qf5, 1, (byte[]) null), null);
        this.store.add(new KeyValue(row, family, qf6, 1, (byte[]) null), null);
        //flush
        flush(3);

        //Get
        result = HBaseTestingUtility.getFromStoreFile(store, get.getRow(), qualifiers);
        //this.store.get(get, qualifiers, result);

        //Need to sort the result since multiple files
        Collections.sort(result, CellComparator.COMPARATOR);

        //Compare
        assertCheck();
    }

    /**
     * Getting data from memstore and files
     * @throws IOException
     */
    @Test
    public void testGet_FromMemStoreAndFiles() throws IOException {
        init(this.name.getMethodName());

        //Put data in memstore
        this.store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), null);
        this.store.add(new KeyValue(row, family, qf2, 1, (byte[]) null), null);
        //flush
        flush(1);

        //Add more data
        this.store.add(new KeyValue(row, family, qf3, 1, (byte[]) null), null);
        this.store.add(new KeyValue(row, family, qf4, 1, (byte[]) null), null);
        //flush
        flush(2);

        //Add more data
        this.store.add(new KeyValue(row, family, qf5, 1, (byte[]) null), null);
        this.store.add(new KeyValue(row, family, qf6, 1, (byte[]) null), null);

        //Get
        result = HBaseTestingUtility.getFromStoreFile(store, get.getRow(), qualifiers);

        //Need to sort the result since multiple files
        Collections.sort(result, CellComparator.COMPARATOR);

        //Compare
        assertCheck();
    }

    private void flush(int storeFilessize) throws IOException {
        this.store.snapshot();
        flushStore(store, id++);
        assertEquals(storeFilessize, this.store.getStorefiles().size());
        assertEquals(0, ((AbstractMemStore) this.store.memstore).getActive().getCellsCount());
    }

    private void assertCheck() {
        assertEquals(expected.size(), result.size());
        for (int i = 0; i < expected.size(); i++) {
            assertEquals(expected.get(i), result.get(i));
        }
    }

    @After
    public void tearDown() throws Exception {
        EnvironmentEdgeManagerTestHelper.reset();
        if (store != null) {
            try {
                store.close();
            } catch (IOException e) {
            }
            store = null;
        }
        if (region != null) {
            region.close();
            region = null;
        }
    }

    @AfterClass
    public static void tearDownAfterClass() throws IOException {
        TEST_UTIL.cleanupTestDir();
    }

    @Test
    public void testHandleErrorsInFlush() throws Exception {
        LOG.info("Setting up a faulty file system that cannot write");

        final Configuration conf = HBaseConfiguration.create();
        User user = User.createUserForTesting(conf, "testhandleerrorsinflush", new String[] { "foo" });
        // Inject our faulty LocalFileSystem
        conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
        user.runAs(new PrivilegedExceptionAction<Object>() {
            @Override
            public Object run() throws Exception {
                // Make sure it worked (above is sensitive to caching details in hadoop core)
                FileSystem fs = FileSystem.get(conf);
                assertEquals(FaultyFileSystem.class, fs.getClass());

                // Initialize region
                init(name.getMethodName(), conf);

                LOG.info("Adding some data");
                store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), null);
                store.add(new KeyValue(row, family, qf2, 1, (byte[]) null), null);
                store.add(new KeyValue(row, family, qf3, 1, (byte[]) null), null);

                LOG.info("Before flush, we should have no files");

                Collection<StoreFileInfo> files = store.getRegionFileSystem()
                        .getStoreFiles(store.getColumnFamilyName());
                assertEquals(0, files != null ? files.size() : 0);

                //flush
                try {
                    LOG.info("Flushing");
                    flush(1);
                    fail("Didn't bubble up IOE!");
                } catch (IOException ioe) {
                    assertTrue(ioe.getMessage().contains("Fault injected"));
                }

                LOG.info("After failed flush, we should still have no files!");
                files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
                assertEquals(0, files != null ? files.size() : 0);
                store.getHRegion().getWAL().close();
                return null;
            }
        });
        FileSystem.closeAllForUGI(user.getUGI());
    }

    /**
     * Faulty file system that will fail if you write past its fault position the FIRST TIME
     * only; thereafter it will succeed.  Used by {@link TestHRegion} too.
     */
    static class FaultyFileSystem extends FilterFileSystem {
        List<SoftReference<FaultyOutputStream>> outStreams = new ArrayList<>();
        private long faultPos = 200;
        AtomicBoolean fault = new AtomicBoolean(true);

        public FaultyFileSystem() {
            super(new LocalFileSystem());
            System.err.println("Creating faulty!");
        }

        @Override
        public FSDataOutputStream create(Path p) throws IOException {
            return new FaultyOutputStream(super.create(p), faultPos, fault);
        }

        @Override
        public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize,
                short replication, long blockSize, Progressable progress) throws IOException {
            return new FaultyOutputStream(
                    super.create(f, permission, overwrite, bufferSize, replication, blockSize, progress), faultPos,
                    fault);
        }

        @Override
        public FSDataOutputStream createNonRecursive(Path f, boolean overwrite, int bufferSize, short replication,
                long blockSize, Progressable progress) throws IOException {
            // Fake it.  Call create instead.  The default implementation throws an IOE
            // that this is not supported.
            return create(f, overwrite, bufferSize, replication, blockSize, progress);
        }
    }

    static class FaultyOutputStream extends FSDataOutputStream {
        volatile long faultPos = Long.MAX_VALUE;
        private final AtomicBoolean fault;

        public FaultyOutputStream(FSDataOutputStream out, long faultPos, final AtomicBoolean fault)
                throws IOException {
            super(out, null);
            this.faultPos = faultPos;
            this.fault = fault;
        }

        @Override
        public void write(byte[] buf, int offset, int length) throws IOException {
            System.err.println("faulty stream write at pos " + getPos());
            injectFault();
            super.write(buf, offset, length);
        }

        private void injectFault() throws IOException {
            if (this.fault.get() && getPos() >= faultPos) {
                throw new IOException("Fault injected");
            }
        }
    }

    private static void flushStore(HStore store, long id) throws IOException {
        StoreFlushContext storeFlushCtx = store.createFlushContext(id);
        storeFlushCtx.prepare();
        storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
        storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
    }

    /**
     * Generate a list of KeyValues for testing based on given parameters
     * @param timestamps
     * @param numRows
     * @param qualifier
     * @param family
     * @return
     */
    List<Cell> getKeyValueSet(long[] timestamps, int numRows, byte[] qualifier, byte[] family) {
        List<Cell> kvList = new ArrayList<>();
        for (int i = 1; i <= numRows; i++) {
            byte[] b = Bytes.toBytes(i);
            for (long timestamp : timestamps) {
                kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
            }
        }
        return kvList;
    }

    /**
     * Test to ensure correctness when using Stores with multiple timestamps
     * @throws IOException
     */
    @Test
    public void testMultipleTimestamps() throws IOException {
        int numRows = 1;
        long[] timestamps1 = new long[] { 1, 5, 10, 20 };
        long[] timestamps2 = new long[] { 30, 80 };

        init(this.name.getMethodName());

        List<Cell> kvList1 = getKeyValueSet(timestamps1, numRows, qf1, family);
        for (Cell kv : kvList1) {
            this.store.add(kv, null);
        }

        this.store.snapshot();
        flushStore(store, id++);

        List<Cell> kvList2 = getKeyValueSet(timestamps2, numRows, qf1, family);
        for (Cell kv : kvList2) {
            this.store.add(kv, null);
        }

        List<Cell> result;
        Get get = new Get(Bytes.toBytes(1));
        get.addColumn(family, qf1);

        get.setTimeRange(0, 15);
        result = HBaseTestingUtility.getFromStoreFile(store, get);
        assertTrue(result.size() > 0);

        get.setTimeRange(40, 90);
        result = HBaseTestingUtility.getFromStoreFile(store, get);
        assertTrue(result.size() > 0);

        get.setTimeRange(10, 45);
        result = HBaseTestingUtility.getFromStoreFile(store, get);
        assertTrue(result.size() > 0);

        get.setTimeRange(80, 145);
        result = HBaseTestingUtility.getFromStoreFile(store, get);
        assertTrue(result.size() > 0);

        get.setTimeRange(1, 2);
        result = HBaseTestingUtility.getFromStoreFile(store, get);
        assertTrue(result.size() > 0);

        get.setTimeRange(90, 200);
        result = HBaseTestingUtility.getFromStoreFile(store, get);
        assertTrue(result.size() == 0);
    }

    /**
     * Test for HBASE-3492 - Test split on empty colfam (no store files).
     *
     * @throws IOException When the IO operations fail.
     */
    @Test
    public void testSplitWithEmptyColFam() throws IOException {
        init(this.name.getMethodName());
        assertFalse(store.getSplitPoint().isPresent());
        store.getHRegion().forceSplit(null);
        assertFalse(store.getSplitPoint().isPresent());
        store.getHRegion().clearSplit();
    }

    @Test
    public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception {
        final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle";
        long anyValue = 10;

        // We'll check that it uses correct config and propagates it appropriately by going thru
        // the simplest "real" path I can find - "throttleCompaction", which just checks whether
        // a number we pass in is higher than some config value, inside compactionPolicy.
        Configuration conf = HBaseConfiguration.create();
        conf.setLong(CONFIG_KEY, anyValue);
        init(name.getMethodName() + "-xml", conf);
        assertTrue(store.throttleCompaction(anyValue + 1));
        assertFalse(store.throttleCompaction(anyValue));

        // HTD overrides XML.
        --anyValue;
        init(name.getMethodName() + "-htd", conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table))
                .setValue(CONFIG_KEY, Long.toString(anyValue)), ColumnFamilyDescriptorBuilder.of(family));
        assertTrue(store.throttleCompaction(anyValue + 1));
        assertFalse(store.throttleCompaction(anyValue));

        // HCD overrides them both.
        --anyValue;
        init(name.getMethodName() + "-hcd", conf,
                TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setValue(CONFIG_KEY,
                        Long.toString(anyValue)),
                ColumnFamilyDescriptorBuilder.newBuilder(family).setValue(CONFIG_KEY, Long.toString(anyValue))
                        .build());
        assertTrue(store.throttleCompaction(anyValue + 1));
        assertFalse(store.throttleCompaction(anyValue));
    }

    public static class DummyStoreEngine extends DefaultStoreEngine {
        public static DefaultCompactor lastCreatedCompactor = null;

        @Override
        protected void createComponents(Configuration conf, HStore store, CellComparator comparator)
                throws IOException {
            super.createComponents(conf, store, comparator);
            lastCreatedCompactor = this.compactor;
        }
    }

    @Test
    public void testStoreUsesSearchEngineOverride() throws Exception {
        Configuration conf = HBaseConfiguration.create();
        conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName());
        init(this.name.getMethodName(), conf);
        assertEquals(DummyStoreEngine.lastCreatedCompactor, this.store.storeEngine.getCompactor());
    }

    private void addStoreFile() throws IOException {
        HStoreFile f = this.store.getStorefiles().iterator().next();
        Path storedir = f.getPath().getParent();
        long seqid = this.store.getMaxSequenceId().orElse(0L);
        Configuration c = TEST_UTIL.getConfiguration();
        FileSystem fs = FileSystem.get(c);
        HFileContext fileContext = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
        StoreFileWriter w = new StoreFileWriter.Builder(c, new CacheConfig(c), fs).withOutputDir(storedir)
                .withFileContext(fileContext).build();
        w.appendMetadata(seqid + 1, false);
        w.close();
        LOG.info("Added store file:" + w.getPath());
    }

    private void archiveStoreFile(int index) throws IOException {
        Collection<HStoreFile> files = this.store.getStorefiles();
        HStoreFile sf = null;
        Iterator<HStoreFile> it = files.iterator();
        for (int i = 0; i <= index; i++) {
            sf = it.next();
        }
        store.getRegionFileSystem().removeStoreFiles(store.getColumnFamilyName(), Lists.newArrayList(sf));
    }

    private void closeCompactedFile(int index) throws IOException {
        Collection<HStoreFile> files = this.store.getStoreEngine().getStoreFileManager().getCompactedfiles();
        HStoreFile sf = null;
        Iterator<HStoreFile> it = files.iterator();
        for (int i = 0; i <= index; i++) {
            sf = it.next();
        }
        sf.closeStoreFile(true);
        store.getStoreEngine().getStoreFileManager().removeCompactedFiles(Lists.newArrayList(sf));
    }

    @Test
    public void testRefreshStoreFiles() throws Exception {
        init(name.getMethodName());

        assertEquals(0, this.store.getStorefilesCount());

        // Test refreshing store files when no store files are there
        store.refreshStoreFiles();
        assertEquals(0, this.store.getStorefilesCount());

        // add some data, flush
        this.store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), null);
        flush(1);
        assertEquals(1, this.store.getStorefilesCount());

        // add one more file
        addStoreFile();

        assertEquals(1, this.store.getStorefilesCount());
        store.refreshStoreFiles();
        assertEquals(2, this.store.getStorefilesCount());

        // add three more files
        addStoreFile();
        addStoreFile();
        addStoreFile();

        assertEquals(2, this.store.getStorefilesCount());
        store.refreshStoreFiles();
        assertEquals(5, this.store.getStorefilesCount());

        closeCompactedFile(0);
        archiveStoreFile(0);

        assertEquals(5, this.store.getStorefilesCount());
        store.refreshStoreFiles();
        assertEquals(4, this.store.getStorefilesCount());

        archiveStoreFile(0);
        archiveStoreFile(1);
        archiveStoreFile(2);

        assertEquals(4, this.store.getStorefilesCount());
        store.refreshStoreFiles();
        assertEquals(1, this.store.getStorefilesCount());

        archiveStoreFile(0);
        store.refreshStoreFiles();
        assertEquals(0, this.store.getStorefilesCount());
    }

    @SuppressWarnings("unchecked")
    @Test
    public void testRefreshStoreFilesNotChanged() throws IOException {
        init(name.getMethodName());

        assertEquals(0, this.store.getStorefilesCount());

        // add some data, flush
        this.store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), null);
        flush(1);
        // add one more file
        addStoreFile();

        HStore spiedStore = spy(store);

        // call first time after files changed
        spiedStore.refreshStoreFiles();
        assertEquals(2, this.store.getStorefilesCount());
        verify(spiedStore, times(1)).replaceStoreFiles(any(Collection.class), any(Collection.class));

        // call second time
        spiedStore.refreshStoreFiles();

        //ensure that replaceStoreFiles is not called if files are not refreshed
        verify(spiedStore, times(0)).replaceStoreFiles(null, null);
    }

    private long countMemStoreScanner(StoreScanner scanner) {
        if (scanner.currentScanners == null) {
            return 0;
        }
        return scanner.currentScanners.stream().filter(s -> !s.isFileScanner()).count();
    }

    @Test
    public void testNumberOfMemStoreScannersAfterFlush() throws IOException {
        long seqId = 100;
        long timestamp = System.currentTimeMillis();
        Cell cell0 = CellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row).setFamily(family)
                .setQualifier(qf1).setTimestamp(timestamp).setType(CellBuilder.DataType.Put).setValue(qf1).build();
        CellUtil.setSequenceId(cell0, seqId);
        testNumberOfMemStoreScannersAfterFlush(Arrays.asList(cell0), Collections.emptyList());

        Cell cell1 = CellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row).setFamily(family)
                .setQualifier(qf2).setTimestamp(timestamp).setType(CellBuilder.DataType.Put).setValue(qf1).build();
        CellUtil.setSequenceId(cell1, seqId);
        testNumberOfMemStoreScannersAfterFlush(Arrays.asList(cell0), Arrays.asList(cell1));

        seqId = 101;
        timestamp = System.currentTimeMillis();
        Cell cell2 = CellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row2).setFamily(family)
                .setQualifier(qf2).setTimestamp(timestamp).setType(CellBuilder.DataType.Put).setValue(qf1).build();
        CellUtil.setSequenceId(cell2, seqId);
        testNumberOfMemStoreScannersAfterFlush(Arrays.asList(cell0), Arrays.asList(cell1, cell2));
    }

    private void testNumberOfMemStoreScannersAfterFlush(List<Cell> inputCellsBeforeSnapshot,
            List<Cell> inputCellsAfterSnapshot) throws IOException {
        init(this.name.getMethodName() + "-" + inputCellsBeforeSnapshot.size());
        TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR);
        long seqId = Long.MIN_VALUE;
        for (Cell c : inputCellsBeforeSnapshot) {
            quals.add(CellUtil.cloneQualifier(c));
            seqId = Math.max(seqId, c.getSequenceId());
        }
        for (Cell c : inputCellsAfterSnapshot) {
            quals.add(CellUtil.cloneQualifier(c));
            seqId = Math.max(seqId, c.getSequenceId());
        }
        inputCellsBeforeSnapshot.forEach(c -> store.add(c, null));
        StoreFlushContext storeFlushCtx = store.createFlushContext(id++);
        storeFlushCtx.prepare();
        inputCellsAfterSnapshot.forEach(c -> store.add(c, null));
        int numberOfMemScannersWhenScaning = inputCellsAfterSnapshot.isEmpty() ? 1 : 2;
        try (StoreScanner s = (StoreScanner) store.getScanner(new Scan(), quals, seqId)) {
            // snaptshot + active (if it isn't empty)
            assertEquals(numberOfMemScannersWhenScaning, countMemStoreScanner(s));
            storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
            storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
            boolean more;
            int cellCount = 0;
            do {
                List<Cell> cells = new ArrayList<>();
                more = s.next(cells);
                cellCount += cells.size();
                assertEquals(more ? numberOfMemScannersWhenScaning : 0, countMemStoreScanner(s));
            } while (more);
            assertEquals(
                    "The number of cells added before snapshot is " + inputCellsBeforeSnapshot.size()
                            + ", The number of cells added after snapshot is " + inputCellsAfterSnapshot.size(),
                    inputCellsBeforeSnapshot.size() + inputCellsAfterSnapshot.size(), cellCount);
            // the current scanners is cleared
            assertEquals(0, countMemStoreScanner(s));
        }
    }

    private Cell createCell(byte[] qualifier, long ts, long sequenceId, byte[] value) throws IOException {
        return createCell(row, qualifier, ts, sequenceId, value);
    }

    private Cell createCell(byte[] row, byte[] qualifier, long ts, long sequenceId, byte[] value)
            throws IOException {
        Cell c = CellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row).setFamily(family)
                .setQualifier(qualifier).setTimestamp(ts).setType(CellBuilder.DataType.Put).setValue(value).build();
        CellUtil.setSequenceId(c, sequenceId);
        return c;
    }

    @Test
    public void testFlushBeforeCompletingScanWoFilter() throws IOException, InterruptedException {
        final AtomicBoolean timeToGoNextRow = new AtomicBoolean(false);
        final int expectedSize = 3;
        testFlushBeforeCompletingScan(new MyListHook() {
            @Override
            public void hook(int currentSize) {
                if (currentSize == expectedSize - 1) {
                    try {
                        flushStore(store, id++);
                        timeToGoNextRow.set(true);
                    } catch (IOException e) {
                        throw new RuntimeException(e);
                    }
                }
            }
        }, new FilterBase() {
            @Override
            public Filter.ReturnCode filterKeyValue(Cell v) throws IOException {
                return ReturnCode.INCLUDE;
            }
        }, expectedSize);
    }

    @Test
    public void testFlushBeforeCompletingScanWithFilter() throws IOException, InterruptedException {
        final AtomicBoolean timeToGoNextRow = new AtomicBoolean(false);
        final int expectedSize = 2;
        testFlushBeforeCompletingScan(new MyListHook() {
            @Override
            public void hook(int currentSize) {
                if (currentSize == expectedSize - 1) {
                    try {
                        flushStore(store, id++);
                        timeToGoNextRow.set(true);
                    } catch (IOException e) {
                        throw new RuntimeException(e);
                    }
                }
            }
        }, new FilterBase() {
            @Override
            public Filter.ReturnCode filterKeyValue(Cell v) throws IOException {
                if (timeToGoNextRow.get()) {
                    timeToGoNextRow.set(false);
                    return ReturnCode.NEXT_ROW;
                } else {
                    return ReturnCode.INCLUDE;
                }
            }
        }, expectedSize);
    }

    @Test
    public void testFlushBeforeCompletingScanWithFilterHint() throws IOException, InterruptedException {
        final AtomicBoolean timeToGetHint = new AtomicBoolean(false);
        final int expectedSize = 2;
        testFlushBeforeCompletingScan(new MyListHook() {
            @Override
            public void hook(int currentSize) {
                if (currentSize == expectedSize - 1) {
                    try {
                        flushStore(store, id++);
                        timeToGetHint.set(true);
                    } catch (IOException e) {
                        throw new RuntimeException(e);
                    }
                }
            }
        }, new FilterBase() {
            @Override
            public Filter.ReturnCode filterKeyValue(Cell v) throws IOException {
                if (timeToGetHint.get()) {
                    timeToGetHint.set(false);
                    return Filter.ReturnCode.SEEK_NEXT_USING_HINT;
                } else {
                    return Filter.ReturnCode.INCLUDE;
                }
            }

            @Override
            public Cell getNextCellHint(Cell currentCell) throws IOException {
                return currentCell;
            }
        }, expectedSize);
    }

    private void testFlushBeforeCompletingScan(MyListHook hook, Filter filter, int expectedSize)
            throws IOException, InterruptedException {
        Configuration conf = HBaseConfiguration.create();
        byte[] r0 = Bytes.toBytes("row0");
        byte[] r1 = Bytes.toBytes("row1");
        byte[] r2 = Bytes.toBytes("row2");
        byte[] value0 = Bytes.toBytes("value0");
        byte[] value1 = Bytes.toBytes("value1");
        byte[] value2 = Bytes.toBytes("value2");
        MemStoreSize memStoreSize = new MemStoreSize();
        long ts = EnvironmentEdgeManager.currentTime();
        long seqId = 100;
        init(name.getMethodName(), conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)),
                ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(1).build(), new MyStoreHook() {
                    @Override
                    public long getSmallestReadPoint(HStore store) {
                        return seqId + 3;
                    }
                });
        // The cells having the value0 won't be flushed to disk because the value of max version is 1
        store.add(createCell(r0, qf1, ts, seqId, value0), memStoreSize);
        store.add(createCell(r0, qf2, ts, seqId, value0), memStoreSize);
        store.add(createCell(r0, qf3, ts, seqId, value0), memStoreSize);
        store.add(createCell(r1, qf1, ts + 1, seqId + 1, value1), memStoreSize);
        store.add(createCell(r1, qf2, ts + 1, seqId + 1, value1), memStoreSize);
        store.add(createCell(r1, qf3, ts + 1, seqId + 1, value1), memStoreSize);
        store.add(createCell(r2, qf1, ts + 2, seqId + 2, value2), memStoreSize);
        store.add(createCell(r2, qf2, ts + 2, seqId + 2, value2), memStoreSize);
        store.add(createCell(r2, qf3, ts + 2, seqId + 2, value2), memStoreSize);
        store.add(createCell(r1, qf1, ts + 3, seqId + 3, value1), memStoreSize);
        store.add(createCell(r1, qf2, ts + 3, seqId + 3, value1), memStoreSize);
        store.add(createCell(r1, qf3, ts + 3, seqId + 3, value1), memStoreSize);
        List<Cell> myList = new MyList<>(hook);
        Scan scan = new Scan().withStartRow(r1).setFilter(filter);
        try (InternalScanner scanner = (InternalScanner) store.getScanner(scan, null, seqId + 3)) {
            // r1
            scanner.next(myList);
            assertEquals(expectedSize, myList.size());
            for (Cell c : myList) {
                byte[] actualValue = CellUtil.cloneValue(c);
                assertTrue("expected:" + Bytes.toStringBinary(value1) + ", actual:"
                        + Bytes.toStringBinary(actualValue), Bytes.equals(actualValue, value1));
            }
            List<Cell> normalList = new ArrayList<>(3);
            // r2
            scanner.next(normalList);
            assertEquals(3, normalList.size());
            for (Cell c : normalList) {
                byte[] actualValue = CellUtil.cloneValue(c);
                assertTrue("expected:" + Bytes.toStringBinary(value2) + ", actual:"
                        + Bytes.toStringBinary(actualValue), Bytes.equals(actualValue, value2));
            }
        }
    }

    @Test
    public void testCreateScannerAndSnapshotConcurrently() throws IOException, InterruptedException {
        Configuration conf = HBaseConfiguration.create();
        conf.set(HStore.MEMSTORE_CLASS_NAME, MyCompactingMemStore.class.getName());
        init(name.getMethodName(), conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
                .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
        byte[] value = Bytes.toBytes("value");
        MemStoreSize memStoreSize = new MemStoreSize();
        long ts = EnvironmentEdgeManager.currentTime();
        long seqId = 100;
        // older data whihc shouldn't be "seen" by client
        store.add(createCell(qf1, ts, seqId, value), memStoreSize);
        store.add(createCell(qf2, ts, seqId, value), memStoreSize);
        store.add(createCell(qf3, ts, seqId, value), memStoreSize);
        TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR);
        quals.add(qf1);
        quals.add(qf2);
        quals.add(qf3);
        StoreFlushContext storeFlushCtx = store.createFlushContext(id++);
        MyCompactingMemStore.START_TEST.set(true);
        Runnable flush = () -> {
            // this is blocked until we create first scanner from pipeline and snapshot -- phase (1/5)
            // recreate the active memstore -- phase (4/5)
            storeFlushCtx.prepare();
        };
        ExecutorService service = Executors.newSingleThreadExecutor();
        service.submit(flush);
        // we get scanner from pipeline and snapshot but they are empty. -- phase (2/5)
        // this is blocked until we recreate the active memstore -- phase (3/5)
        // we get scanner from active memstore but it is empty -- phase (5/5)
        InternalScanner scanner = (InternalScanner) store.getScanner(new Scan(new Get(row)), quals, seqId + 1);
        service.shutdown();
        service.awaitTermination(20, TimeUnit.SECONDS);
        try {
            try {
                List<Cell> results = new ArrayList<>();
                scanner.next(results);
                assertEquals(3, results.size());
                for (Cell c : results) {
                    byte[] actualValue = CellUtil.cloneValue(c);
                    assertTrue("expected:" + Bytes.toStringBinary(value) + ", actual:"
                            + Bytes.toStringBinary(actualValue), Bytes.equals(actualValue, value));
                }
            } finally {
                scanner.close();
            }
        } finally {
            MyCompactingMemStore.START_TEST.set(false);
            storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
            storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
        }
    }

    @Test
    public void testScanWithDoubleFlush() throws IOException {
        Configuration conf = HBaseConfiguration.create();
        // Initialize region
        MyStore myStore = initMyStore(name.getMethodName(), conf, new MyStoreHook() {
            @Override
            public void getScanners(MyStore store) throws IOException {
                final long tmpId = id++;
                ExecutorService s = Executors.newSingleThreadExecutor();
                s.submit(() -> {
                    try {
                        // flush the store before storescanner updates the scanners from store.
                        // The current data will be flushed into files, and the memstore will
                        // be clear.
                        // -- phase (4/4)
                        flushStore(store, tmpId);
                    } catch (IOException ex) {
                        throw new RuntimeException(ex);
                    }
                });
                s.shutdown();
                try {
                    // wait for the flush, the thread will be blocked in HStore#notifyChangedReadersObservers.
                    s.awaitTermination(3, TimeUnit.SECONDS);
                } catch (InterruptedException ex) {
                }
            }
        });
        byte[] oldValue = Bytes.toBytes("oldValue");
        byte[] currentValue = Bytes.toBytes("currentValue");
        MemStoreSize memStoreSize = new MemStoreSize();
        long ts = EnvironmentEdgeManager.currentTime();
        long seqId = 100;
        // older data whihc shouldn't be "seen" by client
        myStore.add(createCell(qf1, ts, seqId, oldValue), memStoreSize);
        myStore.add(createCell(qf2, ts, seqId, oldValue), memStoreSize);
        myStore.add(createCell(qf3, ts, seqId, oldValue), memStoreSize);
        long snapshotId = id++;
        // push older data into snapshot -- phase (1/4)
        StoreFlushContext storeFlushCtx = store.createFlushContext(snapshotId);
        storeFlushCtx.prepare();

        // insert current data into active -- phase (2/4)
        myStore.add(createCell(qf1, ts + 1, seqId + 1, currentValue), memStoreSize);
        myStore.add(createCell(qf2, ts + 1, seqId + 1, currentValue), memStoreSize);
        myStore.add(createCell(qf3, ts + 1, seqId + 1, currentValue), memStoreSize);
        TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR);
        quals.add(qf1);
        quals.add(qf2);
        quals.add(qf3);
        try (InternalScanner scanner = (InternalScanner) myStore.getScanner(new Scan(new Get(row)), quals,
                seqId + 1)) {
            // complete the flush -- phase (3/4)
            storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
            storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));

            List<Cell> results = new ArrayList<>();
            scanner.next(results);
            assertEquals(3, results.size());
            for (Cell c : results) {
                byte[] actualValue = CellUtil.cloneValue(c);
                assertTrue("expected:" + Bytes.toStringBinary(currentValue) + ", actual:"
                        + Bytes.toStringBinary(actualValue), Bytes.equals(actualValue, currentValue));
            }
        }
    }

    @Test
    public void testReclaimChunkWhenScaning() throws IOException {
        init("testReclaimChunkWhenScaning");
        long ts = EnvironmentEdgeManager.currentTime();
        long seqId = 100;
        byte[] value = Bytes.toBytes("value");
        // older data whihc shouldn't be "seen" by client
        store.add(createCell(qf1, ts, seqId, value), null);
        store.add(createCell(qf2, ts, seqId, value), null);
        store.add(createCell(qf3, ts, seqId, value), null);
        TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR);
        quals.add(qf1);
        quals.add(qf2);
        quals.add(qf3);
        try (InternalScanner scanner = (InternalScanner) store.getScanner(new Scan(new Get(row)), quals, seqId)) {
            List<Cell> results = new MyList<>(size -> {
                switch (size) {
                // 1) we get the first cell (qf1)
                // 2) flush the data to have StoreScanner update inner scanners
                // 3) the chunk will be reclaimed after updaing
                case 1:
                    try {
                        flushStore(store, id++);
                    } catch (IOException e) {
                        throw new RuntimeException(e);
                    }
                    break;
                // 1) we get the second cell (qf2)
                // 2) add some cell to fill some byte into the chunk (we have only one chunk)
                case 2:
                    try {
                        byte[] newValue = Bytes.toBytes("newValue");
                        // older data whihc shouldn't be "seen" by client
                        store.add(createCell(qf1, ts + 1, seqId + 1, newValue), null);
                        store.add(createCell(qf2, ts + 1, seqId + 1, newValue), null);
                        store.add(createCell(qf3, ts + 1, seqId + 1, newValue), null);
                    } catch (IOException e) {
                        throw new RuntimeException(e);
                    }
                    break;
                default:
                    break;
                }
            });
            scanner.next(results);
            assertEquals(3, results.size());
            for (Cell c : results) {
                byte[] actualValue = CellUtil.cloneValue(c);
                assertTrue(
                        "expected:" + Bytes.toStringBinary(value) + ", actual:" + Bytes.toStringBinary(actualValue),
                        Bytes.equals(actualValue, value));
            }
        }
    }

    /**
     * If there are two running InMemoryFlushRunnable, the later InMemoryFlushRunnable
     * may change the versionedList. And the first InMemoryFlushRunnable will use the chagned
     * versionedList to remove the corresponding segments.
     * In short, there will be some segements which isn't in merge are removed.
     * @throws IOException
     * @throws InterruptedException
     */
    @Test(timeout = 30000)
    public void testRunDoubleMemStoreCompactors() throws IOException, InterruptedException {
        int flushSize = 500;
        Configuration conf = HBaseConfiguration.create();
        conf.set(HStore.MEMSTORE_CLASS_NAME, MyCompactingMemStoreWithCustomCompactor.class.getName());
        conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(flushSize));
        // Set the lower threshold to invoke the "MERGE" policy
        conf.set(MemStoreCompactor.COMPACTING_MEMSTORE_THRESHOLD_KEY, String.valueOf(0));
        init(name.getMethodName(), conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
                .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
        byte[] value = Bytes.toBytes("thisisavarylargevalue");
        MemStoreSize memStoreSize = new MemStoreSize();
        long ts = EnvironmentEdgeManager.currentTime();
        long seqId = 100;
        // older data whihc shouldn't be "seen" by client
        store.add(createCell(qf1, ts, seqId, value), memStoreSize);
        store.add(createCell(qf2, ts, seqId, value), memStoreSize);
        store.add(createCell(qf3, ts, seqId, value), memStoreSize);
        assertEquals(1, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get());
        StoreFlushContext storeFlushCtx = store.createFlushContext(id++);
        storeFlushCtx.prepare();
        // This shouldn't invoke another in-memory flush because the first compactor thread
        // hasn't accomplished the in-memory compaction.
        store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSize);
        store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSize);
        store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSize);
        assertEquals(1, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get());
        //okay. Let the compaction be completed
        MyMemStoreCompactor.START_COMPACTOR_LATCH.countDown();
        CompactingMemStore mem = (CompactingMemStore) ((HStore) store).memstore;
        while (mem.isMemStoreFlushingInMemory()) {
            TimeUnit.SECONDS.sleep(1);
        }
        // This should invoke another in-memory flush.
        store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSize);
        store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSize);
        store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSize);
        assertEquals(2, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get());
        conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
                String.valueOf(TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE));
        storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
        storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
    }

    @Test
    public void testAge() throws IOException {
        long currentTime = System.currentTimeMillis();
        ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
        edge.setValue(currentTime);
        EnvironmentEdgeManager.injectEdge(edge);
        Configuration conf = TEST_UTIL.getConfiguration();
        ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.of(family);
        initHRegion(name.getMethodName(), conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)), hcd,
                null, false);
        HStore store = new HStore(region, hcd, conf) {

            @Override
            protected StoreEngine<?, ?, ?, ?> createStoreEngine(HStore store, Configuration conf,
                    CellComparator kvComparator) throws IOException {
                List<HStoreFile> storefiles = Arrays.asList(mockStoreFile(currentTime - 10),
                        mockStoreFile(currentTime - 100), mockStoreFile(currentTime - 1000),
                        mockStoreFile(currentTime - 10000));
                StoreFileManager sfm = mock(StoreFileManager.class);
                when(sfm.getStorefiles()).thenReturn(storefiles);
                StoreEngine<?, ?, ?, ?> storeEngine = mock(StoreEngine.class);
                when(storeEngine.getStoreFileManager()).thenReturn(sfm);
                return storeEngine;
            }
        };
        assertEquals(10L, store.getMinStoreFileAge().getAsLong());
        assertEquals(10000L, store.getMaxStoreFileAge().getAsLong());
        assertEquals((10 + 100 + 1000 + 10000) / 4.0, store.getAvgStoreFileAge().getAsDouble(), 1E-4);
    }

    private HStoreFile mockStoreFile(long createdTime) {
        StoreFileInfo info = mock(StoreFileInfo.class);
        when(info.getCreatedTimestamp()).thenReturn(createdTime);
        HStoreFile sf = mock(HStoreFile.class);
        when(sf.getReader()).thenReturn(mock(StoreFileReader.class));
        when(sf.isHFile()).thenReturn(true);
        when(sf.getFileInfo()).thenReturn(info);
        return sf;
    }

    private MyStore initMyStore(String methodName, Configuration conf, MyStoreHook hook) throws IOException {
        return (MyStore) init(methodName, conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)),
                ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(5).build(), hook);
    }

    private class MyStore extends HStore {
        private final MyStoreHook hook;

        MyStore(final HRegion region, final ColumnFamilyDescriptor family, final Configuration confParam,
                MyStoreHook hook, boolean switchToPread) throws IOException {
            super(region, family, confParam);
            this.hook = hook;
        }

        @Override
        public List<KeyValueScanner> getScanners(List<HStoreFile> files, boolean cacheBlocks, boolean usePread,
                boolean isCompaction, ScanQueryMatcher matcher, byte[] startRow, boolean includeStartRow,
                byte[] stopRow, boolean includeStopRow, long readPt, boolean includeMemstoreScanner)
                throws IOException {
            hook.getScanners(this);
            return super.getScanners(files, cacheBlocks, usePread, isCompaction, matcher, startRow, true, stopRow,
                    false, readPt, includeMemstoreScanner);
        }

        @Override
        public long getSmallestReadPoint() {
            return hook.getSmallestReadPoint(this);
        }
    }

    private abstract class MyStoreHook {

        void getScanners(MyStore store) throws IOException {
        }

        long getSmallestReadPoint(HStore store) {
            return store.getHRegion().getSmallestReadPoint();
        }
    }

    @Test
    public void testSwitchingPreadtoStreamParallelyWithCompactionDischarger() throws Exception {
        Configuration conf = HBaseConfiguration.create();
        conf.set("hbase.hstore.engine.class", DummyStoreEngine.class.getName());
        conf.setLong(StoreScanner.STORESCANNER_PREAD_MAX_BYTES, 0);
        // Set the lower threshold to invoke the "MERGE" policy
        MyStore store = initMyStore(name.getMethodName(), conf, new MyStoreHook() {
        });
        MemStoreSize memStoreSize = new MemStoreSize();
        long ts = System.currentTimeMillis();
        long seqID = 1l;
        // Add some data to the region and do some flushes
        for (int i = 1; i < 10; i++) {
            store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")), memStoreSize);
        }
        // flush them
        flushStore(store, seqID);
        for (int i = 11; i < 20; i++) {
            store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")), memStoreSize);
        }
        // flush them
        flushStore(store, seqID);
        for (int i = 21; i < 30; i++) {
            store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")), memStoreSize);
        }
        // flush them
        flushStore(store, seqID);

        assertEquals(3, store.getStorefilesCount());
        Scan scan = new Scan();
        scan.addFamily(family);
        Collection<HStoreFile> storefiles2 = store.getStorefiles();
        ArrayList<HStoreFile> actualStorefiles = Lists.newArrayList(storefiles2);
        StoreScanner storeScanner = (StoreScanner) store.getScanner(scan, scan.getFamilyMap().get(family),
                Long.MAX_VALUE);
        // get the current heap
        KeyValueHeap heap = storeScanner.heap;
        // create more store files
        for (int i = 31; i < 40; i++) {
            store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")), memStoreSize);
        }
        // flush them
        flushStore(store, seqID);

        for (int i = 41; i < 50; i++) {
            store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")), memStoreSize);
        }
        // flush them
        flushStore(store, seqID);
        storefiles2 = store.getStorefiles();
        ArrayList<HStoreFile> actualStorefiles1 = Lists.newArrayList(storefiles2);
        actualStorefiles1.removeAll(actualStorefiles);
        // Do compaction
        MyThread thread = new MyThread(storeScanner);
        thread.start();
        store.replaceStoreFiles(actualStorefiles, actualStorefiles1);
        thread.join();
        KeyValueHeap heap2 = thread.getHeap();
        assertFalse(heap.equals(heap2));
    }

    private static class MyThread extends Thread {
        private StoreScanner scanner;
        private KeyValueHeap heap;

        public MyThread(StoreScanner scanner) {
            this.scanner = scanner;
        }

        public KeyValueHeap getHeap() {
            return this.heap;
        }

        public void run() {
            scanner.trySwitchToStreamRead();
            heap = scanner.heap;
        }
    }

    private static class MyMemStoreCompactor extends MemStoreCompactor {
        private static final AtomicInteger RUNNER_COUNT = new AtomicInteger(0);
        private static final CountDownLatch START_COMPACTOR_LATCH = new CountDownLatch(1);

        public MyMemStoreCompactor(CompactingMemStore compactingMemStore, MemoryCompactionPolicy compactionPolicy) {
            super(compactingMemStore, compactionPolicy);
        }

        @Override
        public boolean start() throws IOException {
            boolean isFirst = RUNNER_COUNT.getAndIncrement() == 0;
            boolean rval = super.start();
            if (isFirst) {
                try {
                    START_COMPACTOR_LATCH.await();
                } catch (InterruptedException ex) {
                    throw new RuntimeException(ex);
                }
            }
            return rval;
        }
    }

    public static class MyCompactingMemStoreWithCustomCompactor extends CompactingMemStore {
        private static final AtomicInteger RUNNER_COUNT = new AtomicInteger(0);

        public MyCompactingMemStoreWithCustomCompactor(Configuration conf, CellComparator c, HStore store,
                RegionServicesForStores regionServices, MemoryCompactionPolicy compactionPolicy)
                throws IOException {
            super(conf, c, store, regionServices, compactionPolicy);
        }

        @Override
        protected MemStoreCompactor createMemStoreCompactor(MemoryCompactionPolicy compactionPolicy) {
            return new MyMemStoreCompactor(this, compactionPolicy);
        }

        @Override
        protected boolean shouldFlushInMemory() {
            boolean rval = super.shouldFlushInMemory();
            if (rval) {
                RUNNER_COUNT.incrementAndGet();
            }
            return rval;
        }
    }

    public static class MyCompactingMemStore extends CompactingMemStore {
        private static final AtomicBoolean START_TEST = new AtomicBoolean(false);
        private final CountDownLatch getScannerLatch = new CountDownLatch(1);
        private final CountDownLatch snapshotLatch = new CountDownLatch(1);

        public MyCompactingMemStore(Configuration conf, CellComparator c, HStore store,
                RegionServicesForStores regionServices, MemoryCompactionPolicy compactionPolicy)
                throws IOException {
            super(conf, c, store, regionServices, compactionPolicy);
        }

        @Override
        protected List<KeyValueScanner> createList(int capacity) {
            if (START_TEST.get()) {
                try {
                    getScannerLatch.countDown();
                    snapshotLatch.await();
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
            }
            return new ArrayList<>(capacity);
        }

        @Override
        protected void pushActiveToPipeline(MutableSegment active) {
            if (START_TEST.get()) {
                try {
                    getScannerLatch.await();
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
            }

            super.pushActiveToPipeline(active);
            if (START_TEST.get()) {
                snapshotLatch.countDown();
            }
        }
    }

    interface MyListHook {
        void hook(int currentSize);
    }

    private static class MyList<T> implements List<T> {
        private final List<T> delegatee = new ArrayList<>();
        private final MyListHook hookAtAdd;

        MyList(final MyListHook hookAtAdd) {
            this.hookAtAdd = hookAtAdd;
        }

        @Override
        public int size() {
            return delegatee.size();
        }

        @Override
        public boolean isEmpty() {
            return delegatee.isEmpty();
        }

        @Override
        public boolean contains(Object o) {
            return delegatee.contains(o);
        }

        @Override
        public Iterator<T> iterator() {
            return delegatee.iterator();
        }

        @Override
        public Object[] toArray() {
            return delegatee.toArray();
        }

        @Override
        public <R> R[] toArray(R[] a) {
            return delegatee.toArray(a);
        }

        @Override
        public boolean add(T e) {
            hookAtAdd.hook(size());
            return delegatee.add(e);
        }

        @Override
        public boolean remove(Object o) {
            return delegatee.remove(o);
        }

        @Override
        public boolean containsAll(Collection<?> c) {
            return delegatee.containsAll(c);
        }

        @Override
        public boolean addAll(Collection<? extends T> c) {
            return delegatee.addAll(c);
        }

        @Override
        public boolean addAll(int index, Collection<? extends T> c) {
            return delegatee.addAll(index, c);
        }

        @Override
        public boolean removeAll(Collection<?> c) {
            return delegatee.removeAll(c);
        }

        @Override
        public boolean retainAll(Collection<?> c) {
            return delegatee.retainAll(c);
        }

        @Override
        public void clear() {
            delegatee.clear();
        }

        @Override
        public T get(int index) {
            return delegatee.get(index);
        }

        @Override
        public T set(int index, T element) {
            return delegatee.set(index, element);
        }

        @Override
        public void add(int index, T element) {
            delegatee.add(index, element);
        }

        @Override
        public T remove(int index) {
            return delegatee.remove(index);
        }

        @Override
        public int indexOf(Object o) {
            return delegatee.indexOf(o);
        }

        @Override
        public int lastIndexOf(Object o) {
            return delegatee.lastIndexOf(o);
        }

        @Override
        public ListIterator<T> listIterator() {
            return delegatee.listIterator();
        }

        @Override
        public ListIterator<T> listIterator(int index) {
            return delegatee.listIterator(index);
        }

        @Override
        public List<T> subList(int fromIndex, int toIndex) {
            return delegatee.subList(fromIndex, toIndex);
        }
    }
}