Java tutorial
/** * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hbase.util; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.replication.ReplicationAdmin; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.io.hfile.TestHFile; import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.RegionStates; import org.apache.hadoop.hbase.master.TableLockManager; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.SplitTransactionImpl; import org.apache.hadoop.hbase.regionserver.SplitTransactionFactory; import org.apache.hadoop.hbase.regionserver.TestEndToEndSplitTransaction; import org.apache.hadoop.hbase.replication.ReplicationFactory; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.ReplicationQueues; import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker; import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import java.io.IOException; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.HashMap; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.*; import static org.junit.Assert.*; @Category({ MiscTests.class, LargeTests.class }) public class TestHBaseFsckOneRS extends BaseTestHBaseFsck { @BeforeClass public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName()); conf.setInt("hbase.regionserver.handler.count", 2); conf.setInt("hbase.regionserver.metahandler.count", 30); conf.setInt("hbase.htable.threads.max", POOL_SIZE); conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE); conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT); conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT); TEST_UTIL.startMiniCluster(1); tableExecutorService = new ThreadPoolExecutor(1, POOL_SIZE, 60, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("testhbck")); hbfsckExecutorService = new ScheduledThreadPoolExecutor(POOL_SIZE); AssignmentManager assignmentManager = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(); regionStates = assignmentManager.getRegionStates(); connection = (ClusterConnection) TEST_UTIL.getConnection(); admin = connection.getAdmin(); admin.setBalancerRunning(false, true); TEST_UTIL.waitUntilAllRegionsAssigned(TableName.META_TABLE_NAME); TEST_UTIL.waitUntilAllRegionsAssigned(TableName.NAMESPACE_TABLE_NAME); } @AfterClass public static void tearDownAfterClass() throws Exception { tableExecutorService.shutdown(); hbfsckExecutorService.shutdown(); admin.close(); TEST_UTIL.shutdownMiniCluster(); } @Before public void setUp() { EnvironmentEdgeManager.reset(); } /** * This creates a clean table and confirms that the table is clean. */ @Test(timeout = 180000) public void testHBaseFsckClean() throws Exception { assertNoErrors(doFsck(conf, false)); TableName table = TableName.valueOf("tableClean"); try { HBaseFsck hbck = doFsck(conf, false); assertNoErrors(hbck); setupTable(table); assertEquals(ROWKEYS.length, countRows()); // We created 1 table, should be fine hbck = doFsck(conf, false); assertNoErrors(hbck); assertEquals(0, hbck.getOverlapGroups(table).size()); assertEquals(ROWKEYS.length, countRows()); } finally { cleanupTable(table); } } /** * Test thread pooling in the case where there are more regions than threads */ @Test(timeout = 180000) public void testHbckThreadpooling() throws Exception { TableName table = TableName.valueOf("tableDupeStartKey"); try { // Create table with 4 regions setupTable(table); // limit number of threads to 1. Configuration newconf = new Configuration(conf); newconf.setInt("hbasefsck.numthreads", 1); assertNoErrors(doFsck(newconf, false)); // We should pass without triggering a RejectedExecutionException } finally { cleanupTable(table); } } @Test(timeout = 180000) public void testTableWithNoRegions() throws Exception { // We might end up with empty regions in a table // see also testNoHdfsTable() TableName table = TableName.valueOf(name.getMethodName()); try { // create table with one region HTableDescriptor desc = new HTableDescriptor(table); HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM)); desc.addFamily(hcd); // If a table has no CF's it doesn't get checked createTable(TEST_UTIL, desc, null); tbl = connection.getTable(table, tableExecutorService); // Mess it up by leaving a hole in the assignment, meta, and hdfs data deleteRegion(conf, tbl.getTableDescriptor(), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, false, true); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_HDFS }); doFsck(conf, true); // fix hole doFsck(conf, true); // check that hole fixed assertNoErrors(doFsck(conf, false)); } finally { cleanupTable(table); } } @Test(timeout = 180000) public void testHbckFixOrphanTable() throws Exception { TableName table = TableName.valueOf("tableInfo"); FileSystem fs = null; Path tableinfo = null; try { setupTable(table); Path hbaseTableDir = FSUtils.getTableDir(FSUtils.getRootDir(conf), table); fs = hbaseTableDir.getFileSystem(conf); FileStatus status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir); tableinfo = status.getPath(); fs.rename(tableinfo, new Path("/.tableinfo")); //to report error if .tableinfo is missing. HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NO_TABLEINFO_FILE }); // fix OrphanTable with default .tableinfo (htd not yet cached on master) hbck = doFsck(conf, true); assertNoErrors(hbck); status = null; status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir); assertNotNull(status); HTableDescriptor htd = admin.getTableDescriptor(table); htd.setValue("NOT_DEFAULT", "true"); admin.disableTable(table); admin.modifyTable(table, htd); admin.enableTable(table); fs.delete(status.getPath(), true); // fix OrphanTable with cache htd = admin.getTableDescriptor(table); // warms up cached htd on master hbck = doFsck(conf, true); assertNoErrors(hbck); status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir); assertNotNull(status); htd = admin.getTableDescriptor(table); assertEquals(htd.getValue("NOT_DEFAULT"), "true"); } finally { if (fs != null) { fs.rename(new Path("/.tableinfo"), tableinfo); } cleanupTable(table); } } @Test(timeout = 180000) public void testReadOnlyProperty() throws Exception { HBaseFsck hbck = doFsck(conf, false); Assert.assertEquals("shouldIgnorePreCheckPermission", true, hbck.shouldIgnorePreCheckPermission()); hbck = doFsck(conf, true); Assert.assertEquals("shouldIgnorePreCheckPermission", false, hbck.shouldIgnorePreCheckPermission()); hbck = doFsck(conf, true); hbck.setIgnorePreCheckPermission(true); Assert.assertEquals("shouldIgnorePreCheckPermission", true, hbck.shouldIgnorePreCheckPermission()); } /** * This creates and fixes a bad table where a region is completely contained * by another region, and there is a hole (sort of like a bad split) */ @Test(timeout = 180000) public void testOverlapAndOrphan() throws Exception { TableName table = TableName.valueOf("tableOverlapAndOrphan"); try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); // Mess it up by creating an overlap in the metadata admin.disableTable(table); deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("B"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID); admin.enableTable(table); HRegionInfo hriOverlap = createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B")); TEST_UTIL.assignRegion(hriOverlap); ServerName server = regionStates.getRegionServerOfRegion(hriOverlap); TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.ORPHAN_HDFS_REGION, HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); // fix the problem. doFsck(conf, true); // verify that overlaps are fixed HBaseFsck hbck2 = doFsck(conf, false); assertNoErrors(hbck2); assertEquals(0, hbck2.getOverlapGroups(table).size()); assertEquals(ROWKEYS.length, countRows()); } finally { cleanupTable(table); } } /** * This creates and fixes a bad table where a region overlaps two regions -- * a start key contained in another region and its end key is contained in * yet another region. */ @Test(timeout = 180000) public void testCoveredStartKey() throws Exception { TableName table = TableName.valueOf("tableCoveredStartKey"); try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); // Mess it up by creating an overlap in the metadata HRegionInfo hriOverlap = createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B2")); TEST_UTIL.assignRegion(hriOverlap); ServerName server = regionStates.getRegionServerOfRegion(hriOverlap); TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.OVERLAP_IN_REGION_CHAIN, HBaseFsck.ErrorReporter.ERROR_CODE.OVERLAP_IN_REGION_CHAIN }); assertEquals(3, hbck.getOverlapGroups(table).size()); assertEquals(ROWKEYS.length, countRows()); // fix the problem. doFsck(conf, true); // verify that overlaps are fixed HBaseFsck hbck2 = doFsck(conf, false); assertErrors(hbck2, new HBaseFsck.ErrorReporter.ERROR_CODE[0]); assertEquals(0, hbck2.getOverlapGroups(table).size()); assertEquals(ROWKEYS.length, countRows()); } finally { cleanupTable(table); } } /** * This creates and fixes a bad table with a missing region -- hole in meta * and data missing in the fs. */ @Test(timeout = 180000) public void testRegionHole() throws Exception { TableName table = TableName.valueOf("tableRegionHole"); try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); // Mess it up by leaving a hole in the assignment, meta, and hdfs data admin.disableTable(table); deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("C"), true, true, true); admin.enableTable(table); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); // holes are separate from overlap groups assertEquals(0, hbck.getOverlapGroups(table).size()); // fix hole doFsck(conf, true); // check that hole fixed assertNoErrors(doFsck(conf, false)); assertEquals(ROWKEYS.length - 2, countRows()); // lost a region so lost a row } finally { cleanupTable(table); } } /** * The region is not deployed when the table is disabled. */ @Test(timeout = 180000) public void testRegionShouldNotBeDeployed() throws Exception { TableName table = TableName.valueOf("tableRegionShouldNotBeDeployed"); try { LOG.info("Starting testRegionShouldNotBeDeployed."); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); assertTrue(cluster.waitForActiveAndReadyMaster()); byte[][] SPLIT_KEYS = new byte[][] { new byte[0], Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), Bytes.toBytes("ddd") }; HTableDescriptor htdDisabled = new HTableDescriptor(table); htdDisabled.addFamily(new HColumnDescriptor(FAM)); // Write the .tableinfo FSTableDescriptors fstd = new FSTableDescriptors(conf); fstd.createTableDescriptor(htdDisabled); List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(conf, htdDisabled, SPLIT_KEYS); // Let's just assign everything to first RS HRegionServer hrs = cluster.getRegionServer(0); // Create region files. admin.disableTable(table); admin.enableTable(table); // Disable the table and close its regions admin.disableTable(table); HRegionInfo region = disabledRegions.remove(0); byte[] regionName = region.getRegionName(); // The region should not be assigned currently assertTrue(cluster.getServerWith(regionName) == -1); // Directly open a region on a region server. // If going through AM/ZK, the region won't be open. // Even it is opened, AM will close it which causes // flakiness of this test. HRegion r = HRegion.openHRegion(region, htdDisabled, hrs.getWAL(region), conf); hrs.addToOnlineRegions(r); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.SHOULD_NOT_BE_DEPLOYED }); // fix this fault doFsck(conf, true); // check result assertNoErrors(doFsck(conf, false)); } finally { admin.enableTable(table); cleanupTable(table); } } /** * This test makes sure that parallel instances of Hbck is disabled. * * @throws Exception */ @Test(timeout = 180000) public void testParallelHbck() throws Exception { final ExecutorService service; final Future<HBaseFsck> hbck1, hbck2; class RunHbck implements Callable<HBaseFsck> { boolean fail = true; @Override public HBaseFsck call() { Configuration c = new Configuration(conf); c.setInt("hbase.hbck.lockfile.attempts", 1); // HBASE-13574 found that in HADOOP-2.6 and later, the create file would internally retry. // To avoid flakiness of the test, set low max wait time. c.setInt("hbase.hbck.lockfile.maxwaittime", 3); try { return doFsck(c, true); // Exclusive hbck only when fixing } catch (Exception e) { if (e.getMessage().contains("Duplicate hbck")) { fail = false; } } // If we reach here, then an exception was caught if (fail) fail(); return null; } } service = Executors.newFixedThreadPool(2); hbck1 = service.submit(new RunHbck()); hbck2 = service.submit(new RunHbck()); service.shutdown(); //wait for 15 seconds, for both hbck calls finish service.awaitTermination(15, TimeUnit.SECONDS); HBaseFsck h1 = hbck1.get(); HBaseFsck h2 = hbck2.get(); // Make sure only one of the calls was successful assert (h1 == null || h2 == null); if (h1 != null) { assert (h1.getRetCode() >= 0); } if (h2 != null) { assert (h2.getRetCode() >= 0); } } /** * This test makes sure that with enough retries both parallel instances * of hbck will be completed successfully. * * @throws Exception */ @Test(timeout = 180000) public void testParallelWithRetriesHbck() throws Exception { final ExecutorService service; final Future<HBaseFsck> hbck1, hbck2; // With the ExponentialBackoffPolicyWithLimit (starting with 200 milliseconds sleep time, and // max sleep time of 5 seconds), we can retry around 15 times within 80 seconds before bail out. // // Note: the reason to use 80 seconds is that in HADOOP-2.6 and later, the create file would // retry up to HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds). See HBASE-13574 for more // details. final int timeoutInSeconds = 80; final int sleepIntervalInMilliseconds = 200; final int maxSleepTimeInMilliseconds = 6000; final int maxRetryAttempts = 15; class RunHbck implements Callable<HBaseFsck> { @Override public HBaseFsck call() throws Exception { // Increase retry attempts to make sure the non-active hbck doesn't get starved Configuration c = new Configuration(conf); c.setInt("hbase.hbck.lockfile.maxwaittime", timeoutInSeconds); c.setInt("hbase.hbck.lockfile.attempt.sleep.interval", sleepIntervalInMilliseconds); c.setInt("hbase.hbck.lockfile.attempt.maxsleeptime", maxSleepTimeInMilliseconds); c.setInt("hbase.hbck.lockfile.attempts", maxRetryAttempts); return doFsck(c, false); } } service = Executors.newFixedThreadPool(2); hbck1 = service.submit(new RunHbck()); hbck2 = service.submit(new RunHbck()); service.shutdown(); //wait for some time, for both hbck calls finish service.awaitTermination(timeoutInSeconds * 2, TimeUnit.SECONDS); HBaseFsck h1 = hbck1.get(); HBaseFsck h2 = hbck2.get(); // Both should be successful assertNotNull(h1); assertNotNull(h2); assert (h1.getRetCode() >= 0); assert (h2.getRetCode() >= 0); } @Test(timeout = 180000) public void testRegionBoundariesCheck() throws Exception { HBaseFsck hbck = doFsck(conf, false); assertNoErrors(hbck); // no errors try { hbck.connect(); // need connection to have access to META hbck.checkRegionBoundaries(); } catch (IllegalArgumentException e) { if (e.getMessage().endsWith("not a valid DFS filename.")) { fail("Table directory path is not valid." + e.getMessage()); } } finally { hbck.close(); } } /** * test region boundaries and make sure store file had been created. * @throws Exception */ @Test(timeout = 180000) public void testRegionBoundariesCheckWithFlushTable() throws Exception { HBaseFsck hbck = doFsck(conf, false); assertNoErrors(hbck); // no errors TableName table = TableName.valueOf("testRegionBoundariesCheckWithFlushTable"); try { setupTable(table); admin.flush(table); hbck.connect(); // need connection to have access to META hbck.checkRegionBoundaries(); assertNoErrors(hbck); // no errors } catch (IllegalArgumentException e) { if (e.getMessage().endsWith("not a valid DFS filename.")) { fail("Table directory path is not valid." + e.getMessage()); } } finally { hbck.close(); } } @Test(timeout = 180000) public void testHbckAfterRegionMerge() throws Exception { TableName table = TableName.valueOf("testMergeRegionFilesInHdfs"); Table meta = null; try { // disable CatalogJanitor TEST_UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false); setupTable(table); assertEquals(ROWKEYS.length, countRows()); try (RegionLocator rl = connection.getRegionLocator(tbl.getName())) { // make sure data in regions, if in wal only there is no data loss admin.flush(table); HRegionInfo region1 = rl.getRegionLocation(Bytes.toBytes("A")).getRegionInfo(); HRegionInfo region2 = rl.getRegionLocation(Bytes.toBytes("B")).getRegionInfo(); int regionCountBeforeMerge = rl.getAllRegionLocations().size(); assertNotEquals(region1, region2); // do a region merge admin.mergeRegionsAsync(region1.getEncodedNameAsBytes(), region2.getEncodedNameAsBytes(), false); // wait until region merged long timeout = System.currentTimeMillis() + 30 * 1000; while (true) { if (rl.getAllRegionLocations().size() < regionCountBeforeMerge) { break; } else if (System.currentTimeMillis() > timeout) { fail("Time out waiting on region " + region1.getEncodedName() + " and " + region2.getEncodedName() + " be merged"); } Thread.sleep(10); } assertEquals(ROWKEYS.length, countRows()); HBaseFsck hbck = doFsck(conf, false); assertNoErrors(hbck); // no errors } } finally { TEST_UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(true); cleanupTable(table); IOUtils.closeQuietly(meta); } } /** * This creates entries in hbase:meta with no hdfs data. This should cleanly * remove the table. */ @Test(timeout = 180000) public void testNoHdfsTable() throws Exception { TableName table = TableName.valueOf("NoHdfsTable"); setupTable(table); assertEquals(ROWKEYS.length, countRows()); // make sure data in regions, if in wal only there is no data loss admin.flush(table); // Mess it up by deleting hdfs dirs deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""), Bytes.toBytes("A"), false, false, true); // don't rm meta deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("B"), false, false, true); // don't rm meta deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("C"), false, false, true); // don't rm meta deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"), Bytes.toBytes(""), false, false, true); // don't rm meta // also remove the table directory in hdfs deleteTableDir(table); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_HDFS, HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_HDFS, HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_HDFS, HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_HDFS, HBaseFsck.ErrorReporter.ERROR_CODE.ORPHAN_TABLE_STATE, }); // holes are separate from overlap groups assertEquals(0, hbck.getOverlapGroups(table).size()); // fix hole doFsck(conf, true); // detect dangling regions and remove those // check that hole fixed assertNoErrors(doFsck(conf, false)); assertFalse("Table " + table + " should have been deleted", admin.tableExists(table)); } /** * when the hbase.version file missing, It is fix the fault. */ @Test(timeout = 180000) public void testNoVersionFile() throws Exception { // delete the hbase.version file Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); Path versionFile = new Path(rootDir, HConstants.VERSION_FILE_NAME); fs.delete(versionFile, true); // test HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NO_VERSION_FILE }); // fix hbase.version missing doFsck(conf, true); // no version file fixed assertNoErrors(doFsck(conf, false)); } @Test(timeout = 180000) public void testNoTableState() throws Exception { // delete the hbase.version file TableName table = TableName.valueOf("testNoTableState"); try { setupTable(table); // make sure data in regions, if in wal only there is no data loss admin.flush(table); MetaTableAccessor.deleteTableState(TEST_UTIL.getConnection(), table); // test HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NO_TABLE_STATE }); // fix table state missing doFsck(conf, true); assertNoErrors(doFsck(conf, false)); assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(table)); } finally { cleanupTable(table); } } /** * This creates two tables and mess both of them and fix them one by one */ @Test(timeout = 180000) public void testFixByTable() throws Exception { TableName table1 = TableName.valueOf("testFixByTable1"); TableName table2 = TableName.valueOf("testFixByTable2"); try { setupTable(table1); // make sure data in regions, if in wal only there is no data loss admin.flush(table1); // Mess them up by leaving a hole in the hdfs data deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("C"), false, false, true); // don't rm meta setupTable(table2); // make sure data in regions, if in wal only there is no data loss admin.flush(table2); // Mess them up by leaving a hole in the hdfs data deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("C"), false, false, true); // don't rm meta HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_HDFS, HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_HDFS }); // fix hole in table 1 doFsck(conf, true, table1); // check that hole in table 1 fixed assertNoErrors(doFsck(conf, false, table1)); // check that hole in table 2 still there assertErrors(doFsck(conf, false, table2), new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_HDFS }); // fix hole in table 2 doFsck(conf, true, table2); // check that hole in both tables fixed assertNoErrors(doFsck(conf, false)); assertEquals(ROWKEYS.length - 2, countRows()); } finally { cleanupTable(table1); cleanupTable(table2); } } /** * A split parent in meta, in hdfs, and not deployed */ @Test(timeout = 180000) public void testLingeringSplitParent() throws Exception { TableName table = TableName.valueOf("testLingeringSplitParent"); Table meta = null; try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); // make sure data in regions, if in wal only there is no data loss admin.flush(table); HRegionLocation location; try (RegionLocator rl = connection.getRegionLocator(tbl.getName())) { location = rl.getRegionLocation(Bytes.toBytes("B")); } // Delete one region from meta, but not hdfs, unassign it. deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("C"), true, true, false); // Create a new meta entry to fake it as a split parent. meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService); HRegionInfo hri = location.getRegionInfo(); HRegionInfo a = new HRegionInfo(tbl.getName(), Bytes.toBytes("B"), Bytes.toBytes("BM")); HRegionInfo b = new HRegionInfo(tbl.getName(), Bytes.toBytes("BM"), Bytes.toBytes("C")); hri.setOffline(true); hri.setSplit(true); MetaTableAccessor.addRegionToMeta(meta, hri, a, b); meta.close(); admin.flush(TableName.META_TABLE_NAME); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.LINGERING_SPLIT_PARENT, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); // regular repair cannot fix lingering split parent hbck = doFsck(conf, true); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.LINGERING_SPLIT_PARENT, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); assertFalse(hbck.shouldRerun()); hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.LINGERING_SPLIT_PARENT, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); // fix lingering split parent hbck = new HBaseFsck(conf, hbfsckExecutorService); hbck.connect(); HBaseFsck.setDisplayFullReport(); // i.e. -details hbck.setTimeLag(0); hbck.setFixSplitParents(true); hbck.onlineHbck(); assertTrue(hbck.shouldRerun()); hbck.close(); Get get = new Get(hri.getRegionName()); Result result = meta.get(get); assertTrue(result.getColumnCells(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER).isEmpty()); assertTrue(result.getColumnCells(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER).isEmpty()); admin.flush(TableName.META_TABLE_NAME); // fix other issues doFsck(conf, true); // check that all are fixed assertNoErrors(doFsck(conf, false)); assertEquals(ROWKEYS.length, countRows()); } finally { cleanupTable(table); IOUtils.closeQuietly(meta); } } /** * Tests that LINGERING_SPLIT_PARENT is not erroneously reported for * valid cases where the daughters are there. */ @Test(timeout = 180000) public void testValidLingeringSplitParent() throws Exception { TableName table = TableName.valueOf("testLingeringSplitParent"); Table meta = null; try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); // make sure data in regions, if in wal only there is no data loss admin.flush(table); try (RegionLocator rl = connection.getRegionLocator(tbl.getName())) { HRegionLocation location = rl.getRegionLocation(Bytes.toBytes("B")); meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService); HRegionInfo hri = location.getRegionInfo(); // do a regular split byte[] regionName = location.getRegionInfo().getRegionName(); admin.splitRegion(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM")); TestEndToEndSplitTransaction.blockUntilRegionSplit(conf, 60000, regionName, true); // TODO: fixHdfsHoles does not work against splits, since the parent dir lingers on // for some time until children references are deleted. HBCK erroneously sees this as // overlapping regions HBaseFsck hbck = doFsck(conf, true, true, false, false, false, true, true, true, false, false, false, null); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] {}); //no LINGERING_SPLIT_PARENT reported // assert that the split hbase:meta entry is still there. Get get = new Get(hri.getRegionName()); Result result = meta.get(get); assertNotNull(result); assertNotNull(MetaTableAccessor.getHRegionInfo(result)); assertEquals(ROWKEYS.length, countRows()); // assert that we still have the split regions assertEquals(rl.getStartKeys().length, SPLITS.length + 1 + 1); //SPLITS + 1 is # regions // pre-split. assertNoErrors(doFsck(conf, false)); } } finally { cleanupTable(table); IOUtils.closeQuietly(meta); } } /** * Split crashed after write to hbase:meta finished for the parent region, but * failed to write daughters (pre HBASE-7721 codebase) */ @Test(timeout = 75000) public void testSplitDaughtersNotInMeta() throws Exception { TableName table = TableName.valueOf("testSplitdaughtersNotInMeta"); Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService); try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); // make sure data in regions, if in wal only there is no data loss admin.flush(table); try (RegionLocator rl = connection.getRegionLocator(tbl.getName())) { HRegionLocation location = rl.getRegionLocation(Bytes.toBytes("B")); HRegionInfo hri = location.getRegionInfo(); // Disable CatalogJanitor to prevent it from cleaning up the parent region // after split. admin.enableCatalogJanitor(false); // do a regular split byte[] regionName = location.getRegionInfo().getRegionName(); admin.splitRegion(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM")); TestEndToEndSplitTransaction.blockUntilRegionSplit(conf, 60000, regionName, true); PairOfSameType<HRegionInfo> daughters = MetaTableAccessor .getDaughterRegions(meta.get(new Get(regionName))); // Delete daughter regions from meta, but not hdfs, unassign it. ServerName firstSN = rl.getRegionLocation(daughters.getFirst().getStartKey()).getServerName(); ServerName secondSN = rl.getRegionLocation(daughters.getSecond().getStartKey()).getServerName(); undeployRegion(connection, firstSN, daughters.getFirst()); undeployRegion(connection, secondSN, daughters.getSecond()); List<Delete> deletes = new ArrayList<>(); deletes.add(new Delete(daughters.getFirst().getRegionName())); deletes.add(new Delete(daughters.getSecond().getRegionName())); meta.delete(deletes); // Remove daughters from regionStates RegionStates regionStates = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() .getRegionStates(); regionStates.deleteRegion(daughters.getFirst()); regionStates.deleteRegion(daughters.getSecond()); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); //no LINGERING_SPLIT_PARENT // now fix it. The fix should not revert the region split, but add daughters to META hbck = doFsck(conf, true, true, false, false, false, false, false, false, false, false, false, null); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); // assert that the split hbase:meta entry is still there. Get get = new Get(hri.getRegionName()); Result result = meta.get(get); assertNotNull(result); assertNotNull(MetaTableAccessor.getHRegionInfo(result)); assertEquals(ROWKEYS.length, countRows()); // assert that we still have the split regions assertEquals(rl.getStartKeys().length, SPLITS.length + 1 + 1); //SPLITS + 1 is # regions // pre-split. assertNoErrors(doFsck(conf, false)); //should be fixed by now } } finally { admin.enableCatalogJanitor(true); meta.close(); cleanupTable(table); } } /** * This creates and fixes a bad table with a missing region which is the 1st region -- hole in * meta and data missing in the fs. */ @Test(timeout = 120000) public void testMissingFirstRegion() throws Exception { TableName table = TableName.valueOf("testMissingFirstRegion"); try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); // Mess it up by leaving a hole in the assignment, meta, and hdfs data admin.disableTable(table); deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""), Bytes.toBytes("A"), true, true, true); admin.enableTable(table); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY }); // fix hole doFsck(conf, true); // check that hole fixed assertNoErrors(doFsck(conf, false)); } finally { cleanupTable(table); } } /** * This creates and fixes a bad table with a missing region which is the 1st region -- hole in * meta and data missing in the fs. */ @Test(timeout = 120000) public void testRegionDeployedNotInHdfs() throws Exception { TableName table = TableName.valueOf("testSingleRegionDeployedNotInHdfs"); try { setupTable(table); admin.flush(table); // Mess it up by deleting region dir deleteRegion(conf, tbl.getTableDescriptor(), HConstants.EMPTY_START_ROW, Bytes.toBytes("A"), false, false, true); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_HDFS }); // fix hole doFsck(conf, true); // check that hole fixed assertNoErrors(doFsck(conf, false)); } finally { cleanupTable(table); } } /** * This creates and fixes a bad table with missing last region -- hole in meta and data missing in * the fs. */ @Test(timeout = 120000) public void testMissingLastRegion() throws Exception { TableName table = TableName.valueOf("testMissingLastRegion"); try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); // Mess it up by leaving a hole in the assignment, meta, and hdfs data admin.disableTable(table); deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"), Bytes.toBytes(""), true, true, true); admin.enableTable(table); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY }); // fix hole doFsck(conf, true); // check that hole fixed assertNoErrors(doFsck(conf, false)); } finally { cleanupTable(table); } } /** * Test -noHdfsChecking option can detect and fix assignments issue. */ @Test(timeout = 180000) public void testFixAssignmentsAndNoHdfsChecking() throws Exception { TableName table = TableName.valueOf("testFixAssignmentsAndNoHdfsChecking"); try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); // Mess it up by closing a region deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("B"), true, false, false, false, HRegionInfo.DEFAULT_REPLICA_ID); // verify there is no other errors HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); // verify that noHdfsChecking report the same errors HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService); fsck.connect(); HBaseFsck.setDisplayFullReport(); // i.e. -details fsck.setTimeLag(0); fsck.setCheckHdfs(false); fsck.onlineHbck(); assertErrors(fsck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); fsck.close(); // verify that fixAssignments works fine with noHdfsChecking fsck = new HBaseFsck(conf, hbfsckExecutorService); fsck.connect(); HBaseFsck.setDisplayFullReport(); // i.e. -details fsck.setTimeLag(0); fsck.setCheckHdfs(false); fsck.setFixAssignments(true); fsck.onlineHbck(); assertTrue(fsck.shouldRerun()); fsck.onlineHbck(); assertNoErrors(fsck); assertEquals(ROWKEYS.length, countRows()); fsck.close(); } finally { cleanupTable(table); } } /** * Test -noHdfsChecking option can detect region is not in meta but deployed. * However, it can not fix it without checking Hdfs because we need to get * the region info from Hdfs in this case, then to patch the meta. */ @Test(timeout = 180000) public void testFixMetaNotWorkingWithNoHdfsChecking() throws Exception { TableName table = TableName.valueOf("testFixMetaNotWorkingWithNoHdfsChecking"); try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); // Mess it up by deleting a region from the metadata deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("B"), false, true, false, false, HRegionInfo.DEFAULT_REPLICA_ID); // verify there is no other errors HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); // verify that noHdfsChecking report the same errors HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService); fsck.connect(); HBaseFsck.setDisplayFullReport(); // i.e. -details fsck.setTimeLag(0); fsck.setCheckHdfs(false); fsck.onlineHbck(); assertErrors(fsck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); fsck.close(); // verify that fixMeta doesn't work with noHdfsChecking fsck = new HBaseFsck(conf, hbfsckExecutorService); fsck.connect(); HBaseFsck.setDisplayFullReport(); // i.e. -details fsck.setTimeLag(0); fsck.setCheckHdfs(false); fsck.setFixAssignments(true); fsck.setFixMeta(true); fsck.onlineHbck(); assertFalse(fsck.shouldRerun()); assertErrors(fsck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); fsck.close(); // fix the cluster so other tests won't be impacted fsck = doFsck(conf, true); assertTrue(fsck.shouldRerun()); fsck = doFsck(conf, true); assertNoErrors(fsck); } finally { cleanupTable(table); } } /** * Test -fixHdfsHoles doesn't work with -noHdfsChecking option, * and -noHdfsChecking can't detect orphan Hdfs region. */ @Test(timeout = 180000) public void testFixHdfsHolesNotWorkingWithNoHdfsChecking() throws Exception { TableName table = TableName.valueOf("testFixHdfsHolesNotWorkingWithNoHdfsChecking"); try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); // Mess it up by creating an overlap in the metadata admin.disableTable(table); deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("B"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID); admin.enableTable(table); HRegionInfo hriOverlap = createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A2"), Bytes.toBytes("B")); TEST_UTIL.assignRegion(hriOverlap); ServerName server = regionStates.getRegionServerOfRegion(hriOverlap); TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.ORPHAN_HDFS_REGION, HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); // verify that noHdfsChecking can't detect ORPHAN_HDFS_REGION HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService); fsck.connect(); HBaseFsck.setDisplayFullReport(); // i.e. -details fsck.setTimeLag(0); fsck.setCheckHdfs(false); fsck.onlineHbck(); assertErrors(fsck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); fsck.close(); // verify that fixHdfsHoles doesn't work with noHdfsChecking fsck = new HBaseFsck(conf, hbfsckExecutorService); fsck.connect(); HBaseFsck.setDisplayFullReport(); // i.e. -details fsck.setTimeLag(0); fsck.setCheckHdfs(false); fsck.setFixHdfsHoles(true); fsck.setFixHdfsOverlaps(true); fsck.setFixHdfsOrphans(true); fsck.onlineHbck(); assertFalse(fsck.shouldRerun()); assertErrors(fsck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); fsck.close(); } finally { if (admin.isTableDisabled(table)) { admin.enableTable(table); } cleanupTable(table); } } /** * This creates a table and then corrupts an hfile. Hbck should quarantine the file. */ @Test(timeout = 180000) public void testQuarantineCorruptHFile() throws Exception { TableName table = TableName.valueOf(name.getMethodName()); try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); admin.flush(table); // flush is async. FileSystem fs = FileSystem.get(conf); Path hfile = getFlushedHFile(fs, table); // Mess it up by leaving a hole in the assignment, meta, and hdfs data admin.disableTable(table); // create new corrupt file called deadbeef (valid hfile name) Path corrupt = new Path(hfile.getParent(), "deadbeef"); TestHFile.truncateFile(fs, hfile, corrupt); LOG.info("Created corrupted file " + corrupt); HBaseFsck.debugLsr(conf, FSUtils.getRootDir(conf)); // we cannot enable here because enable never finished due to the corrupt region. HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table); assertEquals(res.getRetCode(), 0); HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker(); assertEquals(hfcc.getHFilesChecked(), 5); assertEquals(hfcc.getCorrupted().size(), 1); assertEquals(hfcc.getFailures().size(), 0); assertEquals(hfcc.getQuarantined().size(), 1); assertEquals(hfcc.getMissing().size(), 0); // Its been fixed, verify that we can enable. admin.enableTable(table); } finally { cleanupTable(table); } } /** * This creates a table and simulates the race situation where a concurrent compaction or split * has removed an hfile after the corruption checker learned about it. */ @Test(timeout = 180000) public void testQuarantineMissingHFile() throws Exception { TableName table = TableName.valueOf(name.getMethodName()); // inject a fault in the hfcc created. final FileSystem fs = FileSystem.get(conf); HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) { @Override public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException { return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) { AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false); @Override protected void checkHFile(Path p) throws IOException { if (attemptedFirstHFile.compareAndSet(false, true)) { assertTrue(fs.delete(p, true)); // make sure delete happened. } super.checkHFile(p); } }; } }; doQuarantineTest(table, hbck, 4, 0, 0, 0, 1); // 4 attempted, but 1 missing. hbck.close(); } /** * This creates and fixes a bad table with regions that has startkey == endkey */ @Test(timeout = 180000) public void testDegenerateRegions() throws Exception { TableName table = TableName.valueOf("tableDegenerateRegions"); try { setupTable(table); assertNoErrors(doFsck(conf, false)); assertEquals(ROWKEYS.length, countRows()); // Now let's mess it up, by adding a region with a duplicate startkey HRegionInfo hriDupe = createRegion(tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("B")); TEST_UTIL.assignRegion(hriDupe); ServerName server = regionStates.getRegionServerOfRegion(hriDupe); TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.DEGENERATE_REGION, HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS, HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS }); assertEquals(2, hbck.getOverlapGroups(table).size()); assertEquals(ROWKEYS.length, countRows()); // fix the degenerate region. doFsck(conf, true); // check that the degenerate region is gone and no data loss HBaseFsck hbck2 = doFsck(conf, false); assertNoErrors(hbck2); assertEquals(0, hbck2.getOverlapGroups(table).size()); assertEquals(ROWKEYS.length, countRows()); } finally { cleanupTable(table); } } /** * Test mission REGIONINFO_QUALIFIER in hbase:meta */ @Test(timeout = 180000) public void testMissingRegionInfoQualifier() throws Exception { Connection connection = ConnectionFactory.createConnection(conf); TableName table = TableName.valueOf("testMissingRegionInfoQualifier"); try { setupTable(table); // Mess it up by removing the RegionInfo for one region. final List<Delete> deletes = new LinkedList<Delete>(); Table meta = connection.getTable(TableName.META_TABLE_NAME, hbfsckExecutorService); MetaTableAccessor.fullScanRegions(connection, new MetaTableAccessor.Visitor() { @Override public boolean visit(Result rowResult) throws IOException { HRegionInfo hri = MetaTableAccessor.getHRegionInfo(rowResult); if (hri != null && !hri.getTable().isSystemTable()) { Delete delete = new Delete(rowResult.getRow()); delete.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); deletes.add(delete); } return true; } }); meta.delete(deletes); // Mess it up by creating a fake hbase:meta entry with no associated RegionInfo meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).addColumn( HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes("node1:60020"))); meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).addColumn( HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(1362150791183L))); meta.close(); HBaseFsck hbck = doFsck(conf, false); assertTrue( hbck.getErrors().getErrorList().contains(HBaseFsck.ErrorReporter.ERROR_CODE.EMPTY_META_CELL)); // fix reference file hbck = doFsck(conf, true); // check that reference file fixed assertFalse( hbck.getErrors().getErrorList().contains(HBaseFsck.ErrorReporter.ERROR_CODE.EMPTY_META_CELL)); } finally { cleanupTable(table); } connection.close(); } /** * Test pluggable error reporter. It can be plugged in * from system property or configuration. */ @Test(timeout = 180000) public void testErrorReporter() throws Exception { try { MockErrorReporter.calledCount = 0; doFsck(conf, false); assertEquals(MockErrorReporter.calledCount, 0); conf.set("hbasefsck.errorreporter", MockErrorReporter.class.getName()); doFsck(conf, false); assertTrue(MockErrorReporter.calledCount > 20); } finally { conf.set("hbasefsck.errorreporter", HBaseFsck.PrintingErrorReporter.class.getName()); MockErrorReporter.calledCount = 0; } } @Test(timeout = 180000) public void testCheckTableLocks() throws Exception { IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(0); EnvironmentEdgeManager.injectEdge(edge); // check no errors HBaseFsck hbck = doFsck(conf, false); assertNoErrors(hbck); ServerName mockName = ServerName.valueOf("localhost", 60000, 1); final TableName tableName = TableName.valueOf("foo"); // obtain one lock final TableLockManager tableLockManager = TableLockManager.createTableLockManager(conf, TEST_UTIL.getZooKeeperWatcher(), mockName); TableLockManager.TableLock writeLock = tableLockManager.writeLock(tableName, "testCheckTableLocks"); writeLock.acquire(); hbck = doFsck(conf, false); assertNoErrors(hbck); // should not have expired, no problems edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS)); // let table lock expire hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK }); final CountDownLatch latch = new CountDownLatch(1); new Thread() { @Override public void run() { TableLockManager.TableLock readLock = tableLockManager.writeLock(tableName, "testCheckTableLocks"); try { latch.countDown(); readLock.acquire(); } catch (IOException ex) { fail(); } catch (IllegalStateException ex) { return; // expected, since this will be reaped under us. } fail("should not have come here"); }; }.start(); latch.await(); // wait until thread starts Threads.sleep(300); // wait some more to ensure writeLock.acquire() is called hbck = doFsck(conf, false); // still one expired, one not-expired assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK }); edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS)); // let table lock expire hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK, HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK }); // both are expired Configuration localConf = new Configuration(conf); // reaping from ZKInterProcessWriteLock uses znode cTime, // which is not injectable through EnvironmentEdge localConf.setLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, 1); Threads.sleep(10); hbck = doFsck(localConf, true); // now fix both cases hbck = doFsck(localConf, false); assertNoErrors(hbck); // ensure that locks are deleted writeLock = tableLockManager.writeLock(tableName, "should acquire without blocking"); writeLock.acquire(); // this should not block. writeLock.release(); // release for clean state tableLockManager.tableDeleted(tableName); } @Test(timeout = 180000) public void testCheckReplication() throws Exception { // check no errors HBaseFsck hbck = doFsck(conf, false); assertNoErrors(hbck); // create peer ReplicationAdmin replicationAdmin = new ReplicationAdmin(conf); Assert.assertEquals(0, replicationAdmin.getPeersCount()); int zkPort = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT); ReplicationPeerConfig rpc = new ReplicationPeerConfig(); rpc.setClusterKey("127.0.0.1:" + zkPort + ":/hbase"); replicationAdmin.addPeer("1", rpc, null); replicationAdmin.getPeersCount(); Assert.assertEquals(1, replicationAdmin.getPeersCount()); // create replicator ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "Test Hbase Fsck", connection); ReplicationQueues repQueues = ReplicationFactory .getReplicationQueues(new ReplicationQueuesArguments(conf, connection, zkw)); repQueues.init("server1"); // queues for current peer, no errors repQueues.addLog("1", "file1"); repQueues.addLog("1-server2", "file1"); Assert.assertEquals(2, repQueues.getAllQueues().size()); hbck = doFsck(conf, false); assertNoErrors(hbck); // queues for removed peer repQueues.addLog("2", "file1"); repQueues.addLog("2-server2", "file1"); Assert.assertEquals(4, repQueues.getAllQueues().size()); hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE, HBaseFsck.ErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE }); // fix the case hbck = doFsck(conf, true); hbck = doFsck(conf, false); assertNoErrors(hbck); // ensure only "2" is deleted Assert.assertEquals(2, repQueues.getAllQueues().size()); Assert.assertNull(repQueues.getLogsInQueue("2")); Assert.assertNull(repQueues.getLogsInQueue("2-sever2")); replicationAdmin.removePeer("1"); repQueues.removeAllQueues(); zkw.close(); replicationAdmin.close(); } /** * This creates and fixes a bad table with a missing region -- hole in meta * and data present but .regioninfo missing (an orphan hdfs region)in the fs. */ @Test(timeout = 180000) public void testHDFSRegioninfoMissing() throws Exception { TableName table = TableName.valueOf("tableHDFSRegioninfoMissing"); try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); // Mess it up by leaving a hole in the meta data admin.disableTable(table); deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("C"), true, true, false, true, HRegionInfo.DEFAULT_REPLICA_ID); admin.enableTable(table); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.ORPHAN_HDFS_REGION, HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); // holes are separate from overlap groups assertEquals(0, hbck.getOverlapGroups(table).size()); // fix hole doFsck(conf, true); // check that hole fixed assertNoErrors(doFsck(conf, false)); assertEquals(ROWKEYS.length, countRows()); } finally { cleanupTable(table); } } /** * This creates and fixes a bad table with a region that is missing meta and * not assigned to a region server. */ @Test(timeout = 180000) public void testNotInMetaOrDeployedHole() throws Exception { TableName table = TableName.valueOf("tableNotInMetaOrDeployedHole"); try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); // Mess it up by leaving a hole in the meta data admin.disableTable(table); deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("C"), true, true, false); // don't rm from fs admin.enableTable(table); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); // holes are separate from overlap groups assertEquals(0, hbck.getOverlapGroups(table).size()); // fix hole assertErrors(doFsck(conf, true), new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); // check that hole fixed assertNoErrors(doFsck(conf, false)); assertEquals(ROWKEYS.length, countRows()); } finally { cleanupTable(table); } } @Test(timeout = 180000) public void testCleanUpDaughtersNotInMetaAfterFailedSplit() throws Exception { TableName table = TableName.valueOf("testCleanUpDaughtersNotInMetaAfterFailedSplit"); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); try { HTableDescriptor desc = new HTableDescriptor(table); desc.addFamily(new HColumnDescriptor(Bytes.toBytes("f"))); createTable(TEST_UTIL, desc, null); tbl = connection.getTable(desc.getTableName()); for (int i = 0; i < 5; i++) { Put p1 = new Put(("r" + i).getBytes()); p1.addColumn(Bytes.toBytes("f"), "q1".getBytes(), "v".getBytes()); tbl.put(p1); } admin.flush(desc.getTableName()); List<HRegion> regions = cluster.getRegions(desc.getTableName()); int serverWith = cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName()); HRegionServer regionServer = cluster.getRegionServer(serverWith); cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName()); SplitTransactionImpl st = (SplitTransactionImpl) new SplitTransactionFactory( TEST_UTIL.getConfiguration()).create(regions.get(0), Bytes.toBytes("r3")); st.prepare(); st.stepsBeforePONR(regionServer, regionServer, false); AssignmentManager am = cluster.getMaster().getAssignmentManager(); for (RegionState state : am.getRegionStates().getRegionsInTransition()) { am.regionOffline(state.getRegion()); } Map<HRegionInfo, ServerName> regionsMap = new HashMap<HRegionInfo, ServerName>(); regionsMap.put(regions.get(0).getRegionInfo(), regionServer.getServerName()); am.assign(regionsMap); am.waitForAssignment(regions.get(0).getRegionInfo()); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED }); // holes are separate from overlap groups assertEquals(0, hbck.getOverlapGroups(table).size()); // fix hole assertErrors( doFsck(conf, false, true, false, false, false, false, false, false, false, false, false, null), new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED }); // check that hole fixed assertNoErrors(doFsck(conf, false)); assertEquals(5, countRows()); } finally { if (tbl != null) { tbl.close(); tbl = null; } cleanupTable(table); } } /** * This creates fixes a bad table with a hole in meta. */ @Test(timeout = 180000) public void testNotInMetaHole() throws Exception { TableName table = TableName.valueOf("tableNotInMetaHole"); try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); // Mess it up by leaving a hole in the meta data admin.disableTable(table); deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("C"), false, true, false); // don't rm from fs admin.enableTable(table); HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); // holes are separate from overlap groups assertEquals(0, hbck.getOverlapGroups(table).size()); // fix hole assertErrors(doFsck(conf, true), new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, HBaseFsck.ErrorReporter.ERROR_CODE.HOLE_IN_REGION_CHAIN }); // check that hole fixed assertNoErrors(doFsck(conf, false)); assertEquals(ROWKEYS.length, countRows()); } finally { cleanupTable(table); } } /** * This creates and fixes a bad table with a region that is in meta but has * no deployment or data hdfs */ @Test(timeout = 180000) public void testNotInHdfs() throws Exception { TableName table = TableName.valueOf("tableNotInHdfs"); try { setupTable(table); assertEquals(ROWKEYS.length, countRows()); // make sure data in regions, if in wal only there is no data loss admin.flush(table); // Mess it up by leaving a hole in the hdfs data deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("C"), false, false, true); // don't rm meta HBaseFsck hbck = doFsck(conf, false); assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_HDFS }); // holes are separate from overlap groups assertEquals(0, hbck.getOverlapGroups(table).size()); // fix hole doFsck(conf, true); // check that hole fixed assertNoErrors(doFsck(conf, false)); assertEquals(ROWKEYS.length - 2, countRows()); } finally { cleanupTable(table); } } /** * This creates a table and simulates the race situation where a concurrent compaction or split * has removed an colfam dir before the corruption checker got to it. */ // Disabled because fails sporadically. Is this test right? Timing-wise, there could be no // files in a column family on initial creation -- as suggested by Matteo. @Ignore @Test(timeout = 180000) public void testQuarantineMissingFamdir() throws Exception { TableName table = TableName.valueOf(name.getMethodName()); // inject a fault in the hfcc created. final FileSystem fs = FileSystem.get(conf); HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) { @Override public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException { return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) { AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false); @Override protected void checkColFamDir(Path p) throws IOException { if (attemptedFirstHFile.compareAndSet(false, true)) { assertTrue(fs.delete(p, true)); // make sure delete happened. } super.checkColFamDir(p); } }; } }; doQuarantineTest(table, hbck, 3, 0, 0, 0, 1); hbck.close(); } /** * This creates a table and simulates the race situation where a concurrent compaction or split * has removed a region dir before the corruption checker got to it. */ @Test(timeout = 180000) public void testQuarantineMissingRegionDir() throws Exception { TableName table = TableName.valueOf(name.getMethodName()); // inject a fault in the hfcc created. final FileSystem fs = FileSystem.get(conf); HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) { @Override public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException { return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) { AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false); @Override protected void checkRegionDir(Path p) throws IOException { if (attemptedFirstHFile.compareAndSet(false, true)) { assertTrue(fs.delete(p, true)); // make sure delete happened. } super.checkRegionDir(p); } }; } }; doQuarantineTest(table, hbck, 3, 0, 0, 0, 1); hbck.close(); } }