Java tutorial
/** * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hbase.regionserver; import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY_BYTES; import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1; import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestCase; import org.apache.hadoop.hbase.HBaseTestCase.HRegionIncommon; import org.apache.hadoop.hbase.HBaseTestCase.ScannerIncommon; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.UnknownScannerException; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.InclusiveStopFilter; import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.filter.WhileMatchFilter; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; /** * Test of a long-lived scanner validating as we go. */ @Category(SmallTests.class) public class TestScanner { @Rule public TestName name = new TestName(); private final Log LOG = LogFactory.getLog(this.getClass()); private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU(); private static final byte[] FIRST_ROW = HConstants.EMPTY_START_ROW; private static final byte[][] COLS = { HConstants.CATALOG_FAMILY }; private static final byte[][] EXPLICIT_COLS = { HConstants.REGIONINFO_QUALIFIER, HConstants.SERVER_QUALIFIER, // TODO ryan //HConstants.STARTCODE_QUALIFIER }; static final HTableDescriptor TESTTABLEDESC = new HTableDescriptor(TableName.valueOf("testscanner")); static { TESTTABLEDESC.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY) // Ten is an arbitrary number. Keep versions to help debugging. .setMaxVersions(10).setBlockCacheEnabled(false).setBlocksize(8 * 1024)); } /** HRegionInfo for root region */ public static final HRegionInfo REGION_INFO = new HRegionInfo(TESTTABLEDESC.getTableName(), HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY); private static final byte[] ROW_KEY = REGION_INFO.getRegionName(); private static final long START_CODE = Long.MAX_VALUE; private HRegion r; private HRegionIncommon region; private byte[] firstRowBytes, secondRowBytes, thirdRowBytes; final private byte[] col1, col2; public TestScanner() { super(); firstRowBytes = START_KEY_BYTES; secondRowBytes = START_KEY_BYTES.clone(); // Increment the least significant character so we get to next row. secondRowBytes[START_KEY_BYTES.length - 1]++; thirdRowBytes = START_KEY_BYTES.clone(); thirdRowBytes[START_KEY_BYTES.length - 1] += 2; col1 = Bytes.toBytes("column1"); col2 = Bytes.toBytes("column2"); } /** * Test basic stop row filter works. * @throws Exception */ @Test public void testStopRow() throws Exception { byte[] startrow = Bytes.toBytes("bbb"); byte[] stoprow = Bytes.toBytes("ccc"); try { this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null); HBaseTestCase.addContent(this.r, HConstants.CATALOG_FAMILY); List<Cell> results = new ArrayList<Cell>(); // Do simple test of getting one row only first. Scan scan = new Scan(Bytes.toBytes("abc"), Bytes.toBytes("abd")); scan.addFamily(HConstants.CATALOG_FAMILY); InternalScanner s = r.getScanner(scan); int count = 0; while (s.next(results)) { count++; } s.close(); assertEquals(0, count); // Now do something a bit more imvolved. scan = new Scan(startrow, stoprow); scan.addFamily(HConstants.CATALOG_FAMILY); s = r.getScanner(scan); count = 0; Cell kv = null; results = new ArrayList<Cell>(); for (boolean first = true; s.next(results);) { kv = results.get(0); if (first) { assertTrue(CellUtil.matchingRow(kv, startrow)); first = false; } count++; } assertTrue(Bytes.BYTES_COMPARATOR.compare(stoprow, CellUtil.cloneRow(kv)) > 0); // We got something back. assertTrue(count > 10); s.close(); } finally { HRegion.closeHRegion(this.r); } } void rowPrefixFilter(Scan scan) throws IOException { List<Cell> results = new ArrayList<Cell>(); scan.addFamily(HConstants.CATALOG_FAMILY); InternalScanner s = r.getScanner(scan); boolean hasMore = true; while (hasMore) { hasMore = s.next(results); for (Cell kv : results) { assertEquals((byte) 'a', CellUtil.cloneRow(kv)[0]); assertEquals((byte) 'b', CellUtil.cloneRow(kv)[1]); } results.clear(); } s.close(); } void rowInclusiveStopFilter(Scan scan, byte[] stopRow) throws IOException { List<Cell> results = new ArrayList<Cell>(); scan.addFamily(HConstants.CATALOG_FAMILY); InternalScanner s = r.getScanner(scan); boolean hasMore = true; while (hasMore) { hasMore = s.next(results); for (Cell kv : results) { assertTrue(Bytes.compareTo(CellUtil.cloneRow(kv), stopRow) <= 0); } results.clear(); } s.close(); } @Test public void testFilters() throws IOException { try { this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null); HBaseTestCase.addContent(this.r, HConstants.CATALOG_FAMILY); byte[] prefix = Bytes.toBytes("ab"); Filter newFilter = new PrefixFilter(prefix); Scan scan = new Scan(); scan.setFilter(newFilter); rowPrefixFilter(scan); byte[] stopRow = Bytes.toBytes("bbc"); newFilter = new WhileMatchFilter(new InclusiveStopFilter(stopRow)); scan = new Scan(); scan.setFilter(newFilter); rowInclusiveStopFilter(scan, stopRow); } finally { HRegion.closeHRegion(this.r); } } /** * Test that closing a scanner while a client is using it doesn't throw * NPEs but instead a UnknownScannerException. HBASE-2503 * @throws Exception */ @Test public void testRaceBetweenClientAndTimeout() throws Exception { try { this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null); HBaseTestCase.addContent(this.r, HConstants.CATALOG_FAMILY); Scan scan = new Scan(); InternalScanner s = r.getScanner(scan); List<Cell> results = new ArrayList<Cell>(); try { s.next(results); s.close(); s.next(results); fail("We don't want anything more, we should be failing"); } catch (UnknownScannerException ex) { // ok! return; } } finally { HRegion.closeHRegion(this.r); } } /** The test! * @throws IOException */ @Test public void testScanner() throws IOException { try { r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null); region = new HRegionIncommon(r); // Write information to the meta table Put put = new Put(ROW_KEY, System.currentTimeMillis()); put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, REGION_INFO.toByteArray()); region.put(put); // What we just committed is in the memstore. Verify that we can get // it back both with scanning and get scan(false, null); getRegionInfo(); // Close and re-open r.close(); r = HRegion.openHRegion(r, null); region = new HRegionIncommon(r); // Verify we can get the data back now that it is on disk. scan(false, null); getRegionInfo(); // Store some new information String address = HConstants.LOCALHOST_IP + ":" + HBaseTestingUtility.randomFreePort(); put = new Put(ROW_KEY, System.currentTimeMillis()); put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes(address)); // put.add(HConstants.COL_STARTCODE, Bytes.toBytes(START_CODE)); region.put(put); // Validate that we can still get the HRegionInfo, even though it is in // an older row on disk and there is a newer row in the memstore scan(true, address.toString()); getRegionInfo(); // flush cache region.flushcache(); // Validate again scan(true, address.toString()); getRegionInfo(); // Close and reopen r.close(); r = HRegion.openHRegion(r, null); region = new HRegionIncommon(r); // Validate again scan(true, address.toString()); getRegionInfo(); // Now update the information again address = "bar.foo.com:4321"; put = new Put(ROW_KEY, System.currentTimeMillis()); put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes(address)); region.put(put); // Validate again scan(true, address.toString()); getRegionInfo(); // flush cache region.flushcache(); // Validate again scan(true, address.toString()); getRegionInfo(); // Close and reopen r.close(); r = HRegion.openHRegion(r, null); region = new HRegionIncommon(r); // Validate again scan(true, address.toString()); getRegionInfo(); } finally { // clean up HRegion.closeHRegion(r); } } /** Compare the HRegionInfo we read from HBase to what we stored */ private void validateRegionInfo(byte[] regionBytes) throws IOException { HRegionInfo info = HRegionInfo.parseFromOrNull(regionBytes); assertEquals(REGION_INFO.getRegionId(), info.getRegionId()); assertEquals(0, info.getStartKey().length); assertEquals(0, info.getEndKey().length); assertEquals(0, Bytes.compareTo(info.getRegionName(), REGION_INFO.getRegionName())); //assertEquals(0, info.getTableDesc().compareTo(REGION_INFO.getTableDesc())); } /** Use a scanner to get the region info and then validate the results */ private void scan(boolean validateStartcode, String serverName) throws IOException { InternalScanner scanner = null; Scan scan = null; List<Cell> results = new ArrayList<Cell>(); byte[][][] scanColumns = { COLS, EXPLICIT_COLS }; for (int i = 0; i < scanColumns.length; i++) { try { scan = new Scan(FIRST_ROW); for (int ii = 0; ii < EXPLICIT_COLS.length; ii++) { scan.addColumn(COLS[0], EXPLICIT_COLS[ii]); } scanner = r.getScanner(scan); while (scanner.next(results)) { assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER)); byte[] val = CellUtil.cloneValue( getColumn(results, HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER)); validateRegionInfo(val); if (validateStartcode) { // assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY, // HConstants.STARTCODE_QUALIFIER)); // val = getColumn(results, HConstants.CATALOG_FAMILY, // HConstants.STARTCODE_QUALIFIER).getValue(); assertNotNull(val); assertFalse(val.length == 0); long startCode = Bytes.toLong(val); assertEquals(START_CODE, startCode); } if (serverName != null) { assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER)); val = CellUtil.cloneValue( getColumn(results, HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER)); assertNotNull(val); assertFalse(val.length == 0); String server = Bytes.toString(val); assertEquals(0, server.compareTo(serverName)); } } } finally { InternalScanner s = scanner; scanner = null; if (s != null) { s.close(); } } } } private boolean hasColumn(final List<Cell> kvs, final byte[] family, final byte[] qualifier) { for (Cell kv : kvs) { if (CellUtil.matchingFamily(kv, family) && CellUtil.matchingQualifier(kv, qualifier)) { return true; } } return false; } private Cell getColumn(final List<Cell> kvs, final byte[] family, final byte[] qualifier) { for (Cell kv : kvs) { if (CellUtil.matchingFamily(kv, family) && CellUtil.matchingQualifier(kv, qualifier)) { return kv; } } return null; } /** Use get to retrieve the HRegionInfo and validate it */ private void getRegionInfo() throws IOException { Get get = new Get(ROW_KEY); get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); Result result = region.get(get); byte[] bytes = result.value(); validateRegionInfo(bytes); } /** * Tests to do a sync flush during the middle of a scan. This is testing the StoreScanner * update readers code essentially. This is not highly concurrent, since its all 1 thread. * HBase-910. * @throws Exception */ @Test public void testScanAndSyncFlush() throws Exception { this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null); HRegionIncommon hri = new HRegionIncommon(r); try { LOG.info("Added: " + HBaseTestCase.addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY), Bytes.toString(HConstants.REGIONINFO_QUALIFIER))); int count = count(hri, -1, false); assertEquals(count, count(hri, 100, false)); // do a sync flush. } catch (Exception e) { LOG.error("Failed", e); throw e; } finally { HRegion.closeHRegion(this.r); } } /** * Tests to do a concurrent flush (using a 2nd thread) while scanning. This tests both * the StoreScanner update readers and the transition from memstore -> snapshot -> store file. * * @throws Exception */ @Test public void testScanAndRealConcurrentFlush() throws Exception { this.r = TEST_UTIL.createLocalHRegion(TESTTABLEDESC, null, null); HRegionIncommon hri = new HRegionIncommon(r); try { LOG.info("Added: " + HBaseTestCase.addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY), Bytes.toString(HConstants.REGIONINFO_QUALIFIER))); int count = count(hri, -1, false); assertEquals(count, count(hri, 100, true)); // do a true concurrent background thread flush } catch (Exception e) { LOG.error("Failed", e); throw e; } finally { HRegion.closeHRegion(this.r); } } /** * Make sure scanner returns correct result when we run a major compaction * with deletes. * * @throws Exception */ @Test @SuppressWarnings("deprecation") public void testScanAndConcurrentMajorCompact() throws Exception { HTableDescriptor htd = TEST_UTIL.createTableDescriptor(name.getMethodName()); this.r = TEST_UTIL.createLocalHRegion(htd, null, null); HRegionIncommon hri = new HRegionIncommon(r); try { HBaseTestCase.addContent(hri, Bytes.toString(fam1), Bytes.toString(col1), firstRowBytes, secondRowBytes); HBaseTestCase.addContent(hri, Bytes.toString(fam2), Bytes.toString(col1), firstRowBytes, secondRowBytes); Delete dc = new Delete(firstRowBytes); /* delete column1 of firstRow */ dc.deleteColumns(fam1, col1); r.delete(dc); r.flushcache(); HBaseTestCase.addContent(hri, Bytes.toString(fam1), Bytes.toString(col1), secondRowBytes, thirdRowBytes); HBaseTestCase.addContent(hri, Bytes.toString(fam2), Bytes.toString(col1), secondRowBytes, thirdRowBytes); r.flushcache(); InternalScanner s = r.getScanner(new Scan()); // run a major compact, column1 of firstRow will be cleaned. r.compactStores(true); List<Cell> results = new ArrayList<Cell>(); s.next(results); // make sure returns column2 of firstRow assertTrue("result is not correct, keyValues : " + results, results.size() == 1); assertTrue(CellUtil.matchingRow(results.get(0), firstRowBytes)); assertTrue(CellUtil.matchingFamily(results.get(0), fam2)); results = new ArrayList<Cell>(); s.next(results); // get secondRow assertTrue(results.size() == 2); assertTrue(CellUtil.matchingRow(results.get(0), secondRowBytes)); assertTrue(CellUtil.matchingFamily(results.get(0), fam1)); assertTrue(CellUtil.matchingFamily(results.get(1), fam2)); } finally { HRegion.closeHRegion(this.r); } } /* * @param hri Region * @param flushIndex At what row we start the flush. * @param concurrent if the flush should be concurrent or sync. * @return Count of rows found. * @throws IOException */ private int count(final HRegionIncommon hri, final int flushIndex, boolean concurrent) throws IOException { LOG.info("Taking out counting scan"); ScannerIncommon s = hri.getScanner(HConstants.CATALOG_FAMILY, EXPLICIT_COLS, HConstants.EMPTY_START_ROW, HConstants.LATEST_TIMESTAMP); List<Cell> values = new ArrayList<Cell>(); int count = 0; boolean justFlushed = false; while (s.next(values)) { if (justFlushed) { LOG.info("after next() just after next flush"); justFlushed = false; } count++; if (flushIndex == count) { LOG.info("Starting flush at flush index " + flushIndex); Thread t = new Thread() { public void run() { try { hri.flushcache(); LOG.info("Finishing flush"); } catch (IOException e) { LOG.info("Failed flush cache"); } } }; if (concurrent) { t.start(); // concurrently flush. } else { t.run(); // sync flush } LOG.info("Continuing on after kicking off background flush"); justFlushed = true; } } s.close(); LOG.info("Found " + count + " items"); return count; } }