Java tutorial
/** * Copyright (c) 2011 Yahoo! Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. See accompanying LICENSE file. */ package com.yahoo.omid; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.apache.bookkeeper.util.LocalBookKeeper; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.LocalHBaseCluster; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import com.yahoo.omid.tso.TSOServer; public class OmidTestBase { private static final Log LOG = LogFactory.getLog(OmidTestBase.class); private static ExecutorService bkExecutor; private static ExecutorService tsoExecutor; private static LocalHBaseCluster hbasecluster; protected static Configuration hbaseConf; protected static final String TEST_TABLE = "test"; protected static final String TEST_FAMILY = "data"; @BeforeClass public static void setupOmid() throws Exception { LOG.info("Setting up OmidTestBase..."); // Bookkeeper setup bkExecutor = Executors.newSingleThreadExecutor(); Runnable bkTask = new Runnable() { public void run() { try { String[] args = new String[1]; args[0] = "5"; LOG.info("Starting bk"); LocalBookKeeper.main(args); } catch (InterruptedException e) { // go away quietly } catch (Exception e) { LOG.error("Error starting local bk", e); } } }; bkExecutor.execute(bkTask); if (!LocalBookKeeper.waitForServerUp("localhost:2181", 10000)) { throw new Exception("Error starting zookeeper/bookkeeper"); } Thread.sleep(1000); // TSO Setup tsoExecutor = Executors.newSingleThreadExecutor(); Runnable tsoTask = new Runnable() { public void run() { try { String[] args = new String[] { "-port", "1234", "-batch", "100", "-ensemble", "4", "-quorum", "2", "-zk", "localhost:2181" }; TSOServer.main(args); } catch (InterruptedException e) { // go away quietly } catch (Exception e) { LOG.error("Error starting TSO", e); } } }; tsoExecutor.execute(tsoTask); TestUtils.waitForSocketListening("localhost", 1234, 1000); // HBase setup hbaseConf = HBaseConfiguration.create(); hbaseConf.set("hbase.coprocessor.region.classes", "com.yahoo.omid.regionserver.Compacter"); hbaseConf.setInt("hbase.hregion.memstore.flush.size", 100 * 1024); hbaseConf.setInt("hbase.regionserver.nbreservationblocks", 1); hbaseConf.set("tso.host", "localhost"); hbaseConf.setInt("tso.port", 1234); final String rootdir = "/tmp/hbase.test.dir/"; File rootdirFile = new File(rootdir); if (rootdirFile.exists()) { delete(rootdirFile); } hbaseConf.set("hbase.rootdir", rootdir); hbasecluster = new LocalHBaseCluster(hbaseConf, 1, 1); hbasecluster.startup(); while (hbasecluster.getActiveMaster() == null || !hbasecluster.getActiveMaster().isInitialized()) { Thread.sleep(500); } } private static void delete(File f) throws IOException { if (f.isDirectory()) { for (File c : f.listFiles()) delete(c); } if (!f.delete()) throw new FileNotFoundException("Failed to delete file: " + f); } @AfterClass public static void teardownOmid() throws Exception { LOG.info("Tearing down OmidTestBase..."); if (hbasecluster != null) { hbasecluster.shutdown(); } tsoExecutor.shutdownNow(); bkExecutor.shutdownNow(); TestUtils.waitForSocketNotListening("localhost", 1234, 1000); } @Before public void setUp() throws Exception { HBaseAdmin admin = new HBaseAdmin(hbaseConf); if (!admin.tableExists(TEST_TABLE)) { HTableDescriptor desc = new HTableDescriptor(TEST_TABLE); HColumnDescriptor datafam = new HColumnDescriptor(TEST_FAMILY); datafam.setMaxVersions(Integer.MAX_VALUE); desc.addFamily(datafam); admin.createTable(desc); } if (admin.isTableDisabled(TEST_TABLE)) { admin.enableTable(TEST_TABLE); } HTableDescriptor[] tables = admin.listTables(); for (HTableDescriptor t : tables) { LOG.info(t.getNameAsString()); } } @After public void tearDown() { try { LOG.info("tearing Down"); HBaseAdmin admin = new HBaseAdmin(hbaseConf); admin.disableTable(TEST_TABLE); admin.deleteTable(TEST_TABLE); } catch (Exception e) { LOG.error("Error tearing down", e); } } protected static void dumpTable(String table) throws Exception { HTable t = new HTable(hbaseConf, table); if (LOG.isTraceEnabled()) { ResultScanner rs = t.getScanner(new Scan()); Result r = rs.next(); while (r != null) { for (KeyValue kv : r.list()) { LOG.trace("KV: " + kv.getKeyString().toString() + " value:" + Bytes.toString(kv.getValue())); } r = rs.next(); } } } protected static boolean verifyValue(byte[] table, byte[] row, byte[] fam, byte[] col, byte[] value) { try { HTable t = new HTable(hbaseConf, table); Get g = new Get(row).setMaxVersions(1); Result r = t.get(g); KeyValue kv = r.getColumnLatest(fam, col); if (LOG.isTraceEnabled()) { LOG.trace("Value for " + Bytes.toString(table) + ":" + Bytes.toString(row) + ":" + Bytes.toString(fam) + Bytes.toString(col) + "=>" + Bytes.toString(kv.getValue()) + " (" + Bytes.toString(value) + " expected)"); } return Bytes.equals(kv.getValue(), value); } catch (Exception e) { LOG.error("Error reading row " + Bytes.toString(table) + ":" + Bytes.toString(row) + ":" + Bytes.toString(fam) + Bytes.toString(col), e); return false; } } }