Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.pinterest.terrapin.storage; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.twitter.ostrich.stats.Stats; import com.twitter.util.ExecutorServiceFuturePool; import com.twitter.util.FuturePool; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.Compression; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.Executors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import com.pinterest.terrapin.TerrapinUtil; /** * Unit test for HFileReader. The test creates an HFile with upto 10K keys on the local file system. * It then instantiates an HFileReader over the local file system to issue HFile lookups and * ensure that the lookups are executed accurately. */ public class HFileReaderTest { private static HFileReader hfileReader; private static Map<ByteBuffer, ByteBuffer> keyValueMap; private static Set<ByteBuffer> errorKeys; private static String hfilePath; // Class to override HFileReader so we can simulate failed lookups for part of the batch // and return an exception. static class TestHFileReader extends HFileReader { private static Set<ByteBuffer> errorKeys; public TestHFileReader(FileSystem fs, String hfilePath, CacheConfig cacheConfig, FuturePool futurePool, Set<ByteBuffer> errorKeys) throws IOException { super(fs, hfilePath, cacheConfig, futurePool); this.errorKeys = errorKeys; } @Override protected Pair<ByteBuffer, Pair<ByteBuffer, Throwable>> getValueFromHFile(ByteBuffer key) { if (errorKeys.contains(key)) { return new ImmutablePair(key, new ImmutablePair(null, new IOException())); } else { return super.getValueFromHFile(key); } } } @BeforeClass public static void setUp() throws Exception { int randomNum = (int) (Math.random() * Integer.MAX_VALUE); hfilePath = "/tmp/hfile-" + randomNum; Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); keyValueMap = Maps.newHashMapWithExpectedSize(10000); errorKeys = Sets.newHashSetWithExpectedSize(2000); StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, new CacheConfig(conf), fs, 4096) .withFilePath(new Path(hfilePath)).withCompression(Compression.Algorithm.NONE).build(); // Add upto 10K values. for (int i = 0; i < 10000; i++) { byte[] key = String.format("%04d", i).getBytes(); byte[] value = null; // Add a couple of empty values for testing and making sure we return them. if (i <= 1) { value = "".getBytes(); } else { value = ("v" + (i + 1)).getBytes(); } KeyValue kv = new KeyValue(key, Bytes.toBytes("cf"), Bytes.toBytes(""), value); writer.append(kv); keyValueMap.put(ByteBuffer.wrap(key), ByteBuffer.wrap(value)); if (i >= 4000 && i < 6000) { errorKeys.add(ByteBuffer.wrap(key)); } } writer.close(); hfileReader = new TestHFileReader(fs, hfilePath, new CacheConfig(conf), new ExecutorServiceFuturePool(Executors.newFixedThreadPool(1)), errorKeys); } @AfterClass public static void tearDown() throws Exception { hfileReader.close(); FileUtils.forceDelete(new File(hfilePath)); } private void checkKeyValues(List<ByteBuffer> keyList, int expectedTotalCount, int expectedErrorKeyCount, int expectedNotFoundKeyCount) throws Throwable { Map<ByteBuffer, Pair<ByteBuffer, Throwable>> valuesMap = hfileReader.getValues(keyList).get(); int errorKeyCount = 0, notFoundKeyCount = 0; for (ByteBuffer key : keyList) { if (!valuesMap.containsKey(key)) { notFoundKeyCount++; continue; } ByteBuffer value = valuesMap.get(key).getLeft(); Throwable exception = valuesMap.get(key).getRight(); if (errorKeys.contains(key)) { assertNull(value); assertTrue(exception instanceof IOException); errorKeyCount++; } else { assertNull(exception); assertEquals(keyValueMap.get(key), value); } } assertEquals(expectedTotalCount, valuesMap.size()); assertEquals(expectedErrorKeyCount, errorKeyCount); assertEquals(expectedNotFoundKeyCount, notFoundKeyCount); } @Test public void testGetValues() throws Throwable { // Set up a null file set to start with. hfileReader.setFileSet(null); // Test a batch of all even values along with some lookups on non existent keys. List<ByteBuffer> keyList = Lists.newArrayListWithCapacity(5000); for (int i = 0; i < 10000; i += 2) { keyList.add(ByteBuffer.wrap(String.format("%04d", i).getBytes())); } // Add one not found key. keyList.add(ByteBuffer.wrap(String.format("%04d", 15000).getBytes())); checkKeyValues(keyList, 5000, 1000, 1); assertEquals(5001, Stats.getMetric("lookup-latency-ms").apply().count()); assertEquals(1000, Stats.getCounter("lookup-errors").apply()); assertEquals(1, Stats.getCounter("not-found").apply()); // Test a batch of all odd values along with some lookups on non existent keys. // This time, we use "test" as the file set to check if the metrics are recorded // correctly. hfileReader.setFileSet("test"); keyList = Lists.newArrayListWithCapacity(5000); for (int i = 1; i < 10000; i += 2) { keyList.add(ByteBuffer.wrap(String.format("%04d", i).getBytes())); } // Add one not found key. keyList.add(ByteBuffer.wrap(String.format("%04d", 16000).getBytes())); checkKeyValues(keyList, 5000, 1000, 1); assertEquals(10002, Stats.getMetric("lookup-latency-ms").apply().count()); assertEquals(5001, Stats.getMetric("test-lookup-latency-ms").apply().count()); assertEquals(8000, Stats.getMetric("value-size").apply().count()); assertEquals(4000, Stats.getMetric("test-value-size").apply().count()); assertEquals(2000, Stats.getCounter("lookup-errors").apply()); assertEquals(1000, Stats.getCounter("test-lookup-errors").apply()); assertEquals(2, Stats.getCounter("not-found").apply()); assertEquals(1, Stats.getCounter("test-not-found").apply()); } @Test public void testTerrapinPath() { String partName = TerrapinUtil.formatPartitionName(0); Path path = new HFileReader.TerrapinPath("/terrapin/data/meta_user_join/1234/" + partName); assertEquals(partName + "_meta_user_join_1234", path.getName()); } }