Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hdfs; import java.io.BufferedReader; import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; import java.util.Random; import java.util.zip.CRC32; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.hdfs.metrics.LookasideMetrics.LocalMetrics; import org.apache.log4j.Level; /** * This class tests the lookaside cache. */ public class TestLookasideCache extends junit.framework.TestCase { final static Log LOG = LogFactory.getLog("org.apache.hadoop.hdfs.TestLookasideCache"); final static String TEST_DIR = new File(System.getProperty("test.build.data", "build/lookasidecache/test/data")) .getAbsolutePath(); final static int NUM_DATANODES = 3; { ((Log4JLogger) LookasideCache.LOG).getLogger().setLevel(Level.ALL); ((Log4JLogger) LookasideCacheFileSystem.LOG).getLogger().setLevel(Level.ALL); } public void testCache() throws IOException { Configuration conf = new Configuration(); long maxSize = 5 * 1024; long evictPercent = 20; conf.setLong(LookasideCache.CACHESIZE, maxSize); conf.setLong(LookasideCache.CACHEEVICT_PERCENT, evictPercent); LookasideCache cache = new LookasideCache(conf); LocalMetrics metrics = LookasideCache.getLocalMetrics(); metrics.reset(); assertTrue(cache.getCacheMaxSize() == maxSize); assertTrue(cache.getCacheSize() == 0); assertTrue(cache.getCacheEvictPercent() == evictPercent); // insert five elements into the cache, each of size 1024 cache.addCache(new Path("one"), new Path("one"), 1024); cache.addCache(new Path("two"), new Path("two"), 1024); cache.addCache(new Path("three"), new Path("three"), 1024); cache.addCache(new Path("four"), new Path("four"), 1024); cache.addCache(new Path("five"), new Path("five"), 1024); assertTrue(cache.getCacheSize() == 5 * 1024); assertTrue(metrics.numAdd == 5); assertTrue(metrics.numAddNew == 5); assertTrue(metrics.numAddExisting == 0); // the cache is now full. If we add one more element, the oldest // two should be evicted cache.addCache(new Path("six"), new Path("six"), 512); assertTrue("cachesize is " + cache.getCacheSize(), cache.getCacheSize() == 3 * 1024 + 512); // verify that first two are not there. and the rest is there assertTrue(cache.getCache(new Path("one")) == null); assertTrue(cache.getCache(new Path("two")) == null); assertTrue(cache.getCache(new Path("three")) != null); assertTrue(cache.getCache(new Path("four")) != null); assertTrue(cache.getCache(new Path("five")) != null); assertTrue(cache.getCache(new Path("six")) != null); assertTrue(metrics.numEvict == 2); // make three the most recently used assertTrue(cache.getCache(new Path("three")) != null); assertTrue(metrics.numGetAttempts == 7); assertTrue(metrics.numGetHits == 5); // now we insert seven. cache.addCache(new Path("seven"), new Path("seven"), 512); assertTrue(cache.getCacheSize() == 4 * 1024); assertTrue(cache.getCache(new Path("one")) == null); assertTrue(cache.getCache(new Path("two")) == null); assertTrue(cache.getCache(new Path("three")) != null); assertTrue(cache.getCache(new Path("four")) != null); assertTrue(cache.getCache(new Path("five")) != null); assertTrue(cache.getCache(new Path("six")) != null); assertTrue(cache.getCache(new Path("seven")) != null); } public void testCacheFileSystem() throws IOException { // configure a cached filessytem, cache size is 10 KB. mySetup(10 * 1024L); try { // create a 5K file using the LookasideCache. This write // should be cached in the cache. Path file = new Path("/hdfs/testRead"); long crc = createTestFile(lfs, file, 1, 5, 1024L); FileStatus stat = lfs.getFileStatus(file); LOG.info("Created " + file + ", crc=" + crc + ", len=" + stat.getLen()); assertTrue(lfs.lookasideCache.getCacheSize() == 5 * 1024); // Test that readFully via the Lookasidecache fetches correct data // from the cache. FSDataInputStream stm = lfs.open(file); byte[] filebytes = new byte[(int) stat.getLen()]; stm.readFully(0, filebytes); assertEquals(crc, bufferCRC(filebytes)); stm.close(); // assert that there is one element of size 5K in the cache assertEquals(5 * 1024, lfs.lookasideCache.getCacheSize()); // create a 6K file using the LookasideCache. This is an // overwrite of the earlier file, so the cache should reflect // the new size of the file. crc = createTestFile(lfs, file, 1, 6, 1024L); stat = lfs.getFileStatus(file); LOG.info("Created " + file + ", crc=" + crc + ", len=" + stat.getLen()); // assert that there is one element of size 6K in the cache assertEquals(6 * 1024, lfs.lookasideCache.getCacheSize()); // verify reading file2 from the cache stm = lfs.open(file); filebytes = new byte[(int) stat.getLen()]; stm.readFully(0, filebytes); assertEquals(crc, bufferCRC(filebytes)); stm.close(); // add a 5 KB file to the cache. This should start eviction of // the earlier file. Path file2 = new Path("/hdfs/testRead2"); crc = createTestFile(lfs, file2, 1, 5, 1024L); stat = lfs.getFileStatus(file2); LOG.info("Created " + file2 + ", crc=" + crc + ", len=" + stat.getLen()); assertEquals(5 * 1024, lfs.lookasideCache.getCacheSize()); // move file2 to file3 Path file3 = new Path("/hdfs/testRead3"); assertTrue(lfs.rename(file2, file3)); // delete file3. This should clear out the cache. lfs.delete(file3, false); assertEquals(0, lfs.lookasideCache.getCacheSize()); } finally { myTearDown(); } } private MiniDFSCluster dfs; private FileSystem fileSys; private LookasideCacheFileSystem lfs; private String namenode; private String hftp; // setup a LookasideCachedFileSystem private void mySetup(long cacheSize) throws IOException { Configuration conf = new Configuration(); // create a HDFS cluster dfs = new MiniDFSCluster(conf, NUM_DATANODES, true, null); dfs.waitActive(); fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); FileSystem.setDefaultUri(conf, namenode); // create a client-side layered filesystem. // The cache size is 10 KB. lfs = getCachedHdfs(fileSys, conf, cacheSize); } private void myTearDown() throws IOException { if (dfs != null) { dfs.shutdown(); } } /** * Returns a cached filesystem layered on top of the HDFS cluster */ private LookasideCacheFileSystem getCachedHdfs(FileSystem fileSys, Configuration conf, long cacheSize) throws IOException { DistributedFileSystem dfs = (DistributedFileSystem) fileSys; Configuration clientConf = new Configuration(conf); clientConf.setLong(LookasideCache.CACHESIZE, cacheSize); clientConf.set("fs.lookasidecache.dir", TEST_DIR); clientConf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.LookasideCacheFileSystem"); clientConf.set("fs.lookasidecache.underlyingfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); URI dfsUri = dfs.getUri(); FileSystem.closeAll(); FileSystem lfs = FileSystem.get(dfsUri, clientConf); assertTrue("lfs not an instance of LookasideCacheFileSystem", lfs instanceof LookasideCacheFileSystem); return (LookasideCacheFileSystem) lfs; } // // creates a file and populate it with random data. Returns its crc. // private static long createTestFile(FileSystem fileSys, Path name, int repl, int numBlocks, long blocksize) throws IOException { CRC32 crc = new CRC32(); Random rand = new Random(); FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf().getInt("io.file.buffer.size", 4096), (short) repl, blocksize); // fill random data into file final byte[] b = new byte[(int) blocksize]; for (int i = 0; i < numBlocks; i++) { rand.nextBytes(b); stm.write(b); crc.update(b); } stm.close(); return crc.getValue(); } /** * returns the CRC32 of the buffer */ private long bufferCRC(byte[] buf) { CRC32 crc = new CRC32(); crc.update(buf, 0, buf.length); return crc.getValue(); } }