Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.raid; import java.io.File; import java.io.IOException; import java.util.Random; import java.util.zip.CRC32; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.RaidDFSUtil; import org.apache.hadoop.hdfs.TestRaidDfs; import junit.framework.TestCase; public class TestDirectoryRaidEncoder extends TestCase { final static String TEST_DIR = new File(System.getProperty("test.build.data", "build/contrib/raid/test/data")) .getAbsolutePath(); final static Log LOG = LogFactory.getLog("org.apache.hadoop.raid.TestDirectoryRaidEncoder"); final static int NUM_DATANODES = 3; static { ParityFilePair.disableCacheUsedInTestOnly(); } Configuration conf; String namenode = null; MiniDFSCluster dfs = null; FileSystem fileSys = null; private void mySetup() throws Exception { new File(TEST_DIR).mkdirs(); // Make sure data directory exists conf = new Configuration(); conf.setInt("raid.encoder.bufsize", 128); conf.setInt("raid.decoder.bufsize", 128); // scan all policies once every 5 second conf.setLong("raid.policy.rescan.interval", 5000); // Reduce run time for the test. conf.setInt("dfs.client.max.block.acquire.failures", 1); conf.setInt("dfs.client.baseTimeWindow.waitOn.BlockMissingException", 10); // do not use map-reduce cluster for Raiding conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); conf.set("raid.server.address", "localhost:0"); //Don't allow empty file to be raid conf.setLong(RaidNode.MINIMUM_RAIDABLE_FILESIZE_KEY, 1L); dfs = new MiniDFSCluster(conf, NUM_DATANODES, true, null); dfs.waitActive(); fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); TestDirectoryRaidDfs.setupStripeStore(conf, fileSys); FileSystem.setDefaultUri(conf, namenode); } private Codec loadTestCodecs(String erasureCode, int stripeLength, boolean isDirRaid) throws Exception { Utils.loadTestCodecs(conf, stripeLength, stripeLength, 1, 3, "/destraid", "/destraidrs", false, isDirRaid); return Codec.getCodec(erasureCode); } private void myTearDown() throws Exception { if (dfs != null) { dfs.shutdown(); } } private boolean doRaid(Configuration conf, FileSystem fileSys, Path srcPath, Codec codec) throws IOException { return RaidNode.doRaid(conf, fileSys.getFileStatus(srcPath), new Path(codec.parityDirectory), codec, new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE, false, 1, 1); } public void testAbnormalDirectory() throws Exception { mySetup(); Codec codec = loadTestCodecs("xor", 4, true); try { Path sourceDir = new Path("/user/raid"); Path parityFile = new Path("/destraid/user/raid"); assertTrue(fileSys.mkdirs(sourceDir)); LOG.info("Test non-leaf directory"); assertFalse("Couldn't raid non-leaf directory ", doRaid(conf, fileSys, sourceDir.getParent(), codec)); assertFalse(fileSys.exists(parityFile.getParent())); LOG.info("Test empty directory"); assertFalse("Couldn't raid empty directory ", doRaid(conf, fileSys, sourceDir, codec)); assertFalse(fileSys.exists(parityFile)); LOG.info("Test empty file in the directory"); Path emptyFile = new Path(sourceDir, "emptyFile"); TestRaidDfs.createTestFile(fileSys, emptyFile, 1, 0, 8192L); assertTrue(fileSys.exists(emptyFile)); assertFalse("No raidable files in the directory", doRaid(conf, fileSys, sourceDir, codec)); assertFalse(fileSys.exists(parityFile)); LOG.info("Test not enough blocks in the directory"); Path file1 = new Path(sourceDir, "file1"); Path file2 = new Path(sourceDir, "file2"); TestRaidDfs.createTestFile(fileSys, file1, 1, 1, 8192L); TestRaidDfs.createTestFile(fileSys, file2, 1, 1, 8192L); LOG.info("Created two files with two blocks in total"); assertTrue(fileSys.exists(file1)); assertTrue(fileSys.exists(file2)); assertFalse("Not enough blocks in the directory", doRaid(conf, fileSys, sourceDir, codec)); assertFalse(fileSys.exists(parityFile)); } finally { myTearDown(); } } private void validateSingleFile(String code, FileSystem fileSys, Path sourceDir, int stripeLength, int blockNum, boolean lastPartial) throws Exception { LOG.info( "Test file with " + blockNum + " blocks and " + (lastPartial ? "partial" : "full") + " last block"); Codec codec = loadTestCodecs(code, stripeLength, true); Path parityDir = new Path(codec.parityDirectory); RaidDFSUtil.cleanUp(fileSys, sourceDir); RaidDFSUtil.cleanUp(fileSys, parityDir); fileSys.mkdirs(sourceDir); Path file1 = new Path(sourceDir, "file1"); if (!lastPartial) { TestRaidDfs.createTestFile(fileSys, file1, 2, blockNum, 8192L); } else { TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1, 2, blockNum, 8192L); } Path parityFile = RaidNode.getOriginalParityFile(parityDir, sourceDir); // Do directory level raid LOG.info("Create a directory-raid parity file " + parityFile); assertTrue("Cannot raid directory " + sourceDir, doRaid(conf, fileSys, sourceDir, codec)); assertEquals("Modification time should be the same", fileSys.getFileStatus(sourceDir).getModificationTime(), fileSys.getFileStatus(parityFile).getModificationTime()); assertEquals("Replica num of source file should be reduced to 1", fileSys.getFileStatus(file1).getReplication(), 1); assertEquals("Replica num of parity file should be reduced to 1", fileSys.getFileStatus(parityFile).getReplication(), 1); long dirCRC = RaidDFSUtil.getCRC(fileSys, parityFile); long dirLen = fileSys.getFileStatus(parityFile).getLen(); // remove the parity dir RaidDFSUtil.cleanUp(fileSys, parityDir); codec = loadTestCodecs(code, stripeLength, false); Path parityFile1 = RaidNode.getOriginalParityFile(parityDir, file1); LOG.info("Create a file-raid parity file " + parityFile1); assertTrue("Cannot raid file " + file1, doRaid(conf, fileSys, file1, codec)); assertTrue("Parity file doesn't match when the file has " + blockNum + " blocks ", TestRaidDfs.validateFile(fileSys, parityFile1, dirLen, dirCRC)); } public void testOneFileDirectory() throws Exception { mySetup(); int stripeLength = 4; try { for (String code : RaidDFSUtil.codes) { LOG.info("testOneFileDirectory: Test code " + code); Codec codec = loadTestCodecs(code, stripeLength, true); Path sourceDir = new Path("/user/raid", code); assertTrue(fileSys.mkdirs(sourceDir)); Path twoBlockFile = new Path(sourceDir, "twoBlockFile"); ; LOG.info("Test one file with 2 blocks"); TestRaidDfs.createTestFile(fileSys, twoBlockFile, 2, 2, 8192L); assertTrue(fileSys.exists(twoBlockFile)); assertFalse("Not enough blocks in the directory", RaidNode.doRaid(conf, fileSys.getFileStatus(sourceDir), new Path(codec.parityDirectory), codec, new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE, false, 1, 1)); fileSys.delete(twoBlockFile, true); LOG.info("Test one file with blocks less than one stripe"); validateSingleFile(code, fileSys, sourceDir, stripeLength, 3, false); validateSingleFile(code, fileSys, sourceDir, stripeLength, 3, true); LOG.info("Test one file with one stripe blocks"); validateSingleFile(code, fileSys, sourceDir, stripeLength, stripeLength, false); validateSingleFile(code, fileSys, sourceDir, stripeLength, stripeLength, true); LOG.info("Test one file with more than one stripe blocks"); validateSingleFile(code, fileSys, sourceDir, stripeLength, stripeLength + 2, false); validateSingleFile(code, fileSys, sourceDir, stripeLength, stripeLength + 2, true); } } finally { myTearDown(); } } private void validateMultipleFiles(String code, FileSystem fileSys, Path sourceDir, int stripeLength, long[] fileSizes, long blockSize, long singleFileBlockSize) throws Exception { long[] blockSizes = new long[fileSizes.length]; for (int i = 0; i < fileSizes.length; i++) blockSizes[i] = blockSize; validateMultipleFiles(code, fileSys, sourceDir, stripeLength, fileSizes, blockSizes, singleFileBlockSize); } // // creates a file by grouping multiple files together // Returns its crc. // private long createDirectoryFile(FileSystem fileSys, Path name, int repl, long[] fileSizes, long[] blockSizes, int[] seeds, long blockSize) throws IOException { CRC32 crc = new CRC32(); assert fileSizes.length == blockSizes.length; assert fileSizes.length == seeds.length; FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf().getInt("io.file.buffer.size", 4096), (short) repl, blockSize); byte[] zeros = new byte[(int) (blockSize)]; for (int j = 0; j < zeros.length; j++) { zeros[j] = 0; } // fill random data into file for (int i = 0; i < fileSizes.length; i++) { assert blockSizes[i] <= blockSize; byte[] b = new byte[(int) blockSizes[i]]; long numBlocks = fileSizes[i] / blockSizes[i]; Random rand = new Random(seeds[i]); for (int j = 0; j < numBlocks; j++) { rand.nextBytes(b); stm.write(b); crc.update(b); int zeroLen = (int) (blockSize - blockSizes[i]); stm.write(zeros, 0, zeroLen); crc.update(zeros, 0, zeroLen); } long lastBlock = fileSizes[i] - numBlocks * blockSizes[i]; if (lastBlock > 0) { b = new byte[(int) lastBlock]; rand.nextBytes(b); stm.write(b); crc.update(b); if (i + 1 < fileSizes.length) { // Not last block of file, write zero int zeroLen = (int) (blockSize - lastBlock); stm.write(zeros, 0, zeroLen); crc.update(zeros, 0, zeroLen); } } } stm.close(); return crc.getValue(); } private void printFileCRC(FileSystem fs, Path file, long bufferSize) throws IOException { byte[] buffer = new byte[(int) bufferSize]; FSDataInputStream stm = fs.open(file); StringBuilder sb = new StringBuilder(); sb.append("CRC for file: " + file + " size " + fs.getFileStatus(file).getLen() + "\n"); while (stm.read(buffer) >= 0) { CRC32 crc = new CRC32(); crc.update(buffer); sb.append(" " + crc.getValue()); } sb.append("\n"); System.out.println(sb.toString()); stm.close(); } private void validateMultipleFiles(String code, FileSystem fileSys, Path sourceDir, int stripeLength, long[] fileSizes, long[] blockSizes, long blockSize) throws Exception { assert fileSizes.length == blockSizes.length; Codec codec = loadTestCodecs(code, stripeLength, true); Path parityDir = new Path(codec.parityDirectory); RaidDFSUtil.cleanUp(fileSys, sourceDir); RaidDFSUtil.cleanUp(fileSys, parityDir); fileSys.mkdirs(sourceDir); LOG.info("Create files under directory " + sourceDir); Random rand = new Random(); int[] seeds = new int[fileSizes.length]; for (int i = 0; i < fileSizes.length; i++) { Path file = new Path(sourceDir, "file" + i); seeds[i] = rand.nextInt(); TestRaidDfs.createTestFile(fileSys, file, 2, fileSizes[i], blockSizes[i], seeds[i]); } Path parityFile = RaidNode.getOriginalParityFile(parityDir, sourceDir); // Do directory level raid LOG.info("Create a directory-raid parity file " + parityFile); assertTrue("Cannot raid directory " + sourceDir, doRaid(conf, fileSys, sourceDir, codec)); this.printFileCRC(fileSys, parityFile, blockSize); long dirCRC = RaidDFSUtil.getCRC(fileSys, parityFile); long dirLen = fileSys.getFileStatus(parityFile).getLen(); assertEquals("Modification time should be the same", fileSys.getFileStatus(sourceDir).getModificationTime(), fileSys.getFileStatus(parityFile).getModificationTime()); assertEquals("Replica num of parity file should be reduced to 1", fileSys.getFileStatus(parityFile).getReplication(), 1); for (int i = 0; i < fileSizes.length; i++) { Path file = new Path(sourceDir, "file" + i); assertEquals("Replica num of source file should be reduced to 1", fileSys.getFileStatus(file).getReplication(), 1); } // remove the source dir and parity dir RaidDFSUtil.cleanUp(fileSys, sourceDir); RaidDFSUtil.cleanUp(fileSys, parityDir); fileSys.mkdirs(sourceDir); codec = loadTestCodecs(code, stripeLength, false); Path file1 = new Path(sourceDir, "file1"); Path parityFile1 = RaidNode.getOriginalParityFile(parityDir, file1); LOG.info("Create a source file " + file1); this.createDirectoryFile(fileSys, file1, 1, fileSizes, blockSizes, seeds, blockSize); LOG.info("Create a file-raid parity file " + parityFile1); assertTrue("Cannot raid file " + file1, doRaid(conf, fileSys, file1, codec)); this.printFileCRC(fileSys, parityFile1, blockSize); assertTrue("Parity file doesn't match", TestRaidDfs.validateFile(fileSys, parityFile1, dirLen, dirCRC)); } public void testSmallFileDirectory() throws Exception { mySetup(); int stripeLength = 4; long blockSize = 8192L; try { for (String code : RaidDFSUtil.codes) { LOG.info("testSmallFileDirectory: Test code " + code); Path sourceDir = new Path("/user/raid"); validateMultipleFiles(code, fileSys, sourceDir, stripeLength, new long[] { 1000L, 4000L, 1000L }, blockSize, 4096L); validateMultipleFiles(code, fileSys, sourceDir, stripeLength, new long[] { 2000L, 3000L, 2000L, 3000L }, blockSize, 3072L); validateMultipleFiles(code, fileSys, sourceDir, stripeLength, new long[] { 3000L, 3000L, 3000L, 3000L }, blockSize, 3072L); validateMultipleFiles(code, fileSys, sourceDir, stripeLength, new long[] { 511L, 3584L, 3000L, 1234L, 512L, 1234L, 3000L, 3234L, 511L }, blockSize, 3584L); } } finally { myTearDown(); } } public void testIdenticalBlockSizeFileDirectory() throws Exception { mySetup(); int stripeLength = 4; long blockSize = 8192L; try { for (String code : RaidDFSUtil.codes) { LOG.info("testIdenticalBlockSizeFileDirectory: Test code " + code); Path sourceDir = new Path("/user/raid"); validateMultipleFiles(code, fileSys, sourceDir, stripeLength, new long[] { 1000L, blockSize, 2 * blockSize, 4000L }, blockSize, blockSize); validateMultipleFiles(code, fileSys, sourceDir, stripeLength, new long[] { blockSize, 2 * blockSize, 3 * blockSize, 4 * blockSize }, blockSize, blockSize); int halfBlock = (int) blockSize / 2; validateMultipleFiles( code, fileSys, sourceDir, stripeLength, new long[] { blockSize + halfBlock, 2 * blockSize + halfBlock, 3 * blockSize + halfBlock, 4 * blockSize + halfBlock }, blockSize, blockSize); validateMultipleFiles(code, fileSys, sourceDir, stripeLength, new long[] { blockSize + 1, 9 * blockSize + 1, 2 * blockSize + 1, 3 * blockSize + 1 }, blockSize, blockSize); } } finally { myTearDown(); } } public void testDifferentBlockSizeFileDirectory() throws Exception { mySetup(); int stripeLength = 3; long blockSize = 8192L; try { for (String code : RaidDFSUtil.codes) { LOG.info("testDifferentBlockSizeFileDirectory: Test code " + code); Path sourceDir = new Path("/user/raid"); validateMultipleFiles(code, fileSys, sourceDir, stripeLength, new long[] { 1000, blockSize, 2 * blockSize, 2 * blockSize + 1 }, new long[] { blockSize, blockSize, 2 * blockSize, blockSize }, 2 * blockSize); validateMultipleFiles(code, fileSys, sourceDir, stripeLength, new long[] { blockSize, 2 * blockSize, 3 * blockSize, 4 * blockSize }, new long[] { blockSize, 2 * blockSize, 3 * blockSize, blockSize }, 3 * blockSize); validateMultipleFiles(code, fileSys, sourceDir, stripeLength, new long[] { blockSize + 1, 9 * blockSize + 1, 2 * blockSize + 1, blockSize + 1 }, new long[] { blockSize, 2 * blockSize, 3 * blockSize, blockSize }, 2 * blockSize + 512); } } finally { myTearDown(); } } }