Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hdfs; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.EOFException; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; import java.net.URI; import java.util.Random; import java.util.zip.CRC32; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.log4j.Level; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.raid.Codec; import org.apache.hadoop.raid.ParityFilePair; import org.apache.hadoop.raid.RaidNode; import org.apache.hadoop.raid.RaidUtils; import org.apache.hadoop.raid.TestDirectoryRaidDfs; import org.apache.hadoop.raid.Utils; import org.apache.hadoop.util.StringUtils; import org.junit.Test; public class TestRaidDfs { final static String TEST_DIR = new File(System.getProperty("test.build.data", "build/contrib/raid/test/data")) .getAbsolutePath(); final static long RELOAD_INTERVAL = 1000; final static Log LOG = LogFactory.getLog("org.apache.hadoop.raid.TestRaidDfs"); final static int NUM_DATANODES = 3; static { ParityFilePair.disableCacheUsedInTestOnly(); } Configuration conf; String namenode = null; String hftp = null; MiniDFSCluster dfs = null; FileSystem fileSys = null; String jobTrackerName = null; Codec codec; int stripeLength; private void mySetup(String erasureCode, int rsParityLength) throws Exception { mySetup(erasureCode, rsParityLength, false, false); } private void mySetup(String erasureCode, int rsParityLength, boolean isDirRaid) throws Exception { mySetup(erasureCode, rsParityLength, isDirRaid, false); } private void mySetup(String erasureCode, int rsParityLength, boolean isDirRaid, boolean specialBufSize) throws Exception { new File(TEST_DIR).mkdirs(); // Make sure data directory exists conf = new Configuration(); if (!specialBufSize) { conf.setInt("raid.encoder.bufsize", 128); conf.setInt("raid.decoder.bufsize", 128); } else { // set buffer size to be not dividable conf.setInt("raid.encoder.bufsize", 127); conf.setInt("raid.decoder.bufsize", 127); conf.setInt("raid.encoder.max.buffer.size", 7); conf.setInt("raid.decoder.max.buffer.size", 7); } Utils.loadTestCodecs(conf, stripeLength, stripeLength, 1, rsParityLength, "/destraid", "/destraidrs", false, isDirRaid); codec = Codec.getCodec(erasureCode); // scan all policies once every 5 second conf.setLong("raid.policy.rescan.interval", 5000); // Reduce run time for the test. conf.setInt("dfs.client.max.block.acquire.failures", 1); conf.setInt("dfs.client.baseTimeWindow.waitOn.BlockMissingException", 10); // do not use map-reduce cluster for Raiding conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode"); conf.set("raid.server.address", "localhost:0"); dfs = new MiniDFSCluster(conf, NUM_DATANODES, true, null); dfs.waitActive(); fileSys = dfs.getFileSystem(); TestDirectoryRaidDfs.setupStripeStore(conf, fileSys); namenode = fileSys.getUri().toString(); hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort(); FileSystem.setDefaultUri(conf, namenode); } private void myTearDown() throws Exception { if (dfs != null) { dfs.shutdown(); } } private LocatedBlocks getBlockLocations(Path file, long length) throws IOException { DistributedFileSystem dfs = (DistributedFileSystem) fileSys; return RaidDFSUtil.getBlockLocations(dfs, file.toUri().getPath(), 0, length); } private LocatedBlocks getBlockLocations(Path file) throws IOException { FileStatus stat = fileSys.getFileStatus(file); return getBlockLocations(file, stat.getLen()); } private DistributedRaidFileSystem getRaidFS() throws IOException { DistributedFileSystem dfs = (DistributedFileSystem) fileSys; Configuration clientConf = new Configuration(conf); clientConf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedRaidFileSystem"); clientConf.set("fs.raid.underlyingfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); clientConf.setBoolean("fs.hdfs.impl.disable.cache", true); URI dfsUri = dfs.getUri(); return (DistributedRaidFileSystem) FileSystem.get(dfsUri, clientConf); } public static void waitForFileRaided(Log logger, FileSystem fileSys, Path file, Path destPath) throws IOException, InterruptedException { waitForFileRaided(logger, fileSys, file, destPath, (short) 1); } public static void waitForReplicasReduction(FileSystem fileSys, Path srcPath, short targetReplication) throws IOException, InterruptedException { // Make sure datanode report block deletion fast enough assertEquals(fileSys.getConf().getLong("dfs.blockreport.intervalMsec", FSConstants.BLOCKREPORT_INTERVAL), 8000); FileStatus fstat = fileSys.getFileStatus(srcPath); FileStatus[] listStats = null; if (fstat.isDir()) { listStats = fileSys.listStatus(srcPath); } else { listStats = new FileStatus[] { fstat }; } long start; for (FileStatus stat : listStats) { assertEquals("File " + stat.getPath() + " current repl: " + stat.getReplication() + " expected: " + targetReplication, stat.getReplication(), targetReplication); } for (FileStatus stat : listStats) { start = System.currentTimeMillis(); boolean reduced = false; while (System.currentTimeMillis() - start < 60000 && !reduced) { BlockLocation[] bls = fileSys.getFileBlockLocations(stat, 0L, stat.getLen()); reduced = true; for (BlockLocation bl : bls) { if (bl.getHosts().length > targetReplication) { reduced = false; break; } } if (!reduced) Thread.sleep(1000); } assertTrue("Replicas of " + stat.getPath() + " are more than " + targetReplication, reduced); } } public static void waitForFileRaided(Log logger, FileSystem fileSys, Path file, Path destPath, short targetReplication) throws IOException, InterruptedException { FileStatus parityStat = null; String fileName = file.getName().toString(); long startTime = System.currentTimeMillis(); // wait till file is raided while (parityStat == null && System.currentTimeMillis() - startTime < 120000) { logger.info("Waiting for files to be raided."); try { FileStatus[] listPaths = fileSys.listStatus(destPath); if (listPaths != null) { for (FileStatus f : listPaths) { logger.info("File raided so far : " + f.getPath()); String found = f.getPath().getName().toString(); if (fileName.equals(found)) { parityStat = f; break; } } } } catch (FileNotFoundException e) { //ignore } Thread.sleep(1000); // keep waiting } while (true) { LocatedBlocks locations = null; DistributedFileSystem dfs = (DistributedFileSystem) fileSys; locations = RaidDFSUtil.getBlockLocations(dfs, file.toUri().getPath(), 0, parityStat.getLen()); if (!locations.isUnderConstruction()) { break; } Thread.sleep(1000); } while (true) { FileStatus stat = fileSys.getFileStatus(file); if (stat.getReplication() == targetReplication) break; Thread.sleep(1000); } } public static void waitForDirRaided(Log logger, FileSystem fileSys, Path file, Path destPath) throws IOException, InterruptedException { waitForDirRaided(logger, fileSys, file, destPath, (short) 1); } public static void waitForDirRaided(Log logger, FileSystem fileSys, Path file, Path destPath, short targetReplication) throws IOException, InterruptedException { waitForDirRaided(logger, fileSys, file, destPath, targetReplication, 90000); } public static void waitForDirRaided(Log logger, FileSystem fileSys, Path file, Path destPath, short targetReplication, long waitMillis) throws IOException, InterruptedException { FileStatus parityStat = null; String fileName = file.getName().toString(); long startTime = System.currentTimeMillis(); FileStatus srcStat = fileSys.getFileStatus(file); // wait till file is raided while (parityStat == null && System.currentTimeMillis() - startTime < waitMillis) { logger.info("Waiting for files to be raided."); try { FileStatus[] listPaths = fileSys.listStatus(destPath); if (listPaths != null) { for (FileStatus f : listPaths) { logger.info("File raided so far : " + f.getPath()); String found = f.getPath().getName().toString(); if (fileName.equals(found) && srcStat.getModificationTime() == f.getModificationTime()) { parityStat = f; break; } } } } catch (FileNotFoundException e) { //ignore } Thread.sleep(1000); // keep waiting } assertTrue("Parity file is not generated", parityStat != null); assertEquals(srcStat.getModificationTime(), parityStat.getModificationTime()); for (FileStatus stat : fileSys.listStatus(file)) { assertEquals(stat.getReplication(), targetReplication); } } private void corruptBlockAndValidate(Path srcFile, Path destPath, int[] listBlockNumToCorrupt, long blockSize, int numBlocks, MiniDFSCluster cluster) throws IOException, InterruptedException { RaidDFSUtil.cleanUp(fileSys, srcFile.getParent()); fileSys.mkdirs(srcFile.getParent()); int repl = 1; long crc = createTestFilePartialLastBlock(fileSys, srcFile, repl, numBlocks, blockSize); long length = fileSys.getFileStatus(srcFile).getLen(); if (codec.isDirRaid) { RaidNode.doRaid(conf, fileSys.getFileStatus(srcFile.getParent()), destPath, codec, new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE, false, repl, repl); } else { RaidNode.doRaid(conf, fileSys.getFileStatus(srcFile), destPath, codec, new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE, false, repl, repl); } // Delete first block of file for (int blockNumToCorrupt : listBlockNumToCorrupt) { LOG.info("Corrupt block " + blockNumToCorrupt + " of file " + srcFile); LocatedBlocks locations = getBlockLocations(srcFile); corruptBlock(srcFile, locations.get(blockNumToCorrupt).getBlock(), NUM_DATANODES, true, cluster); } // Validate DistributedRaidFileSystem raidfs = getRaidFS(); assertTrue(validateFile(raidfs, srcFile, length, crc)); } @Test public void testRaidDfsXorSpecialBufferSize() throws Exception { testRaidDfsXorCore(false, true); } @Test public void testRaidDfsDirXorSpecialBufferSize() throws Exception { testRaidDfsXorCore(true, true); } @Test public void testRaidDfsRsSpecialBufferSize() throws Exception { testRaidDfsRsCore(false, true); } @Test public void testRaidDfsDirRsSpecialBufferSize() throws Exception { testRaidDfsRsCore(true, true); } /** * Create a file, corrupt several blocks in it and ensure that the file can be * read through DistributedRaidFileSystem by ReedSolomon coding. */ private void testRaidDfsRsCore(boolean isDirRaid, boolean specialBufSize) throws Exception { LOG.info("Test testRaidDfs started"); long blockSize = 8192L; int numBlocks = 8; stripeLength = 3; mySetup("rs", 3, isDirRaid, specialBufSize); int[][] corrupt = { { 1, 2, 3 }, { 1, 4, 7 }, { 3, 6, 7 } }; try { for (int i = 0; i < corrupt.length; i++) { Path file = new Path("/user/dhruba/raidtest/file" + i); corruptBlockAndValidate(file, new Path("/destraidrs"), corrupt[i], blockSize, numBlocks, dfs); } } catch (Exception e) { LOG.info("testRaidDfs Exception " + e + StringUtils.stringifyException(e)); throw e; } finally { myTearDown(); } LOG.info("Test testRaidDfs completed."); } @Test public void testRaidDfsRs() throws Exception { testRaidDfsRsCore(false, false); } @Test public void testRaidDfsDirRs() throws Exception { testRaidDfsRsCore(true, false); } /** * Test DistributedRaidFileSystem with relative path */ @Test public void testRelativePath() throws Exception { stripeLength = 3; mySetup("xor", 1); try { DistributedRaidFileSystem raidfs = getRaidFS(); Path file = new Path(raidfs.getHomeDirectory(), "raidtest/file1"); Path file1 = new Path("raidtest/file1"); long crc = createTestFile(raidfs.getFileSystem(), file, 1, 8, 8192L); FileStatus stat = fileSys.getFileStatus(file); LOG.info("Created " + file + ", crc=" + crc + ", len=" + stat.getLen()); byte[] filebytes = new byte[(int) stat.getLen()]; // Test that readFully returns the correct CRC when there are no errors. FSDataInputStream stm = raidfs.open(file); stm.readFully(0, filebytes); assertEquals(crc, bufferCRC(filebytes)); stm.close(); stm = raidfs.open(file1); stm.readFully(0, filebytes); assertEquals(crc, bufferCRC(filebytes)); stm.close(); } finally { myTearDown(); } } /** * Test DistributedRaidFileSystem.readFully() */ private void testReadFullyCore(boolean isDirRaid) throws Exception { stripeLength = 3; mySetup("xor", 1, isDirRaid); try { Path file = new Path("/user/raid/raidtest/file1"); long crc = createTestFile(fileSys, file, 1, 8, 8192L); FileStatus stat = fileSys.getFileStatus(file); LOG.info("Created " + file + ", crc=" + crc + ", len=" + stat.getLen()); byte[] filebytes = new byte[(int) stat.getLen()]; // Test that readFully returns the correct CRC when there are no errors. DistributedRaidFileSystem raidfs = getRaidFS(); FSDataInputStream stm = raidfs.open(file); stm.readFully(0, filebytes); assertEquals(crc, bufferCRC(filebytes)); stm.close(); // Generate parity. if (isDirRaid) { RaidNode.doRaid(conf, fileSys.getFileStatus(file.getParent()), new Path("/destraid"), codec, new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE, false, 1, 1); } else { RaidNode.doRaid(conf, fileSys.getFileStatus(file), new Path("/destraid"), codec, new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE, false, 1, 1); } int[] corrupt = { 0, 4, 7 }; // first, last and middle block for (int blockIdx : corrupt) { LOG.info("Corrupt block " + blockIdx + " of file " + file); LocatedBlocks locations = getBlockLocations(file); removeAndReportBlock((DistributedFileSystem) fileSys, file, locations.get(blockIdx), dfs); } // Test that readFully returns the correct CRC when there are errors. stm = raidfs.open(file); stm.readFully(0, filebytes); assertEquals(crc, bufferCRC(filebytes)); } finally { myTearDown(); } } @Test public void testReadFully() throws Exception { testReadFullyCore(false); } @Test public void testDirReadFully() throws Exception { testReadFullyCore(true); } @Test public void testSeek() throws Exception { stripeLength = 3; mySetup("xor", 1); try { Path file = new Path("/user/raid/raidtest/file1"); long crc = createTestFile(fileSys, file, 1, 8, 8192L); FileStatus stat = fileSys.getFileStatus(file); LOG.info("Created " + file + ", crc=" + crc + ", len=" + stat.getLen()); // Test that readFully returns the correct CRC when there are no errors. DistributedRaidFileSystem raidfs = getRaidFS(); FSDataInputStream stm = raidfs.open(file); //Test end of file LOG.info("Seek to " + (stat.getLen() - 1) + ", len=" + stat.getLen()); stm.seek(stat.getLen() - 1); assertEquals(stat.getLen() - 1, stm.getPos()); LOG.info("Seek to " + stat.getLen() + ", len=" + stat.getLen()); stm.seek(stat.getLen()); assertEquals(stat.getLen(), stm.getPos()); // Should work. LOG.info("Seek to " + (stat.getLen() / 2) + ", len=" + stat.getLen()); stm.seek(stat.getLen() / 2); assertEquals(stat.getLen() / 2, stm.getPos()); // Should work. LOG.info("Seek to " + stat.getLen() + ", len=" + stat.getLen()); stm.seek(stat.getLen()); assertEquals(stat.getLen(), stm.getPos()); LOG.info("Seek to " + (stat.getLen() + 1) + ", len=" + stat.getLen()); boolean expectedExceptionThrown = false; try { stm.seek(stat.getLen() + 1); } catch (EOFException e) { expectedExceptionThrown = true; } assertTrue(expectedExceptionThrown); } finally { myTearDown(); } } /** * Create a file, corrupt a block in it and ensure that the file can be * read through DistributedRaidFileSystem by XOR code. * @specialBufSize test raiding files with block size not dividable by * buffer size */ private void testRaidDfsXorCore(boolean isDirRaid, boolean specialBufSize) throws Exception { LOG.info("Test testRaidDfs started."); long blockSize = 8192L; int numBlocks = 8; stripeLength = 3; mySetup("xor", 1, isDirRaid, specialBufSize); // heavy test!! try to corrupt every block to test the partial reading int[][] corrupt = { { 0 }, { 1 }, { 2 }, { 3 }, { 4 }, { 5 }, { 6 }, { 7 } }; try { for (int i = 0; i < corrupt.length; i++) { Path file = new Path("/user/dhruba/raidtest/xor" + i); corruptBlockAndValidate(file, new Path("/destraid"), corrupt[i], blockSize, numBlocks, dfs); } } catch (Exception e) { LOG.info("testRaidDfs Exception " + e + StringUtils.stringifyException(e)); throw e; } finally { myTearDown(); } LOG.info("Test testRaidDfs completed."); } @Test public void testRaidDfsXor() throws Exception { testRaidDfsXorCore(false, false); } @Test public void testRaidDfsDirXor() throws Exception { testRaidDfsXorCore(true, false); } @Test public void testRead() throws Exception { mySetup("xor", 1); try { Path p = new Path("/tmp/testRead." + System.currentTimeMillis()); long crc = createTestFile(fileSys, p, 1, 8, 8192L); FileStatus stat = fileSys.getFileStatus(p); LOG.info("Created " + p + ", crc=" + crc + ", len=" + stat.getLen()); // Test that readFully returns the correct CRC when there are no errors. DistributedRaidFileSystem raidfs = getRaidFS(); FSDataInputStream stm = raidfs.open(p); // Should be able to read all bytes. for (int i = 0; i < stat.getLen(); i++) { int val = stm.read(); if (val < 0) { LOG.error("Invalid value " + val + " at index " + i); } assertTrue(val >= 0); } stm.close(); // One more read should return -1 to indicate EOF. assertEquals(-1, stm.read()); } finally { myTearDown(); } } @Test public void testZeroLengthFile() throws Exception { mySetup("xor", 1); try { Path p = new Path("/tmp/testZero." + System.currentTimeMillis()); fileSys.create(p).close(); FileStatus stat = fileSys.getFileStatus(p); LOG.info("Created " + p + ", len=" + stat.getLen()); // Test that readFully returns the correct CRC when there are no errors. DistributedRaidFileSystem raidfs = getRaidFS(); FSDataInputStream stm = raidfs.open(p); assertEquals(-1, stm.read()); } finally { myTearDown(); } } private void testTooManyErrorsDecodeXORCore(boolean isDirRaid) throws Exception { long blockSize = 8192L; int numBlocks = 8; stripeLength = 3; mySetup("xor", 1, isDirRaid); try { int[] corrupt = { 0, 1 }; // Two blocks in the same stripe is too much. Path file = new Path("/user/dhruba/raidtest/file1"); boolean expectedExceptionThrown = false; try { corruptBlockAndValidate(file, new Path("/destraid"), corrupt, blockSize, numBlocks, dfs); // Should not reach. } catch (IOException e) { LOG.info("Expected exception caught" + e); expectedExceptionThrown = true; } assertTrue(expectedExceptionThrown); } finally { myTearDown(); } } @Test public void testTooManyErrorsDecodeXOR() throws Exception { testTooManyErrorsDecodeXORCore(false); } @Test public void testTooManyErrorsDecodeDirXOR() throws Exception { testTooManyErrorsDecodeXORCore(true); } private void testTooManyErrorsDecodeRSCore(boolean isDirRaid) throws Exception { long blockSize = 8192L; int numBlocks = 8; stripeLength = 3; mySetup("rs", 1, isDirRaid); try { int[] corrupt = { 0, 1 }; // Two blocks in the same stripe is too much. Path file = new Path("/user/dhruba/raidtest/file2"); boolean expectedExceptionThrown = false; try { corruptBlockAndValidate(file, new Path("/destraidrs"), corrupt, blockSize, numBlocks, dfs); } catch (IOException e) { LOG.info("Expected exception caught" + e); expectedExceptionThrown = true; } assertTrue(expectedExceptionThrown); } finally { myTearDown(); } } @Test public void testTooManyErrorsDecodeRS() throws Exception { testTooManyErrorsDecodeRSCore(false); } @Test public void testTooManyErrorsDecodeDirRS() throws Exception { testTooManyErrorsDecodeRSCore(true); } private void testTooManyErrorsEncodeCore(boolean isDirRaid) throws Exception { long blockSize = 8192L; int numBlocks = 8; stripeLength = 3; mySetup("xor", 1, isDirRaid); // Encoding with XOR should fail when even one block is corrupt. try { Path destPath = new Path("/destraid/user/dhruba/raidtest"); Path file = new Path("/user/dhruba/raidtest/file1"); int repl = 1; createTestFilePartialLastBlock(fileSys, file, repl, numBlocks, blockSize); int blockNumToCorrupt = 0; LOG.info("Corrupt block " + blockNumToCorrupt + " of file " + file); LocatedBlocks locations = getBlockLocations(file); removeAndReportBlock((DistributedFileSystem) fileSys, file, locations.get(blockNumToCorrupt), dfs); boolean expectedExceptionThrown = false; try { if (isDirRaid) { RaidNode.doRaid(conf, fileSys.getFileStatus(file.getParent()), destPath, codec, new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE, false, repl, repl); } else { RaidNode.doRaid(conf, fileSys.getFileStatus(file), destPath, codec, new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE, false, repl, repl); } } catch (IOException e) { LOG.info("Expected exception caught" + e); expectedExceptionThrown = true; } assertTrue(expectedExceptionThrown); } finally { myTearDown(); } } @Test public void testTooManyErrorsEncode() throws Exception { testTooManyErrorsEncodeCore(false); } @Test public void testTooManyErrorsDirEncode() throws Exception { testTooManyErrorsEncodeCore(true); } private void testTooManyErrorsEncodeRSCore(boolean isDirRaid) throws Exception { long blockSize = 8192L; int numBlocks = 8; stripeLength = 3; mySetup("rs", 1, isDirRaid); // Encoding with RS should fail when even one block is corrupt. try { Path destPath = new Path("/destraidrs/user/dhruba/raidtest"); Path file = new Path("/user/dhruba/raidtest/file2"); int repl = 1; createTestFilePartialLastBlock(fileSys, file, repl, numBlocks, blockSize); int blockNumToCorrupt = 0; LOG.info("Corrupt block " + blockNumToCorrupt + " of file " + file); LocatedBlocks locations = getBlockLocations(file); removeAndReportBlock((DistributedFileSystem) fileSys, file, locations.get(blockNumToCorrupt), dfs); boolean expectedExceptionThrown = false; try { if (isDirRaid) { RaidNode.doRaid(conf, fileSys.getFileStatus(file.getParent()), destPath, codec, new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE, false, repl, repl); } else { RaidNode.doRaid(conf, fileSys.getFileStatus(file), destPath, codec, new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE, false, repl, repl); } } catch (IOException e) { expectedExceptionThrown = true; LOG.info("Expected exception caught" + e); } assertTrue(expectedExceptionThrown); } finally { myTearDown(); } } @Test public void testTooManyErrorsEncodeRS() throws Exception { testTooManyErrorsEncodeRSCore(false); } @Test public void testTooManyErrorsEncodeDirRS() throws Exception { testTooManyErrorsEncodeRSCore(true); } // // creates a file and populate it with random data. Returns its crc. // public static long createTestFile(FileSystem fileSys, Path name, int repl, int numBlocks, long blocksize) throws IOException { CRC32 crc = new CRC32(); Random rand = new Random(); FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf().getInt("io.file.buffer.size", 4096), (short) repl, blocksize); // fill random data into file final byte[] b = new byte[(int) blocksize]; for (int i = 0; i < numBlocks; i++) { rand.nextBytes(b); stm.write(b); crc.update(b); } stm.close(); return crc.getValue(); } // // creates a file given a specific file size and it with random data. // Returns its crc. // public static long createTestFile(FileSystem fileSys, Path name, int repl, long fileSize, long blockSize, int seed) throws IOException { CRC32 crc = new CRC32(); Random rand = new Random(seed); FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf().getInt("io.file.buffer.size", 4096), (short) repl, blockSize); LOG.info("create file " + name + " size: " + fileSize + " blockSize: " + blockSize + " repl: " + repl); // fill random data into file byte[] b = new byte[(int) blockSize]; long numBlocks = fileSize / blockSize; for (int i = 0; i < numBlocks; i++) { rand.nextBytes(b); stm.write(b); crc.update(b); } long lastBlock = fileSize - numBlocks * blockSize; if (lastBlock > 0) { b = new byte[(int) lastBlock]; rand.nextBytes(b); stm.write(b); crc.update(b); } stm.close(); return crc.getValue(); } /** * Create a bunch of files under a directory srcDir. * all files' lengths are in fileSizes * all files' block sizes are in blockSizs * we will generate these files and put their checksum into crcs array * The seeds we use to generate files are stored in seeds array */ public static Path[] createTestFiles(Path srcDir, long[] fileSizes, long[] blockSizes, long[] crcs, int[] seeds, FileSystem fileSys, short repl) throws IOException { Path[] files = new Path[fileSizes.length]; fileSys.mkdirs(srcDir); LOG.info("Create files under directory " + srcDir); Random rand = new Random(); for (int i = 0; i < fileSizes.length; i++) { Path file = files[i] = new Path(srcDir, "file" + i); seeds[i] = rand.nextInt(); crcs[i] = TestRaidDfs.createTestFile(fileSys, files[i], repl, fileSizes[i], blockSizes[i], seeds[i]); assertEquals("file size is not expected", fileSizes[i], fileSys.getFileStatus(file).getLen()); } return files; } // // Creates a file with partially full last block. Populate it with random // data. Returns its crc. // public static long createTestFilePartialLastBlock(FileSystem fileSys, Path name, int repl, int numBlocks, long blocksize) throws IOException { CRC32 crc = new CRC32(); Random rand = new Random(); FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf().getInt("io.file.buffer.size", 4096), (short) repl, blocksize); // Write whole blocks. byte[] b = new byte[(int) blocksize]; for (int i = 1; i < numBlocks; i++) { rand.nextBytes(b); stm.write(b); crc.update(b); } // Write partial block. b = new byte[(int) blocksize / 2 - 1]; rand.nextBytes(b); stm.write(b); crc.update(b); stm.close(); return crc.getValue(); } static long bufferCRC(byte[] buf) { CRC32 crc = new CRC32(); crc.update(buf, 0, buf.length); return crc.getValue(); } // // validates that file matches the crc. // public static boolean validateFile(FileSystem fileSys, Path name, long length, long crc) throws IOException { long numRead = 0; CRC32 newcrc = new CRC32(); FSDataInputStream stm = fileSys.open(name); final byte[] b = new byte[4192]; int num = 0; while (num >= 0) { num = stm.read(b); if (num < 0) { break; } numRead += num; newcrc.update(b, 0, num); } stm.close(); if (numRead != length) { LOG.info("Number of bytes read " + numRead + " does not match file size " + length); return false; } LOG.info(" Newcrc " + newcrc.getValue() + " old crc " + crc); if (newcrc.getValue() != crc) { LOG.info("CRC mismatch of file " + name + ": " + newcrc.getValue() + " vs. " + crc); return false; } return true; } /* * The Data directories for a datanode */ private static File[] getDataNodeDirs(int i, MiniDFSCluster cluster) throws IOException { File[] dir = new File[2]; dir[0] = cluster.getBlockDirectory("data" + (2 * i + 1)); dir[1] = cluster.getBlockDirectory("data" + (2 * i + 2)); return dir; } public static void removeAndReportBlock(DistributedFileSystem blockDfs, Path filePath, int[] blockIdxs, MiniDFSCluster cluster) throws IOException { FileStatus stat = blockDfs.getFileStatus(filePath); LocatedBlocks blocks = blockDfs.getLocatedBlocks(filePath, 0, stat.getLen()); for (int blockIdx : blockIdxs) { removeAndReportBlock(blockDfs, filePath, blocks.get(blockIdx), cluster); } } /** * removes a specified block from MiniDFS storage and reports it as corrupt */ public static void removeAndReportBlock(DistributedFileSystem blockDfs, Path filePath, LocatedBlock block, MiniDFSCluster cluster) throws IOException { corruptBlock(filePath, block.getBlock(), NUM_DATANODES, true, cluster); // report deleted block to the name node LocatedBlock[] toReport = { block }; blockDfs.getClient().namenode.reportBadBlocks(toReport); } // // Delete/Corrupt specified block of file // public static void corruptBlock(Path file, Block blockNum, int numDataNodes, boolean delete, MiniDFSCluster cluster) throws IOException { long id = blockNum.getBlockId(); // Now deliberately remove/truncate data blocks from the block. int numDeleted = 0; int numCorrupted = 0; for (int i = 0; i < numDataNodes; i++) { File[] dirs = getDataNodeDirs(i, cluster); for (int j = 0; j < dirs.length; j++) { File[] blocks = dirs[j].listFiles(); assertTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.length >= 0)); for (int idx = 0; idx < blocks.length; idx++) { LOG.info("block file: " + blocks[idx]); if (blocks[idx].getName().startsWith("blk_" + id) && !blocks[idx].getName().endsWith(".meta")) { if (delete) { blocks[idx].delete(); LOG.info("Deleted block " + blocks[idx]); numDeleted++; } else { // Corrupt File f = blocks[idx]; long seekPos = f.length() / 2; RandomAccessFile raf = new RandomAccessFile(f, "rw"); raf.seek(seekPos); int data = raf.readInt(); raf.seek(seekPos); raf.writeInt(data + 1); LOG.info("Corrupted block " + blocks[idx]); numCorrupted++; } } } } } assertTrue("Nothing corrupted or deleted", (numCorrupted + numDeleted) > 0); } public static void corruptBlock(Path file, Block blockNum, int numDataNodes, long offset, MiniDFSCluster cluster) throws IOException { long id = blockNum.getBlockId(); // Now deliberately remove/truncate data blocks from the block. // for (int i = 0; i < numDataNodes; i++) { File[] dirs = getDataNodeDirs(i, cluster); for (int j = 0; j < dirs.length; j++) { File[] blocks = dirs[j].listFiles(); assertTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.length >= 0)); for (int idx = 0; idx < blocks.length; idx++) { if (blocks[idx].getName().startsWith("blk_" + id) && !blocks[idx].getName().endsWith(".meta")) { // Corrupt File f = blocks[idx]; RandomAccessFile raf = new RandomAccessFile(f, "rw"); raf.seek(offset); int data = raf.readInt(); raf.seek(offset); raf.writeInt(data + 1); LOG.info("Corrupted block " + blocks[idx]); } } } } } }