Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hdfs.tools.offlineImageViewer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.BufferedReader; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction; import org.apache.hadoop.security.token.Token; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; /** * Test function of OfflineImageViewer by: * * confirming it can correctly process a valid fsimage file and that * the processing generates a correct representation of the namespace * * confirming it correctly fails to process an fsimage file with a layout * version it shouldn't be able to handle * * confirm it correctly bails on malformed image files, in particular, a * file that ends suddenly. */ public class TestOfflineImageViewer { private static final Log LOG = LogFactory.getLog(OfflineImageViewer.class); private static final int NUM_DIRS = 3; private static final int FILES_PER_DIR = 4; private static final String TEST_RENEWER = "JobTracker"; private static File originalFsimage = null; // Elements of lines of ls-file output to be compared to FileStatus instance private static class LsElements { public String perms; public int replication; public String username; public String groupname; public long filesize; public char dir; // d if dir, - otherwise } // namespace as written to dfs, to be compared with viewer's output final static HashMap<String, FileStatus> writtenFiles = new HashMap<String, FileStatus>(); private static String ROOT = System.getProperty("test.build.data", "build/test/data"); // Create a populated namespace for later testing. Save its contents to a // data structure and store its fsimage location. // We only want to generate the fsimage file once and use it for // multiple tests. @BeforeClass public static void createOriginalFSImage() throws IOException { MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000); conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); conf.set("hadoop.security.auth_to_local", "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT"); cluster = new MiniDFSCluster(conf, 4, true, null); cluster.waitActive(); FileSystem hdfs = cluster.getFileSystem(); int filesize = 256; // Create a reasonable namespace for (int i = 0; i < NUM_DIRS; i++) { Path dir = new Path("/dir" + i); hdfs.mkdirs(dir); writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString())); for (int j = 0; j < FILES_PER_DIR; j++) { Path file = new Path(dir, "file" + j); FSDataOutputStream o = hdfs.create(file); o.write(new byte[filesize++]); o.close(); writtenFiles.put(file.toString(), pathToFileEntry(hdfs, file.toString())); } } // Get delegation tokens so we log the delegation token op Token<?> delegationToken = hdfs.getDelegationToken(TEST_RENEWER); LOG.debug("got token " + delegationToken); // Write results to the fsimage file cluster.getNameNode().setSafeMode(SafeModeAction.SAFEMODE_ENTER); cluster.getNameNode().saveNamespace(); // Determine location of fsimage file File[] files = cluster.getNameDirs().toArray(new File[0]); originalFsimage = new File(files[0], "current/fsimage"); if (!originalFsimage.exists()) fail("Didn't generate or can't find fsimage."); } finally { if (cluster != null) cluster.shutdown(); } } @AfterClass public static void deleteOriginalFSImage() throws IOException { if (originalFsimage != null && originalFsimage.exists()) { originalFsimage.delete(); } } // Convenience method to generate a file status from file system for // later comparison private static FileStatus pathToFileEntry(FileSystem hdfs, String file) throws IOException { return hdfs.getFileStatus(new Path(file)); } // Verify that we can correctly generate an ls-style output for a valid // fsimage @Test public void outputOfLSVisitor() throws IOException { File testFile = new File(ROOT, "/basicCheck"); File outputFile = new File(ROOT, "/basicCheckOutput"); try { copyFile(originalFsimage, testFile); ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true); OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false); oiv.go(); HashMap<String, LsElements> fileOutput = readLsfile(outputFile); compareNamespaces(writtenFiles, fileOutput); } finally { if (testFile.exists()) testFile.delete(); if (outputFile.exists()) outputFile.delete(); } LOG.debug("Correctly generated ls-style output."); } // Confirm that attempting to read an fsimage file with an unsupported // layout results in an error @Test public void unsupportedFSLayoutVersion() throws IOException { File testFile = new File(ROOT, "/invalidLayoutVersion"); File outputFile = new File(ROOT, "invalidLayoutVersionOutput"); try { int badVersionNum = -432; changeLayoutVersion(originalFsimage, testFile, badVersionNum); ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true); OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false); try { oiv.go(); fail("Shouldn't be able to read invalid laytout version"); } catch (IOException e) { if (!e.getMessage().contains(Integer.toString(badVersionNum))) throw e; // wasn't error we were expecting LOG.debug("Correctly failed at reading bad image version."); } } finally { if (testFile.exists()) testFile.delete(); if (outputFile.exists()) outputFile.delete(); } } // Verify that image viewer will bail on a file that ends unexpectedly @Test public void truncatedFSImage() throws IOException { File testFile = new File(ROOT, "/truncatedFSImage"); File outputFile = new File(ROOT, "/trucnatedFSImageOutput"); try { copyPartOfFile(originalFsimage, testFile); assertTrue("Created truncated fsimage", testFile.exists()); ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true); OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false); try { oiv.go(); fail("Managed to process a truncated fsimage file"); } catch (EOFException e) { LOG.debug("Correctly handled EOF"); } } finally { if (testFile.exists()) testFile.delete(); if (outputFile.exists()) outputFile.delete(); } } // Test that our ls file has all the same compenents of the original namespace private void compareNamespaces(HashMap<String, FileStatus> written, HashMap<String, LsElements> fileOutput) { assertEquals("Should be the same number of files in both, plus one for root" + " in fileoutput", fileOutput.keySet().size(), written.keySet().size() + 1); Set<String> inFile = fileOutput.keySet(); // For each line in the output file, verify that the namespace had a // filestatus counterpart for (String path : inFile) { if (path.equals("/")) // root's not included in output from system call continue; assertTrue("Path in file (" + path + ") was written to fs", written.containsKey(path)); compareFiles(written.get(path), fileOutput.get(path)); written.remove(path); } assertEquals("No more files were written to fs", 0, written.size()); } // Compare two files as listed in the original namespace FileStatus and // the output of the ls file from the image processor private void compareFiles(FileStatus fs, LsElements elements) { assertEquals("directory listed as such", fs.isDir() ? 'd' : '-', elements.dir); assertEquals("perms string equal", fs.getPermission().toString(), elements.perms); assertEquals("replication equal", fs.getReplication(), elements.replication); assertEquals("owner equal", fs.getOwner(), elements.username); assertEquals("group equal", fs.getGroup(), elements.groupname); assertEquals("lengths equal", fs.getLen(), elements.filesize); } // Read the contents of the file created by the Ls processor private HashMap<String, LsElements> readLsfile(File lsFile) throws IOException { BufferedReader br = new BufferedReader(new FileReader(lsFile)); String line = null; HashMap<String, LsElements> fileContents = new HashMap<String, LsElements>(); while ((line = br.readLine()) != null) readLsLine(line, fileContents); return fileContents; } // Parse a line from the ls output. Store permissions, replication, // username, groupname and filesize in hashmap keyed to the path name private void readLsLine(String line, HashMap<String, LsElements> fileContents) { String elements[] = line.split("\\s+"); assertEquals("Not enough elements in ls output", 8, elements.length); LsElements lsLine = new LsElements(); lsLine.dir = elements[0].charAt(0); lsLine.perms = elements[0].substring(1); lsLine.replication = elements[1].equals("-") ? 0 : Integer.valueOf(elements[1]); lsLine.username = elements[2]; lsLine.groupname = elements[3]; lsLine.filesize = Long.valueOf(elements[4]); // skipping date and time String path = elements[7]; // Check that each file in the ls output was listed once assertFalse("LS file had duplicate file entries", fileContents.containsKey(path)); fileContents.put(path, lsLine); } // Copy one fsimage to another, changing the layout version in the process private void changeLayoutVersion(File src, File dest, int newVersion) throws IOException { DataInputStream in = null; DataOutputStream out = null; try { in = new DataInputStream(new FileInputStream(src)); out = new DataOutputStream(new FileOutputStream(dest)); in.readInt(); out.writeInt(newVersion); byte[] b = new byte[1024]; while (in.read(b) > 0) { out.write(b); } } finally { if (in != null) in.close(); if (out != null) out.close(); } } // Only copy part of file into the other. Used for testing truncated fsimage private void copyPartOfFile(File src, File dest) throws IOException { InputStream in = null; OutputStream out = null; byte[] b = new byte[256]; int bytesWritten = 0; int count; int maxBytes = 700; try { in = new FileInputStream(src); out = new FileOutputStream(dest); while ((count = in.read(b)) > 0 && bytesWritten < maxBytes) { out.write(b); bytesWritten += count; } } finally { if (in != null) in.close(); if (out != null) out.close(); } } // Copy one file's contents into the other private void copyFile(File src, File dest) throws IOException { InputStream in = null; OutputStream out = null; try { in = new FileInputStream(src); out = new FileOutputStream(dest); byte[] b = new byte[1024]; while (in.read(b) > 0) { out.write(b); } } finally { if (in != null) in.close(); if (out != null) out.close(); } } @Test public void outputOfFileDistributionVisitor() throws IOException { File testFile = new File(ROOT, "/basicCheck"); File outputFile = new File(ROOT, "/fileDistributionCheckOutput"); int totalFiles = 0; try { copyFile(originalFsimage, testFile); ImageVisitor v = new FileDistributionVisitor(outputFile.getPath(), 0, 0); OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false); oiv.go(); BufferedReader reader = new BufferedReader(new FileReader(outputFile)); String line = reader.readLine(); assertEquals(line, "Size\tNumFiles"); while ((line = reader.readLine()) != null) { String[] row = line.split("\t"); assertEquals(row.length, 2); totalFiles += Integer.parseInt(row[1]); } } finally { if (testFile.exists()) testFile.delete(); if (outputFile.exists()) outputFile.delete(); } assertEquals(totalFiles, NUM_DIRS * FILES_PER_DIR); } private static class TestImageVisitor extends ImageVisitor { private List<String> delegationTokenRenewers = new LinkedList<String>(); TestImageVisitor() { } List<String> getDelegationTokenRenewers() { return delegationTokenRenewers; } @Override void start() throws IOException { } @Override void finish() throws IOException { } @Override void finishAbnormally() throws IOException { } @Override void visit(ImageElement element, String value) throws IOException { if (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_RENEWER) { delegationTokenRenewers.add(value); } } @Override void visitEnclosingElement(ImageElement element) throws IOException { } @Override void visitEnclosingElement(ImageElement element, ImageElement key, String value) throws IOException { } @Override void leaveEnclosingElement() throws IOException { } } @Test public void outputOfTestVisitor() throws IOException { File testFile = new File(ROOT, "/basicCheck"); try { copyFile(originalFsimage, testFile); TestImageVisitor v = new TestImageVisitor(); OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, true); oiv.go(); // Validated stored delegation token identifiers. List<String> dtrs = v.getDelegationTokenRenewers(); assertEquals(1, dtrs.size()); assertEquals(TEST_RENEWER, dtrs.get(0)); } finally { if (testFile.exists()) testFile.delete(); } LOG.debug("Passed TestVisitor validation."); } }