org.apache.hadoop.hdfs.server.namenode.TestFSEditLogLoader.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hdfs.server.namenode.TestFSEditLogLoader.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hdfs.server.namenode;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.spy;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
import java.util.HashMap;
import java.util.Map;
import java.util.SortedMap;

import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.PathUtils;
import org.apache.log4j.Level;
import org.junit.Test;

import com.google.common.collect.Maps;
import com.google.common.io.Files;

public class TestFSEditLogLoader {

    static {
        ((Log4JLogger) FSImage.LOG).getLogger().setLevel(Level.ALL);
        ((Log4JLogger) FSEditLogLoader.LOG).getLogger().setLevel(Level.ALL);
    }

    private static final File TEST_DIR = PathUtils.getTestDir(TestFSEditLogLoader.class);

    private static final int NUM_DATA_NODES = 0;

    @Test
    public void testDisplayRecentEditLogOpCodes() throws IOException {
        // start a cluster 
        Configuration conf = new HdfsConfiguration();
        MiniDFSCluster cluster = null;
        FileSystem fileSys = null;
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES)
                .enableManagedDfsDirsRedundancy(false).build();
        cluster.waitActive();
        fileSys = cluster.getFileSystem();
        final FSNamesystem namesystem = cluster.getNamesystem();

        FSImage fsimage = namesystem.getFSImage();
        for (int i = 0; i < 20; i++) {
            fileSys.mkdirs(new Path("/tmp/tmp" + i));
        }
        StorageDirectory sd = fsimage.getStorage().dirIterator(NameNodeDirType.EDITS).next();
        cluster.shutdown();

        File editFile = FSImageTestUtil.findLatestEditsLog(sd).getFile();
        assertTrue("Should exist: " + editFile, editFile.exists());

        // Corrupt the edits file.
        long fileLen = editFile.length();
        RandomAccessFile rwf = new RandomAccessFile(editFile, "rw");
        rwf.seek(fileLen - 40);
        for (int i = 0; i < 20; i++) {
            rwf.write(FSEditLogOpCodes.OP_DELETE.getOpCode());
        }
        rwf.close();

        StringBuilder bld = new StringBuilder();
        bld.append("^Error replaying edit log at offset \\d+.  ");
        bld.append("Expected transaction ID was \\d+\n");
        bld.append("Recent opcode offsets: (\\d+\\s*){4}$");
        try {
            cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES)
                    .enableManagedDfsDirsRedundancy(false).format(false).build();
            fail("should not be able to start");
        } catch (IOException e) {
            assertTrue("error message contains opcodes message", e.getMessage().matches(bld.toString()));
        }
    }

    /**
     * Test that, if the NN restarts with a new minimum replication,
     * any files created with the old replication count will get
     * automatically bumped up to the new minimum upon restart.
     */
    @Test
    public void testReplicationAdjusted() throws Exception {
        // start a cluster 
        Configuration conf = new HdfsConfiguration();
        // Replicate and heartbeat fast to shave a few seconds off test
        conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
        conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);

        MiniDFSCluster cluster = null;
        try {
            cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
            cluster.waitActive();
            FileSystem fs = cluster.getFileSystem();

            // Create a file with replication count 1
            Path p = new Path("/testfile");
            DFSTestUtil.createFile(fs, p, 10, /*repl*/ (short) 1, 1);
            DFSTestUtil.waitReplication(fs, p, (short) 1);

            // Shut down and restart cluster with new minimum replication of 2
            cluster.shutdown();
            cluster = null;

            conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, 2);

            cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(false).build();
            cluster.waitActive();
            fs = cluster.getFileSystem();

            // The file should get adjusted to replication 2 when
            // the edit log is replayed.
            DFSTestUtil.waitReplication(fs, p, (short) 2);
        } finally {
            if (cluster != null) {
                cluster.shutdown();
            }
        }
    }

    /**
     * Corrupt the byte at the given offset in the given file,
     * by subtracting 1 from it.
     */
    private void corruptByteInFile(File file, long offset) throws IOException {
        RandomAccessFile raf = new RandomAccessFile(file, "rw");
        try {
            raf.seek(offset);
            int origByte = raf.read();
            raf.seek(offset);
            raf.writeByte(origByte - 1);
        } finally {
            IOUtils.closeStream(raf);
        }
    }

    /**
     * Truncate the given file to the given length
     */
    private void truncateFile(File logFile, long newLength) throws IOException {
        RandomAccessFile raf = new RandomAccessFile(logFile, "rw");
        raf.setLength(newLength);
        raf.close();
    }

    /**
     * Return the length of bytes in the given file after subtracting
     * the trailer of 0xFF (OP_INVALID)s.
     * This seeks to the end of the file and reads chunks backwards until
     * it finds a non-0xFF byte.
     * @throws IOException if the file cannot be read
     */
    private static long getNonTrailerLength(File f) throws IOException {
        final int chunkSizeToRead = 256 * 1024;
        FileInputStream fis = new FileInputStream(f);
        try {

            byte buf[] = new byte[chunkSizeToRead];

            FileChannel fc = fis.getChannel();
            long size = fc.size();
            long pos = size - (size % chunkSizeToRead);

            while (pos >= 0) {
                fc.position(pos);

                int readLen = (int) Math.min(size - pos, chunkSizeToRead);
                IOUtils.readFully(fis, buf, 0, readLen);
                for (int i = readLen - 1; i >= 0; i--) {
                    if (buf[i] != FSEditLogOpCodes.OP_INVALID.getOpCode()) {
                        return pos + i + 1; // + 1 since we count this byte!
                    }
                }

                pos -= chunkSizeToRead;
            }
            return 0;
        } finally {
            fis.close();
        }
    }

    @Test
    public void testStreamLimiter() throws IOException {
        final File LIMITER_TEST_FILE = new File(TEST_DIR, "limiter.test");

        FileOutputStream fos = new FileOutputStream(LIMITER_TEST_FILE);
        try {
            fos.write(0x12);
            fos.write(0x12);
            fos.write(0x12);
        } finally {
            fos.close();
        }

        FileInputStream fin = new FileInputStream(LIMITER_TEST_FILE);
        BufferedInputStream bin = new BufferedInputStream(fin);
        FSEditLogLoader.PositionTrackingInputStream tracker = new FSEditLogLoader.PositionTrackingInputStream(bin);
        try {
            tracker.setLimit(2);
            tracker.mark(100);
            tracker.read();
            tracker.read();
            try {
                tracker.read();
                fail("expected to get IOException after reading past the limit");
            } catch (IOException e) {
            }
            tracker.reset();
            tracker.mark(100);
            byte arr[] = new byte[3];
            try {
                tracker.read(arr);
                fail("expected to get IOException after reading past the limit");
            } catch (IOException e) {
            }
            tracker.reset();
            arr = new byte[2];
            tracker.read(arr);
        } finally {
            tracker.close();
        }
    }

    /**
     * Create an unfinalized edit log for testing purposes
     *
     * @param testDir           Directory to create the edit log in
     * @param numTx             Number of transactions to add to the new edit log
     * @param offsetToTxId      A map from transaction IDs to offsets in the 
     *                          edit log file.
     * @return                  The new edit log file name.
     * @throws IOException
     */
    static private File prepareUnfinalizedTestEditLog(File testDir, int numTx, SortedMap<Long, Long> offsetToTxId)
            throws IOException {
        File inProgressFile = new File(testDir, NNStorage.getInProgressEditsFileName(1));
        FSEditLog fsel = null, spyLog = null;
        try {
            fsel = FSImageTestUtil.createStandaloneEditLog(testDir);
            spyLog = spy(fsel);
            // Normally, the in-progress edit log would be finalized by
            // FSEditLog#endCurrentLogSegment.  For testing purposes, we
            // disable that here.
            doNothing().when(spyLog).endCurrentLogSegment(true);
            spyLog.openForWrite(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
            assertTrue("should exist: " + inProgressFile, inProgressFile.exists());

            for (int i = 0; i < numTx; i++) {
                long trueOffset = getNonTrailerLength(inProgressFile);
                long thisTxId = spyLog.getLastWrittenTxId() + 1;
                offsetToTxId.put(trueOffset, thisTxId);
                System.err.println("txid " + thisTxId + " at offset " + trueOffset);
                spyLog.logDelete("path" + i, i, false);
                spyLog.logSync();
            }
        } finally {
            if (spyLog != null) {
                spyLog.close();
            } else if (fsel != null) {
                fsel.close();
            }
        }
        return inProgressFile;
    }

    @Test
    public void testValidateEditLogWithCorruptHeader() throws IOException {
        File testDir = new File(TEST_DIR, "testValidateEditLogWithCorruptHeader");
        SortedMap<Long, Long> offsetToTxId = Maps.newTreeMap();
        File logFile = prepareUnfinalizedTestEditLog(testDir, 2, offsetToTxId);
        RandomAccessFile rwf = new RandomAccessFile(logFile, "rw");
        try {
            rwf.seek(0);
            rwf.writeLong(42); // corrupt header
        } finally {
            rwf.close();
        }
        EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile);
        assertTrue(validation.hasCorruptHeader());
    }

    @Test
    public void testValidateEditLogWithCorruptBody() throws IOException {
        File testDir = new File(TEST_DIR, "testValidateEditLogWithCorruptBody");
        SortedMap<Long, Long> offsetToTxId = Maps.newTreeMap();
        final int NUM_TXNS = 20;
        File logFile = prepareUnfinalizedTestEditLog(testDir, NUM_TXNS, offsetToTxId);
        // Back up the uncorrupted log
        File logFileBak = new File(testDir, logFile.getName() + ".bak");
        Files.copy(logFile, logFileBak);
        EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile);
        assertTrue(!validation.hasCorruptHeader());
        // We expect that there will be an OP_START_LOG_SEGMENT, followed by
        // NUM_TXNS opcodes, followed by an OP_END_LOG_SEGMENT.
        assertEquals(NUM_TXNS + 1, validation.getEndTxId());
        // Corrupt each edit and verify that validation continues to work
        for (Map.Entry<Long, Long> entry : offsetToTxId.entrySet()) {
            long txOffset = entry.getKey();
            long txId = entry.getValue();

            // Restore backup, corrupt the txn opcode
            Files.copy(logFileBak, logFile);
            corruptByteInFile(logFile, txOffset);
            validation = EditLogFileInputStream.validateEditLog(logFile);
            long expectedEndTxId = (txId == (NUM_TXNS + 1)) ? NUM_TXNS : (NUM_TXNS + 1);
            assertEquals("Failed when corrupting txn opcode at " + txOffset, expectedEndTxId,
                    validation.getEndTxId());
            assertTrue(!validation.hasCorruptHeader());
        }

        // Truncate right before each edit and verify that validation continues
        // to work
        for (Map.Entry<Long, Long> entry : offsetToTxId.entrySet()) {
            long txOffset = entry.getKey();
            long txId = entry.getValue();

            // Restore backup, corrupt the txn opcode
            Files.copy(logFileBak, logFile);
            truncateFile(logFile, txOffset);
            validation = EditLogFileInputStream.validateEditLog(logFile);
            long expectedEndTxId = (txId == 0) ? HdfsServerConstants.INVALID_TXID : (txId - 1);
            assertEquals("Failed when corrupting txid " + txId + " txn opcode " + "at " + txOffset, expectedEndTxId,
                    validation.getEndTxId());
            assertTrue(!validation.hasCorruptHeader());
        }
    }

    @Test
    public void testValidateEmptyEditLog() throws IOException {
        File testDir = new File(TEST_DIR, "testValidateEmptyEditLog");
        SortedMap<Long, Long> offsetToTxId = Maps.newTreeMap();
        File logFile = prepareUnfinalizedTestEditLog(testDir, 0, offsetToTxId);
        // Truncate the file so that there is nothing except the header and
        // layout flags section.
        truncateFile(logFile, 8);
        EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile);
        assertTrue(!validation.hasCorruptHeader());
        assertEquals(HdfsServerConstants.INVALID_TXID, validation.getEndTxId());
    }

    private static final Map<Byte, FSEditLogOpCodes> byteToEnum = new HashMap<Byte, FSEditLogOpCodes>();
    static {
        for (FSEditLogOpCodes opCode : FSEditLogOpCodes.values()) {
            byteToEnum.put(opCode.getOpCode(), opCode);
        }
    }

    private static FSEditLogOpCodes fromByte(byte opCode) {
        return byteToEnum.get(opCode);
    }

    @Test
    public void testFSEditLogOpCodes() throws IOException {
        //try all codes
        for (FSEditLogOpCodes c : FSEditLogOpCodes.values()) {
            final byte code = c.getOpCode();
            assertEquals("c=" + c + ", code=" + code, c, FSEditLogOpCodes.fromByte(code));
        }

        //try all byte values
        for (int b = 0; b < (1 << Byte.SIZE); b++) {
            final byte code = (byte) b;
            assertEquals("b=" + b + ", code=" + code, fromByte(code), FSEditLogOpCodes.fromByte(code));
        }
    }
}