org.apache.hadoop.hdfs.server.namenode.TestNameNodeReconfigure.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hdfs.server.namenode.TestNameNodeReconfigure.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hdfs.server.namenode;

import java.io.IOException;
import java.io.File;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.List;

import org.junit.Test;
import org.junit.Before;
import org.junit.After;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.ReconfigurationException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.net.Node;

public class TestNameNodeReconfigure {

    public static final Log LOG = LogFactory.getLog(TestNameNodeReconfigure.class);

    public static class MockPlacementPolicy extends BlockPlacementPolicyDefault {
        private static AtomicInteger callCounter = null;

        public static synchronized void setCallCounter(AtomicInteger callCounter) {
            MockPlacementPolicy.callCounter = callCounter;
        }

        public static synchronized AtomicInteger getCallCounter() {
            return callCounter;
        }

        @Override
        DatanodeDescriptor[] chooseTarget(int numOfReplicas, DatanodeDescriptor writer,
                List<DatanodeDescriptor> chosenNodes, List<Node> exlcNodes, long blocksize) {
            getCallCounter().getAndIncrement();

            return super.chooseTarget(numOfReplicas, writer, chosenNodes, exlcNodes, blocksize);
        }

    }

    private MiniDFSCluster cluster;
    private FileSystem fs;

    public void setUp() throws IOException {
        Configuration conf = new Configuration();
        cluster = new MiniDFSCluster(conf, 3, true, null);
        fs = cluster.getFileSystem();
    }

    /**
     * Test that we can change the block placement policy through the
     * reconfigurable API.
     */
    @Test
    public void testChangeBlockPlacementPolicy() throws IOException, ReconfigurationException {
        setUp();
        AtomicInteger callCounter = new AtomicInteger(0);
        MockPlacementPolicy.setCallCounter(callCounter);

        DFSTestUtil util = new DFSTestUtil("", 2, 1, 512);

        // write some files with the default block placement policy
        util.createFiles(fs, "/reconfdat1", (short) 3);
        util.waitReplication(fs, "/reconfdat1", (short) 3);

        assertTrue("calls already made to MockPlacementPolicy", callCounter.get() == 0);

        // switch over to the mock placement policy
        cluster.getNameNode().reconfigureProperty("dfs.block.replicator.classname", "org.apache.hadoop.hdfs.server."
                + "namenode." + "TestNameNodeReconfigure$" + "MockPlacementPolicy");

        // write some files with the mock placement policy
        util.createFiles(fs, "/reconfdat2", (short) 3);
        util.waitReplication(fs, "/reconfdat2", (short) 3);

        int callsMade1 = callCounter.get();

        // check that calls were made to mock placement policy
        assertTrue("no calls made to MockPlacementPolicy", callsMade1 > 0);
        LOG.info("" + callsMade1 + " calls made to MockPlacementPolicy");

        // now try to change it to a non-existent class
        try {
            cluster.getNameNode().reconfigureProperty("dfs.block.replicator.classname", "does.not.exist");
            fail("ReconfigurationException expected");
        } catch (RuntimeException expected) {
            assertTrue("exception should have cause", expected.getCause() != null);
            assertTrue("exception's cause should have cause", expected.getCause().getCause() != null);
            assertTrue(
                    "ClassNotFoundException expected but got "
                            + expected.getCause().getCause().getClass().getCanonicalName(),
                    expected.getCause().getCause() instanceof ClassNotFoundException);
        }

        // write some files, they should still go to the mock placemeny policy
        util.createFiles(fs, "/reconfdat3", (short) 3);
        util.waitReplication(fs, "/reconfdat3", (short) 3);

        int callsMade2 = callCounter.get();

        // check that more calls were made to mock placement policy
        assertTrue("no calls made to MockPlacementPolicy", callsMade2 > callsMade1);
        LOG.info("" + (callsMade2 - callsMade1) + " calls made to MockPlacementPolicy");

        // now revert back to the default policy
        cluster.getNameNode().reconfigureProperty("dfs.block.replicator.classname", null);

        // write some files with the default block placement policy
        util.createFiles(fs, "/reconfdat4", (short) 3);
        util.waitReplication(fs, "/reconfdat4", (short) 3);

        // make sure that no more calls were made to mock placement policy
        assertTrue("more calls made to MockPlacementPolicy", callCounter.get() == callsMade2);

        util.cleanup(fs, "/reconfdat1");
        util.cleanup(fs, "/reconfdat2");
        util.cleanup(fs, "/reconfdat3");
        util.cleanup(fs, "/reconfdat4");
    }

    /**
     * Test that we can modify configuration properties.
     */
    @Test
    public void testReconfigure() throws IOException, ReconfigurationException {
        setUp();
        // change properties
        cluster.getNameNode().reconfigureProperty("dfs.heartbeat.interval", "" + 6);
        cluster.getNameNode().reconfigureProperty("heartbeat.recheck.interval", "" + (10 * 60 * 1000));
        cluster.getNameNode().reconfigureProperty("dfs.persist.blocks", "true");
        cluster.getNameNode().reconfigureProperty("dfs.permissions.audit.log", "true");

        // try invalid values
        try {
            cluster.getNameNode().reconfigureProperty("dfs.heartbeat.interval", "text");
            fail("ReconfigurationException expected");
        } catch (ReconfigurationException expected) {
        }
        try {
            cluster.getNameNode().reconfigureProperty("heartbeat.recheck.interval", "text");
            fail("ReconfigurationException expected");
        } catch (ReconfigurationException expected) {
        }
        try {
            cluster.getNameNode().reconfigureProperty("dfs.persist.blocks", "text");
            fail("ReconfigurationException expected");
        } catch (ReconfigurationException expected) {
        }
        try {
            cluster.getNameNode().reconfigureProperty("dfs.permissions.audit.log", "text");
            fail("ReconfigurationException expected");
        } catch (ReconfigurationException expected) {
        }

        // verify change
        assertEquals("dfs.heartbeat.interval has wrong value", 6000L,
                cluster.getNameNode().namesystem.heartbeatInterval);
        assertEquals("heartbeat.recheck.interval has wrong value", 10 * 60 * 1000,
                cluster.getNameNode().namesystem.heartbeatRecheckInterval);
        assertTrue("dfs.persist.blocks has wrong value", cluster.getNameNode().namesystem.getPersistBlocks());
        assertTrue("dfs.permissions.audit.log has wrong value",
                cluster.getNameNode().namesystem.getPermissionAuditLog());

        // revert to defaults
        cluster.getNameNode().reconfigureProperty("dfs.heartbeat.interval", null);
        cluster.getNameNode().reconfigureProperty("heartbeat.recheck.interval", null);
        cluster.getNameNode().reconfigureProperty("dfs.persist.blocks", null);
        cluster.getNameNode().reconfigureProperty("dfs.permissions.audit.log", null);

        // verify defaults
        assertEquals("dfs.heartbeat.interval has wrong value", 3000L,
                cluster.getNameNode().namesystem.heartbeatInterval);
        assertEquals("heartbeat.recheck.interval has wrong value", 5 * 60 * 1000,
                cluster.getNameNode().namesystem.heartbeatRecheckInterval);
        assertFalse("dfs.persist.blocks has wrong value", cluster.getNameNode().namesystem.getPersistBlocks());
        assertFalse("dfs.permissions.audit.log has wrong value",
                cluster.getNameNode().namesystem.getPermissionAuditLog());
    }

    /**
     * Test that includes/excludes will be ignored
     * if dfs.ignore.missing.include.files is set 
     */
    @Test
    public void testIncludesExcludesConfigure() throws IOException {
        String inFile = "/tmp/inFileNotExists";
        String exFile = "/tmp/exFileNotExists";
        File include = new File(inFile);
        File exclude = new File(exFile);
        include.delete();
        exclude.delete();
        assertFalse(include.exists());
        assertFalse(exclude.exists());

        Configuration conf = new Configuration();
        conf.set("dfs.hosts.ignoremissing", "true");
        conf.set(FSConstants.DFS_HOSTS, inFile);
        conf.set("dfs.hosts.exclude", exFile);
        cluster = new MiniDFSCluster(conf, 3, true, null);
    }

    @After
    public void shutDown() throws IOException {
        if (fs != null) {
            fs.close();
        }

        if (cluster != null) {
            cluster.shutdown();
        }
    }
}