Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hdfs.server.namenode; import java.io.IOException; import java.io.File; import java.util.concurrent.atomic.AtomicInteger; import java.util.List; import org.junit.Test; import org.junit.Before; import org.junit.After; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.ReconfigurationException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.net.Node; public class TestNameNodeReconfigure { public static final Log LOG = LogFactory.getLog(TestNameNodeReconfigure.class); public static class MockPlacementPolicy extends BlockPlacementPolicyDefault { private static AtomicInteger callCounter = null; public static synchronized void setCallCounter(AtomicInteger callCounter) { MockPlacementPolicy.callCounter = callCounter; } public static synchronized AtomicInteger getCallCounter() { return callCounter; } @Override DatanodeDescriptor[] chooseTarget(int numOfReplicas, DatanodeDescriptor writer, List<DatanodeDescriptor> chosenNodes, List<Node> exlcNodes, long blocksize) { getCallCounter().getAndIncrement(); return super.chooseTarget(numOfReplicas, writer, chosenNodes, exlcNodes, blocksize); } } private MiniDFSCluster cluster; private FileSystem fs; public void setUp() throws IOException { Configuration conf = new Configuration(); cluster = new MiniDFSCluster(conf, 3, true, null); fs = cluster.getFileSystem(); } /** * Test that we can change the block placement policy through the * reconfigurable API. */ @Test public void testChangeBlockPlacementPolicy() throws IOException, ReconfigurationException { setUp(); AtomicInteger callCounter = new AtomicInteger(0); MockPlacementPolicy.setCallCounter(callCounter); DFSTestUtil util = new DFSTestUtil("", 2, 1, 512); // write some files with the default block placement policy util.createFiles(fs, "/reconfdat1", (short) 3); util.waitReplication(fs, "/reconfdat1", (short) 3); assertTrue("calls already made to MockPlacementPolicy", callCounter.get() == 0); // switch over to the mock placement policy cluster.getNameNode().reconfigureProperty("dfs.block.replicator.classname", "org.apache.hadoop.hdfs.server." + "namenode." + "TestNameNodeReconfigure$" + "MockPlacementPolicy"); // write some files with the mock placement policy util.createFiles(fs, "/reconfdat2", (short) 3); util.waitReplication(fs, "/reconfdat2", (short) 3); int callsMade1 = callCounter.get(); // check that calls were made to mock placement policy assertTrue("no calls made to MockPlacementPolicy", callsMade1 > 0); LOG.info("" + callsMade1 + " calls made to MockPlacementPolicy"); // now try to change it to a non-existent class try { cluster.getNameNode().reconfigureProperty("dfs.block.replicator.classname", "does.not.exist"); fail("ReconfigurationException expected"); } catch (RuntimeException expected) { assertTrue("exception should have cause", expected.getCause() != null); assertTrue("exception's cause should have cause", expected.getCause().getCause() != null); assertTrue( "ClassNotFoundException expected but got " + expected.getCause().getCause().getClass().getCanonicalName(), expected.getCause().getCause() instanceof ClassNotFoundException); } // write some files, they should still go to the mock placemeny policy util.createFiles(fs, "/reconfdat3", (short) 3); util.waitReplication(fs, "/reconfdat3", (short) 3); int callsMade2 = callCounter.get(); // check that more calls were made to mock placement policy assertTrue("no calls made to MockPlacementPolicy", callsMade2 > callsMade1); LOG.info("" + (callsMade2 - callsMade1) + " calls made to MockPlacementPolicy"); // now revert back to the default policy cluster.getNameNode().reconfigureProperty("dfs.block.replicator.classname", null); // write some files with the default block placement policy util.createFiles(fs, "/reconfdat4", (short) 3); util.waitReplication(fs, "/reconfdat4", (short) 3); // make sure that no more calls were made to mock placement policy assertTrue("more calls made to MockPlacementPolicy", callCounter.get() == callsMade2); util.cleanup(fs, "/reconfdat1"); util.cleanup(fs, "/reconfdat2"); util.cleanup(fs, "/reconfdat3"); util.cleanup(fs, "/reconfdat4"); } /** * Test that we can modify configuration properties. */ @Test public void testReconfigure() throws IOException, ReconfigurationException { setUp(); // change properties cluster.getNameNode().reconfigureProperty("dfs.heartbeat.interval", "" + 6); cluster.getNameNode().reconfigureProperty("heartbeat.recheck.interval", "" + (10 * 60 * 1000)); cluster.getNameNode().reconfigureProperty("dfs.persist.blocks", "true"); cluster.getNameNode().reconfigureProperty("dfs.permissions.audit.log", "true"); // try invalid values try { cluster.getNameNode().reconfigureProperty("dfs.heartbeat.interval", "text"); fail("ReconfigurationException expected"); } catch (ReconfigurationException expected) { } try { cluster.getNameNode().reconfigureProperty("heartbeat.recheck.interval", "text"); fail("ReconfigurationException expected"); } catch (ReconfigurationException expected) { } try { cluster.getNameNode().reconfigureProperty("dfs.persist.blocks", "text"); fail("ReconfigurationException expected"); } catch (ReconfigurationException expected) { } try { cluster.getNameNode().reconfigureProperty("dfs.permissions.audit.log", "text"); fail("ReconfigurationException expected"); } catch (ReconfigurationException expected) { } // verify change assertEquals("dfs.heartbeat.interval has wrong value", 6000L, cluster.getNameNode().namesystem.heartbeatInterval); assertEquals("heartbeat.recheck.interval has wrong value", 10 * 60 * 1000, cluster.getNameNode().namesystem.heartbeatRecheckInterval); assertTrue("dfs.persist.blocks has wrong value", cluster.getNameNode().namesystem.getPersistBlocks()); assertTrue("dfs.permissions.audit.log has wrong value", cluster.getNameNode().namesystem.getPermissionAuditLog()); // revert to defaults cluster.getNameNode().reconfigureProperty("dfs.heartbeat.interval", null); cluster.getNameNode().reconfigureProperty("heartbeat.recheck.interval", null); cluster.getNameNode().reconfigureProperty("dfs.persist.blocks", null); cluster.getNameNode().reconfigureProperty("dfs.permissions.audit.log", null); // verify defaults assertEquals("dfs.heartbeat.interval has wrong value", 3000L, cluster.getNameNode().namesystem.heartbeatInterval); assertEquals("heartbeat.recheck.interval has wrong value", 5 * 60 * 1000, cluster.getNameNode().namesystem.heartbeatRecheckInterval); assertFalse("dfs.persist.blocks has wrong value", cluster.getNameNode().namesystem.getPersistBlocks()); assertFalse("dfs.permissions.audit.log has wrong value", cluster.getNameNode().namesystem.getPermissionAuditLog()); } /** * Test that includes/excludes will be ignored * if dfs.ignore.missing.include.files is set */ @Test public void testIncludesExcludesConfigure() throws IOException { String inFile = "/tmp/inFileNotExists"; String exFile = "/tmp/exFileNotExists"; File include = new File(inFile); File exclude = new File(exFile); include.delete(); exclude.delete(); assertFalse(include.exists()); assertFalse(exclude.exists()); Configuration conf = new Configuration(); conf.set("dfs.hosts.ignoremissing", "true"); conf.set(FSConstants.DFS_HOSTS, inFile); conf.set("dfs.hosts.exclude", exFile); cluster = new MiniDFSCluster(conf, 3, true, null); } @After public void shutDown() throws IOException { if (fs != null) { fs.close(); } if (cluster != null) { cluster.shutdown(); } } }