Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hdfs.server.namenode; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Random; import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.FSConstants; import org.junit.After; import org.junit.Before; import junit.framework.TestCase; public class TestReplicationPolicy extends TestCase { final static Log LOG = LogFactory.getLog(TestReplicationPolicy.class); private static final int BLOCK_SIZE = 1024; private static final int NUM_OF_DATANODES = 8; private static final String filename = "/dummyfile.txt"; private Configuration CONF; private NetworkTopology cluster; private NameNode namenode; private BlockPlacementPolicy replicator; private DatanodeDescriptor dataNodes[]; private DatanodeDescriptor NODE = new DatanodeDescriptor(new DatanodeID("h7:5020"), "/d2/r4"); @Before public void setUp() throws IOException { CONF = new Configuration(); dataNodes = new DatanodeDescriptor[] { new DatanodeDescriptor(new DatanodeID("h1:5020"), "/d1/r1"), new DatanodeDescriptor(new DatanodeID("h2:5020"), "/d1/r1"), new DatanodeDescriptor(new DatanodeID("h3:5020"), "/d1/r2"), new DatanodeDescriptor(new DatanodeID("h4:5020"), "/d1/r2"), new DatanodeDescriptor(new DatanodeID("h5:5020"), "/d2/r3"), new DatanodeDescriptor(new DatanodeID("h6:5020"), "/d2/r3"), new DatanodeDescriptor(new DatanodeID("h7:5020"), "/d1/r2"), new DatanodeDescriptor(new DatanodeID("h8:5020"), "/d1/r1") }; FileSystem.setDefaultUri(CONF, "hdfs://localhost:0"); CONF.set("dfs.http.address", "0.0.0.0:0"); NameNode.format(CONF); namenode = new NameNode(CONF); FSNamesystem fsNamesystem = namenode.getNamesystem(); replicator = fsNamesystem.replicator; cluster = fsNamesystem.clusterMap; // construct network topology for (int i = 0; i < NUM_OF_DATANODES; i++) { cluster.add(dataNodes[i]); } for (int i = 0; i < NUM_OF_DATANODES; i++) { dataNodes[i].updateHeartbeat(2 * FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 2 * FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 0); } } @After public void tearDown() { if (namenode != null) { namenode.stop(); } } /** * In this testcase, client is dataNodes[0]. So the 1st replica should be * placed on dataNodes[0], the 2nd replica should be placed on * different rack and third should be placed on different node * of rack chosen for 2nd node. * The only excpetion is when the <i>numOfReplicas</i> is 2, * the 1st is on dataNodes[0] and the 2nd is on a different rack. * @throws Exception */ public void testChooseTarget1() throws Exception { dataNodes[0].updateHeartbeat(2 * FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 4); // overloaded try { DatanodeDescriptor[] targets; targets = replicator.chooseTarget(filename, 0, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 0); targets = replicator.chooseTarget(filename, 1, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 1); assertEquals(targets[0], dataNodes[0]); targets = replicator.chooseTarget(filename, 2, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 2); assertEquals(targets[0], dataNodes[0]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); targets = replicator.chooseTarget(filename, 3, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 3); assertEquals(targets[0], dataNodes[0]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); targets = replicator.chooseTarget(filename, 4, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 4); assertEquals(targets[0], dataNodes[0]); assertTrue( cluster.isOnSameRack(targets[1], targets[2]) || cluster.isOnSameRack(targets[2], targets[3])); assertFalse(cluster.isOnSameRack(targets[0], targets[2])); } finally { dataNodes[0].updateHeartbeat(2 * FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 0); } } /** * Test if the network topology correctly keeps track of racks. * Test both adding and removing nodes. */ public void testRacksList() { int datanodeCount = 1000; Random r = new Random(); Collection<String> locations = new HashSet<String>(); for (int i = 0; i < NUM_OF_DATANODES; i++) { // remove original datanodes cluster.remove(dataNodes[i]); } // no racks in the topology assertEquals(0, cluster.getRacks().size()); assertEquals(0, cluster.getNumOfRacks()); dataNodes = new DatanodeDescriptor[datanodeCount]; for (int i = 0; i < datanodeCount; i++) { String did = "h" + i + ":5020"; String loc = "/d" + r.nextInt(20) + "/r" + r.nextInt(20); locations.add(loc); dataNodes[i] = new DatanodeDescriptor(new DatanodeID(did), loc); // add new datanodes cluster.add(dataNodes[i]); // make sure the racks are correct List<String> racks = cluster.getRacks(); for (String rack : racks) { assertTrue(locations.contains(rack)); } assertEquals(locations.size(), racks.size()); } for (int i = 0; i < datanodeCount; i++) { cluster.remove(dataNodes[i]); Collection<String> locationsLeft = locationsLeft(i); // make sure the racks are correct List<String> racks = cluster.getRacks(); for (String rack : racks) { assertTrue(locationsLeft.contains(rack)); } assertEquals(locationsLeft.size(), racks.size()); } } /** * Test that the list of racks does not contain * duplicates. */ public void testRacksListOneRack() { for (int i = 0; i < NUM_OF_DATANODES; i++) { // remove original datanodes cluster.remove(dataNodes[i]); } assertEquals(0, cluster.getNumOfRacks()); dataNodes = new DatanodeDescriptor[10]; for (int i = 0; i < 10; i++) { String did = "h" + i + ":5020"; String loc = "/d0/r0"; dataNodes[i] = new DatanodeDescriptor(new DatanodeID(did), loc); // add new datanodes cluster.add(dataNodes[i]); // make sure we have only one rack assertEquals(1, cluster.getNumOfRacks()); assertEquals(loc, cluster.getRacks().get(0)); } } /** * Return all locations of datanodes starting at * removedUpToIndex + 1 index */ private Collection<String> locationsLeft(int removedUpToIndex) { Collection<String> locationsLeft = new HashSet<String>(); for (int i = removedUpToIndex + 1; i < dataNodes.length; i++) { locationsLeft.add(dataNodes[i].getNetworkLocation()); } return locationsLeft; } /** * Test NetworkTopology.chooseRack(Set<String>) */ public void testChooseRack() throws Exception { // for this test create 100 datanodes Random r = new Random(); Collection<String> locations = new HashSet<String>(); for (int i = 0; i < NUM_OF_DATANODES; i++) { // remove original datanodes cluster.remove(dataNodes[i]); } dataNodes = new DatanodeDescriptor[100]; for (int i = 0; i < 100; i++) { String did = "h" + i + ":5020"; String loc = "/d" + r.nextInt(5) + "/r" + r.nextInt(5); locations.add(loc); dataNodes[i] = new DatanodeDescriptor(new DatanodeID(did), loc); // add new datanodes cluster.add(dataNodes[i]); } LOG.info("Locations : " + locations.size() + " racks in total"); int numOfDatanodes = dataNodes.length; Set<String> excludedRacks = new HashSet<String>(); // randomly exclude racks for (int i = 0; i < 20; i++) { excludedRacks.clear(); int numToExclude = r.nextInt(locations.size() - 1) + 1; LOG.info("Iteration : " + i + " will exclude: " + numToExclude); // exclude a random number of racks while (excludedRacks.size() < numToExclude) { excludedRacks.add(dataNodes[r.nextInt(numOfDatanodes)].getNetworkLocation()); } LOG.info("Excluded racks: " + excludedRacks); String chosenNode = cluster.chooseRack(excludedRacks); LOG.info("Chosen node: " + chosenNode + " should not be null"); // chosen node cannot be in the excluded list for (String exluded : excludedRacks) { assertFalse(exluded.equals(chosenNode)); } // if the number of locations in the excluded list was less that the // total number of location, then we should not get null if (locations.size() > excludedRacks.size()) { assertNotNull(chosenNode); } } // exclude all nodes, and we should get null LOG.info("Excluding all nodes"); excludedRacks.clear(); for (int n = 0; n < numOfDatanodes; n++) { excludedRacks.add(dataNodes[n].getNetworkLocation()); } String chosenNode = cluster.chooseRack(excludedRacks); LOG.info("Chosen node: " + chosenNode + " should be null"); assertNull(chosenNode); } /** * In this testcase, client is dataNodes[0], but the dataNodes[1] is * not allowed to be chosen. So the 1st replica should be * placed on dataNodes[0], the 2nd replica should be placed on a different * rack, the 3rd should be on same rack as the 2nd replica, and the rest * should be placed on a third rack. * @throws Exception */ public void testChooseTarget2() throws Exception { List<Node> excludedNodes; DatanodeDescriptor[] targets; BlockPlacementPolicyDefault replicator1 = (BlockPlacementPolicyDefault) replicator; List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>(); excludedNodes = new ArrayList<Node>(); excludedNodes.add(dataNodes[1]); targets = replicator1.chooseTarget(0, dataNodes[0], chosenNodes, excludedNodes, BLOCK_SIZE); assertEquals(targets.length, 0); excludedNodes.clear(); chosenNodes.clear(); excludedNodes.add(dataNodes[1]); targets = replicator1.chooseTarget(1, dataNodes[0], chosenNodes, excludedNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertEquals(targets[0], dataNodes[0]); excludedNodes.clear(); chosenNodes.clear(); excludedNodes.add(dataNodes[1]); targets = replicator1.chooseTarget(2, dataNodes[0], chosenNodes, excludedNodes, BLOCK_SIZE); assertEquals(targets.length, 2); assertEquals(targets[0], dataNodes[0]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); excludedNodes.clear(); chosenNodes.clear(); excludedNodes.add(dataNodes[1]); targets = replicator1.chooseTarget(3, dataNodes[0], chosenNodes, excludedNodes, BLOCK_SIZE); assertEquals(targets.length, 3); assertEquals(targets[0], dataNodes[0]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); excludedNodes.clear(); chosenNodes.clear(); excludedNodes.add(dataNodes[1]); excludedNodes.add(dataNodes[6]); excludedNodes.add(dataNodes[7]); targets = replicator1.chooseTarget(4, dataNodes[0], chosenNodes, excludedNodes, BLOCK_SIZE); assertEquals(targets.length, 4); assertEquals(targets[0], dataNodes[0]); for (int i = 1; i < 4; i++) { assertFalse(cluster.isOnSameRack(targets[0], targets[i])); } assertTrue(cluster.isOnSameRack(targets[1], targets[2]) || cluster.isOnSameRack(targets[2], targets[3])); assertFalse(cluster.isOnSameRack(targets[1], targets[3])); } /** * In this testcase, client is dataNodes[0], but dataNodes[0] is not qualified * to be chosen. So the 1st replica should be placed on dataNodes[1], * the 2nd replica should be placed on a different rack, * the 3rd replica should be placed on the same rack as the 2nd replica, * and the rest should be placed on the third rack. * @throws Exception */ public void testChooseTarget3() throws Exception { // make data node 0, 6 & 7 to be not qualified to choose dataNodes[0].updateHeartbeat(2 * FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, (FSConstants.MIN_BLOCKS_FOR_WRITE - 1) * BLOCK_SIZE, 0L, 0); // no space dataNodes[6].updateHeartbeat(2 * FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, (FSConstants.MIN_BLOCKS_FOR_WRITE - 1) * BLOCK_SIZE, 0L, 0); // no space dataNodes[7].updateHeartbeat(2 * FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, (FSConstants.MIN_BLOCKS_FOR_WRITE - 1) * BLOCK_SIZE, 0L, 0); // no space try { DatanodeDescriptor[] targets; targets = replicator.chooseTarget(filename, 0, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 0); targets = replicator.chooseTarget(filename, 1, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 1); assertTrue(targets[0].equals(dataNodes[1]) || targets[0].equals(dataNodes[7])); targets = replicator.chooseTarget(filename, 2, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 2); assertTrue(targets[0].equals(dataNodes[1]) || targets[0].equals(dataNodes[7])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); targets = replicator.chooseTarget(filename, 3, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 3); assertTrue(targets[0].equals(dataNodes[1]) || targets[0].equals(dataNodes[7])); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); targets = replicator.chooseTarget(filename, 4, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 4); assertTrue(targets[0].equals(dataNodes[1]) || targets[0].equals(dataNodes[7])); for (int i = 1; i < 4; i++) { assertFalse(cluster.isOnSameRack(targets[0], targets[i])); } assertTrue( cluster.isOnSameRack(targets[1], targets[2]) || cluster.isOnSameRack(targets[2], targets[3])); assertFalse(cluster.isOnSameRack(targets[1], targets[3])); } finally { dataNodes[0].updateHeartbeat(2 * FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 0); dataNodes[6].updateHeartbeat(2 * FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 0); dataNodes[7].updateHeartbeat(2 * FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 0); } } /** * In this testcase, client is dataNodes[4], but none of the nodes on rack 2 * is qualified to be chosen. So the 1st replica should be placed on either * rack 1 or rack 3. * the 2nd replica should be placed on a different rack, * the 3rd replica should be placed on the same rack as the 1st replica, * @throws Exception */ public void testChoooseTarget4() throws Exception { // make data node 4 & 5 to be not qualified to choose: not enough disk space for (int i = 4; i < 6; i++) { dataNodes[i].updateHeartbeat(2 * FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, (FSConstants.MIN_BLOCKS_FOR_WRITE - 1) * BLOCK_SIZE, 0L, 0); } try { DatanodeDescriptor[] targets; targets = replicator.chooseTarget(filename, 0, dataNodes[0], BLOCK_SIZE); assertEquals(targets.length, 0); targets = replicator.chooseTarget(filename, 1, dataNodes[4], BLOCK_SIZE); assertEquals(targets.length, 1); assertFalse(cluster.isOnSameRack(targets[0], dataNodes[4])); targets = replicator.chooseTarget(filename, 2, dataNodes[4], BLOCK_SIZE); assertEquals(targets.length, 2); assertFalse(cluster.isOnSameRack(targets[0], dataNodes[4])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); targets = replicator.chooseTarget(filename, 3, dataNodes[4], BLOCK_SIZE); assertEquals(targets.length, 3); for (int i = 0; i < 3; i++) { assertFalse(cluster.isOnSameRack(targets[i], dataNodes[4])); } assertTrue( cluster.isOnSameRack(targets[0], targets[1]) || cluster.isOnSameRack(targets[1], targets[2])); assertFalse(cluster.isOnSameRack(targets[0], targets[2])); } finally { for (int i = 0; i < 2; i++) { dataNodes[i].updateHeartbeat(2 * FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, FSConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 0); } } } /** * In this testcase, client is is a node outside of file system. * So the 1st replica can be placed on any node. * the 2nd replica should be placed on a different rack, * the 3rd replica should be placed on the same rack as the 2nd replica, * @throws Exception */ public void testChooseTarget5() throws Exception { DatanodeDescriptor[] targets; targets = replicator.chooseTarget(filename, 0, NODE, BLOCK_SIZE); assertEquals(targets.length, 0); targets = replicator.chooseTarget(filename, 1, NODE, BLOCK_SIZE); assertEquals(targets.length, 1); targets = replicator.chooseTarget(filename, 2, NODE, BLOCK_SIZE); assertEquals(targets.length, 2); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); targets = replicator.chooseTarget(filename, 3, NODE, BLOCK_SIZE); assertEquals(targets.length, 3); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); } /** * This testcase tests re-replication, when dataNodes[0] is already chosen. * So the 1st replica can be placed on random rack. * the 2nd replica should be placed on different node by same rack as * the 1st replica. The 3rd replica can be placed randomly. * @throws Exception */ public void testRereplicate1() throws Exception { List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>(); chosenNodes.add(dataNodes[0]); DatanodeDescriptor[] targets; targets = replicator.chooseTarget(filename, 0, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 0); targets = replicator.chooseTarget(filename, 1, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); targets = replicator.chooseTarget(filename, 2, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 2); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); targets = replicator.chooseTarget(filename, 3, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 3); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); assertFalse(cluster.isOnSameRack(targets[0], targets[2])); } /** * This testcase tests re-replication, * when dataNodes[0] and dataNodes[1] are already chosen. * So the 1st replica should be placed on a different rack than rack 1. * the rest replicas can be placed randomly, * @throws Exception */ public void testRereplicate2() throws Exception { List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>(); chosenNodes.add(dataNodes[0]); chosenNodes.add(dataNodes[1]); DatanodeDescriptor[] targets; targets = replicator.chooseTarget(filename, 0, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 0); targets = replicator.chooseTarget(filename, 1, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); chosenNodes.add(dataNodes[7]); targets = replicator.chooseTarget(filename, 2, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 2); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[1])); } /** * This testcase tests re-replication, * when dataNodes[0] and dataNodes[2] are already chosen. * So the 1st replica should be placed on the rack that the writer resides. * the rest replicas can be placed randomly, * @throws Exception */ public void testRereplicate3() throws Exception { List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>(); chosenNodes.add(dataNodes[0]); chosenNodes.add(dataNodes[2]); DatanodeDescriptor[] targets; targets = replicator.chooseTarget(filename, 0, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 0); targets = replicator.chooseTarget(filename, 1, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); assertFalse(cluster.isOnSameRack(dataNodes[2], targets[0])); targets = replicator.chooseTarget(filename, 1, dataNodes[2], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertTrue(cluster.isOnSameRack(dataNodes[2], targets[0])); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); targets = replicator.chooseTarget(filename, 2, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 2); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); targets = replicator.chooseTarget(filename, 2, dataNodes[2], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 2); assertTrue(cluster.isOnSameRack(dataNodes[2], targets[0])); } /** * This testcase tests decommission, when dataNodes[0] is being decommissioned. * So the 1st replica can be placed on the same rack. * the 2nd replica should be placed on a different rack as * the 1st replica. The 3rd replica is the same rack as the 2nd replica. * @throws Exception */ public void testDecommission1() throws Exception { List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>(); dataNodes[0].startDecommission(); chosenNodes.add(dataNodes[0]); DatanodeDescriptor[] targets; try { targets = replicator.chooseTarget(filename, 0, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 0); targets = replicator.chooseTarget(filename, 1, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); targets = replicator.chooseTarget(filename, 2, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 2); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); targets = replicator.chooseTarget(filename, 3, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 3); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[1])); assertFalse(cluster.isOnSameRack(targets[1], targets[2])); } finally { dataNodes[0].stopDecommission(); } } /** * This testcase tests decommision, * when dataNodes[0] and dataNodes[1] are already chosen * and dataNodes[0] is being decommissioned. * So the 1st replica should be placed on a different rack than rack 1. * and the 2nd replica should be placed on the same rack as rack 1. * the rest replicas can be placed randomly, * @throws Exception */ public void testDecommission2() throws Exception { List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>(); dataNodes[0].startDecommission(); chosenNodes.add(dataNodes[0]); chosenNodes.add(dataNodes[1]); DatanodeDescriptor[] targets; try { targets = replicator.chooseTarget(filename, 0, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 0); targets = replicator.chooseTarget(filename, 1, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); targets = replicator.chooseTarget(filename, 2, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 2); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[1])); } finally { dataNodes[0].stopDecommission(); } } /** * This testcase tests re-replication, * when dataNodes[0] and dataNodes[2] are already chosen. * So the 1st replica should be placed on the rack that the writer resides. * the rest replicas can be placed randomly, * @throws Exception */ public void testDecommion3() throws Exception { List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>(); chosenNodes.add(dataNodes[0]); chosenNodes.add(dataNodes[2]); DatanodeDescriptor[] targets; dataNodes[0].startDecommission(); try { targets = replicator.chooseTarget(filename, 0, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 0); targets = replicator.chooseTarget(filename, 1, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); assertFalse(cluster.isOnSameRack(dataNodes[2], targets[0])); targets = replicator.chooseTarget(filename, 1, dataNodes[2], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertFalse(cluster.isOnSameRack(dataNodes[2], targets[0])); targets = replicator.chooseTarget(filename, 2, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 2); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); targets = replicator.chooseTarget(filename, 2, dataNodes[2], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 2); assertTrue(cluster.isOnSameRack(dataNodes[2], targets[0])); } finally { dataNodes[0].stopDecommission(); } } /** * This testcase tests decommission * when dataNodes[0] and dataNodes[2] and dataNodes[3] are already chosen. * @throws Exception */ public void testDecommion4() throws Exception { List<DatanodeDescriptor> chosenNodes = new ArrayList<DatanodeDescriptor>(); chosenNodes.add(dataNodes[0]); chosenNodes.add(dataNodes[2]); chosenNodes.add(dataNodes[3]); DatanodeDescriptor[] targets; dataNodes[0].startDecommission(); try { targets = replicator.chooseTarget(filename, 1, dataNodes[0], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); } finally { dataNodes[0].stopDecommission(); } dataNodes[2].startDecommission(); try { targets = replicator.chooseTarget(filename, 1, dataNodes[2], chosenNodes, BLOCK_SIZE); assertEquals(targets.length, 1); assertTrue(cluster.isOnSameRack(dataNodes[2], targets[0])); } finally { dataNodes[0].stopDecommission(); } } }