org.apache.hadoop.hdfs.TestCloseFile.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hdfs.TestCloseFile.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hdfs;

import java.util.Random;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;

import org.junit.After;
import static org.junit.Assert.*;
import org.junit.Before;
import org.junit.Test;

public class TestCloseFile {
    private MiniDFSCluster cluster;
    private Configuration conf;
    private static int BLOCK_SIZE = 1024;
    private static int BLOCKS = 20;
    private static int FILE_LEN = BLOCK_SIZE * BLOCKS;
    private FileSystem fs;
    private static Random random = new Random();
    private static volatile boolean pass = true;
    private static Log LOG = LogFactory.getLog(TestCloseFile.class);
    private static int CLOSE_FILE_TIMEOUT = 20000;

    @Before
    public void setUp() throws Exception {
        conf = new Configuration();
        conf.setBoolean("dfs.permissions", false);
        conf.setInt("dfs.client.closefile.timeout", CLOSE_FILE_TIMEOUT);
        conf.setInt("dfs.close.replication.min", 2);
        conf.setInt("dfs.replication.pending.timeout.sec", 1);
        cluster = new MiniDFSCluster(conf, 3, true, null);
        fs = cluster.getFileSystem();
    }

    @After
    public void tearDown() throws Exception {
        if (fs != null) {
            fs.close();
        }
        if (cluster != null) {
            cluster.shutdown();
        }
    }

    @Test
    public void testCloseReplicationWithNoInValidReplicas() throws Exception {
        testCloseReplication(false);
    }

    @Test
    public void testCloseReplicationWithOneInValidReplicas() throws Exception {
        testCloseReplication(true);
    }

    /**
     * Test that close.replication.min of 2 works
     * @throws Exception
     */
    private void testCloseReplication(boolean restartDN) throws Exception {
        String file = "/testRestartDatanodeNode";

        // Create a file with a replication factor of 2 and write data.
        final FSDataOutputStream out = fs.create(new Path(file), (short) 2);
        byte[] buffer = new byte[FILE_LEN];
        random.nextBytes(buffer);
        out.write(buffer);
        ((DFSOutputStream) out.getWrappedStream()).sync();

        // take down one datanode in the pipeline
        DatanodeInfo[] locs = cluster.getNameNode().getBlockLocations(file, 0, FILE_LEN).getLocatedBlocks().get(0)
                .getLocations();
        assertEquals(2, locs.length);
        DatanodeInfo loc = locs[0];
        DataNodeProperties dnprop = cluster.stopDataNode(loc.getName());
        cluster.getNameNode().namesystem.removeDatanode(loc);
        out.write(buffer);
        ((DFSOutputStream) out.getWrappedStream()).sync();
        // the pipeline length drops to 1
        assertEquals(1, (((DFSOutputStream) out.getWrappedStream()).getNumCurrentReplicas()));

        if (restartDN) {
            // restart introduces an invalid replica
            cluster.restartDataNode(dnprop);
            // the number of targets becomes 2 with 1 invalid replica
            locs = cluster.getNameNode().getBlockLocations(file, 0, FILE_LEN).getLocatedBlocks().get(0)
                    .getLocations();
            assertEquals(2, locs.length);
        }

        //  now close the file
        Thread closeThread = new Thread() {
            public void run() {
                try {
                    out.close();
                    pass = true;
                    LOG.info("File closed");
                } catch (Exception e) {
                    pass = false;
                }
            }
        };
        closeThread.run();

        // wait until the last block is replicated and the file is closed
        closeThread.join(CLOSE_FILE_TIMEOUT);
        assertTrue(pass);
    }

    public void testRestartNameNode(boolean waitSafeMode) throws Exception {
        String file = "/testRestartNameNode" + waitSafeMode;

        // Create a file and write data.
        FSDataOutputStream out = fs.create(new Path(file));
        String clientName = ((DistributedFileSystem) fs).getClient().getClientName();
        byte[] buffer = new byte[FILE_LEN];
        random.nextBytes(buffer);
        out.write(buffer);
        ((DFSOutputStream) out.getWrappedStream()).sync();

        // Now shutdown the namenode and try to close the file.
        cluster.shutdownNameNode(0);
        Thread closeThread = new CloseThread(out, file, clientName);
        closeThread.start();
        Thread.sleep(CLOSE_FILE_TIMEOUT / 4);

        // Restart the namenode and verify the close file worked.
        if (!waitSafeMode) {
            cluster.restartNameNode(0, new String[] {}, false);
            cluster.getNameNode().setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
        } else {
            cluster.restartNameNode(0);
        }
        closeThread.join(5000);
        assertTrue(pass);
    }

    @Test
    public void testRestartNameNodeWithSafeMode() throws Exception {
        testRestartNameNode(true);
    }

    @Test
    public void testRestartNameNodeWithoutSafeMode() throws Exception {
        testRestartNameNode(false);
    }

    private class CloseThread extends Thread {
        private final FSDataOutputStream out;
        private final String file;
        private final String clientName;

        public CloseThread(FSDataOutputStream out, String file, String clientName) {
            this.out = out;
            this.file = file;
            this.clientName = clientName;
        }

        public void run() {
            try {
                out.close();
            } catch (Exception e) {
                LOG.warn("Close failed", e);
                while (true) {
                    try {
                        Thread.sleep(1000);
                        cluster.getNameNode().recoverLease(file, clientName);
                        break;
                    } catch (SafeModeException se) {
                        LOG.warn("Retrying lease recovery for failed close", se);
                    } catch (Exception ee) {
                        LOG.warn("Lease recovery failed", ee);
                        pass = false;
                    }
                }
            }
        }
    }
}