org.apache.hadoop.yarn.server.nodemanager.TestLocalDirsHandlerService.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.yarn.server.nodemanager.TestLocalDirsHandlerService.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.yarn.server.nodemanager;

import java.io.File;
import java.io.IOException;

import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

public class TestLocalDirsHandlerService {
    private static final File testDir = new File("target", TestDirectoryCollection.class.getName())
            .getAbsoluteFile();
    private static final File testFile = new File(testDir, "testfile");

    @Before
    public void setup() throws IOException {
        testDir.mkdirs();
        testFile.createNewFile();
    }

    @After
    public void teardown() {
        FileUtil.fullyDelete(testDir);
    }

    @Test
    public void testDirStructure() throws Exception {
        Configuration conf = new YarnConfiguration();
        String localDir1 = new File("file:///" + testDir, "localDir1").getPath();
        conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir1);
        String logDir1 = new File("file:///" + testDir, "logDir1").getPath();
        conf.set(YarnConfiguration.NM_LOG_DIRS, logDir1);
        LocalDirsHandlerService dirSvc = new LocalDirsHandlerService();
        dirSvc.init(conf);
        Assert.assertEquals(1, dirSvc.getLocalDirs().size());
        dirSvc.close();
    }

    @Test
    public void testValidPathsDirHandlerService() throws Exception {
        Configuration conf = new YarnConfiguration();
        String localDir1 = new File("file:///" + testDir, "localDir1").getPath();
        String localDir2 = new File("hdfs:///" + testDir, "localDir2").getPath();
        conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir1 + "," + localDir2);
        String logDir1 = new File("file:///" + testDir, "logDir1").getPath();
        conf.set(YarnConfiguration.NM_LOG_DIRS, logDir1);
        LocalDirsHandlerService dirSvc = new LocalDirsHandlerService();
        try {
            dirSvc.init(conf);
            Assert.fail("Service should have thrown an exception due to wrong URI");
        } catch (YarnRuntimeException e) {
        }
        Assert.assertEquals("Service should not be inited", STATE.STOPPED, dirSvc.getServiceState());
        dirSvc.close();
    }

    @Test
    public void testGetFullDirs() throws Exception {
        Configuration conf = new YarnConfiguration();

        conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
        FileContext localFs = FileContext.getLocalFSFileContext(conf);

        String localDir1 = new File(testDir, "localDir1").getPath();
        String localDir2 = new File(testDir, "localDir2").getPath();
        String logDir1 = new File(testDir, "logDir1").getPath();
        String logDir2 = new File(testDir, "logDir2").getPath();
        Path localDir1Path = new Path(localDir1);
        Path logDir1Path = new Path(logDir1);
        FsPermission dirPermissions = new FsPermission((short) 0410);
        localFs.mkdir(localDir1Path, dirPermissions, true);
        localFs.mkdir(logDir1Path, dirPermissions, true);

        conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir1 + "," + localDir2);
        conf.set(YarnConfiguration.NM_LOG_DIRS, logDir1 + "," + logDir2);
        conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 0.0f);
        NodeManagerMetrics nm = NodeManagerMetrics.create();
        LocalDirsHandlerService dirSvc = new LocalDirsHandlerService(nm);
        dirSvc.init(conf);
        Assert.assertEquals(0, dirSvc.getLocalDirs().size());
        Assert.assertEquals(0, dirSvc.getLogDirs().size());
        Assert.assertEquals(1, dirSvc.getDiskFullLocalDirs().size());
        Assert.assertEquals(1, dirSvc.getDiskFullLogDirs().size());
        // check the metrics
        Assert.assertEquals(2, nm.getBadLocalDirs());
        Assert.assertEquals(2, nm.getBadLogDirs());
        Assert.assertEquals(0, nm.getGoodLocalDirsDiskUtilizationPerc());
        Assert.assertEquals(0, nm.getGoodLogDirsDiskUtilizationPerc());

        Assert.assertEquals("", dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOCAL_DIRS));
        Assert.assertEquals("", dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOG_DIRS));
        Assert.assertEquals(localDir1 + "," + localDir2, dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS));
        Assert.assertEquals(logDir1 + "," + logDir2, dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS));

        conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100.0f);
        nm = NodeManagerMetrics.create();
        dirSvc = new LocalDirsHandlerService(nm);
        dirSvc.init(conf);
        Assert.assertEquals(1, dirSvc.getLocalDirs().size());
        Assert.assertEquals(1, dirSvc.getLogDirs().size());
        Assert.assertEquals(0, dirSvc.getDiskFullLocalDirs().size());
        Assert.assertEquals(0, dirSvc.getDiskFullLogDirs().size());
        // check the metrics
        File dir = new File(localDir1);
        int utilizationPerc = (int) ((dir.getTotalSpace() - dir.getUsableSpace()) * 100 / dir.getTotalSpace());
        Assert.assertEquals(1, nm.getBadLocalDirs());
        Assert.assertEquals(1, nm.getBadLogDirs());
        Assert.assertEquals(utilizationPerc, nm.getGoodLocalDirsDiskUtilizationPerc());
        Assert.assertEquals(utilizationPerc, nm.getGoodLogDirsDiskUtilizationPerc());

        Assert.assertEquals(localDir2, dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOCAL_DIRS));
        Assert.assertEquals(logDir2, dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOG_DIRS));
        Assert.assertEquals(localDir1 + "," + localDir2, dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS));
        Assert.assertEquals(logDir1 + "," + logDir2, dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS));

        FileUtils.deleteDirectory(new File(localDir1));
        FileUtils.deleteDirectory(new File(localDir2));
        FileUtils.deleteDirectory(new File(logDir1));
        FileUtils.deleteDirectory(new File(logDir2));
        dirSvc.close();
    }
}