org.springframework.yarn.test.support.StandaloneYarnCluster.java Source code

Introduction

Here is the source code for org.springframework.yarn.test.support.StandaloneYarnCluster.java

Source

/*
 * Copyright 2013 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.springframework.yarn.test.support;

import java.io.File;
import java.io.IOException;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.springframework.yarn.test.context.YarnCluster;

/**
 * Standalone simple mini cluster having Yarn
 * and Hdfs nodes.
 * 
 * @author Janne Valkealahti
 *
 */
public class StandaloneYarnCluster implements YarnCluster {

    private final static Log log = LogFactory.getLog(StandaloneYarnCluster.class);

    /** Yarn specific mini cluster */
    private MiniYARNCluster yarnCluster = null;

    /** Hdfs specific mini cluster */
    private MiniDFSCluster dfsCluster = null;

    /** Unique cluster name */
    private final String clusterName;

    /** Configuration build at runtime */
    private Configuration configuration;

    /** Monitor sync for start and stop */
    private final Object startupShutdownMonitor = new Object();

    /** Flag for cluster state */
    private boolean started;

    /** Number of nodes for yarn and dfs */
    private int nodes = 1;

    /**
     * Instantiates a mini cluster with default
     * cluster node count.
     * 
     * @param clusterName the unique cluster name
     */
    public StandaloneYarnCluster(String clusterName) {
        this.clusterName = clusterName;
    }

    /**
     * Instantiates a mini cluster with given
     * cluster node count.
     * 
     * @param clusterName the unique cluster name
     * @param nodes the node count
     */
    public StandaloneYarnCluster(String clusterName, int nodes) {
        this.clusterName = clusterName;
        this.nodes = nodes;
    }

    @Override
    public Configuration getConfiguration() {
        return configuration;
    }

    @Override
    public void start() throws IOException {
        log.info("Checking if cluster=" + clusterName + " needs to be started");
        synchronized (this.startupShutdownMonitor) {
            if (started) {
                return;
            }
            log.info("Starting cluster=" + clusterName);
            configuration = new YarnConfiguration();
            configuration.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "target/" + clusterName + "-dfs");

            dfsCluster = new MiniDFSCluster.Builder(configuration).numDataNodes(nodes).build();

            yarnCluster = new MiniYARNCluster(clusterName, nodes, 1, 1);
            yarnCluster.init(configuration);
            yarnCluster.start();

            log.info("Started cluster=" + clusterName);
            started = true;
        }
    }

    @Override
    public void stop() {
        log.info("Checking if cluster=" + clusterName + " needs to be stopped");
        synchronized (this.startupShutdownMonitor) {
            if (!started) {
                return;
            }
            if (yarnCluster != null) {
                yarnCluster.stop();
                yarnCluster = null;
            }
            if (dfsCluster != null) {
                dfsCluster.shutdown();
                dfsCluster = null;
            }
            log.info("Stopped cluster=" + clusterName);
            started = false;
        }
    }

    @Override
    public File getYarnWorkDir() {
        return yarnCluster != null ? yarnCluster.getTestWorkDir() : null;
    }

    /**
     * Sets a number of nodes for cluster. Every node
     * will act as yarn and dfs role. Default is one node.
     * 
     * @param nodes the number of nodes
     */
    public void setNodes(int nodes) {
        this.nodes = nodes;
    }

}