org.elasticsearch.hadoop.integration.yarn.YarnTestCluster.java Source code

Introduction

Here is the source code for org.elasticsearch.hadoop.integration.yarn.YarnTestCluster.java

Source

/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.elasticsearch.hadoop.integration.yarn;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.junit.rules.ExternalResource;

public class YarnTestCluster extends ExternalResource {

    private MiniYARNCluster yarnCluster = null;
    private MiniDFSCluster dfsCluster = null;

    private int nodes = 1;
    private final String CLUSTER_NAME = "es-yarn-cluster";

    private String tempDir;

    private YarnConfiguration cfg;

    YarnTestCluster(YarnConfiguration cfg) {
        this(cfg, 1);
    }

    YarnTestCluster(YarnConfiguration cfg, int nodes) {
        this.cfg = cfg;
        this.nodes = nodes;
    }

    MiniYARNCluster yarnCluster() {
        return yarnCluster;
    }

    DistributedFileSystem fs() {
        try {
            return dfsCluster.getFileSystem();
        } catch (IOException ex) {
            throw new IllegalStateException(ex);
        }
    }

    @Override
    protected void before() throws Throwable {
        if (yarnCluster != null) {
            return;
        }

        System.out.println("Starting YARN cluster...");

        //tempDir = System.getProperty("java.io.tmpdir");
        //System.setProperty("java.io.tmpdir", "/tmp-yarn");

        //        FileContext.DEFAULT_PERM.fromShort((short) 777);
        //        FileContext.DIR_DEFAULT_PERM.fromShort((short) 777);
        //        FileContext.FILE_DEFAULT_PERM.fromShort((short) 777);
        //ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION.fromShort((short) 777);

        // make sure to use IP discovery
        Configuration.addDefaultResource("yarn-test-site.xml");

        cfg.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "build/es-yarn/" + CLUSTER_NAME + "-dfs");

        dfsCluster = new MiniDFSCluster.Builder(cfg).numDataNodes(nodes).build();
        System.out.println("Started DFS cluster...");

        cfg.set(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, "true");
        yarnCluster = new MiniYARNCluster(CLUSTER_NAME, nodes, 1, 1);
        yarnCluster.init(cfg);
        yarnCluster.start();
        System.out.println("Started YARN cluster...");
    }

    @Override
    protected void after() {
        //System.setProperty("java.io.tmpdir", tempDir);

        if (yarnCluster != null) {
            System.out.println("Stopping YARN cluster...");
            // same as close but without the IOException
            yarnCluster.stop();
            yarnCluster = null;
        }

        if (dfsCluster != null) {
            System.out.println("Stopping DFS cluster...");
            dfsCluster.shutdown();
            dfsCluster = null;
        }
    }
}