org.apache.slider.client.SliderClient.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.slider.client.SliderClient.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/

package org.apache.slider.client;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.types.RegistryPathStatus;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.registry.client.api.RegistryOperations;

import static org.apache.hadoop.registry.client.binding.RegistryUtils.*;

import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.hadoop.registry.client.exceptions.NoRecordException;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.api.proto.Messages;
import org.apache.slider.common.Constants;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.params.AbstractActionArgs;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.params.ActionDiagnosticArgs;
import org.apache.slider.common.params.ActionExistsArgs;
import org.apache.slider.common.params.ActionInstallKeytabArgs;
import org.apache.slider.common.params.ActionInstallPackageArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
import org.apache.slider.common.params.ActionCreateArgs;
import org.apache.slider.common.params.ActionEchoArgs;
import org.apache.slider.common.params.ActionFlexArgs;
import org.apache.slider.common.params.ActionFreezeArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionListArgs;
import org.apache.slider.common.params.ActionLookupArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionResolveArgs;
import org.apache.slider.common.params.ActionStatusArgs;
import org.apache.slider.common.params.ActionThawArgs;
import org.apache.slider.common.params.Arguments;
import org.apache.slider.common.params.ClientArgs;
import org.apache.slider.common.params.CommonArgs;
import org.apache.slider.common.params.LaunchArgsAccessor;
import org.apache.slider.common.tools.ConfigHelper;
import org.apache.slider.common.tools.Duration;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.common.tools.SliderVersionInfo;
import org.apache.slider.core.build.InstanceBuilder;
import org.apache.slider.core.build.InstanceIO;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.conf.ResourcesInputPropertiesValidator;
import org.apache.slider.core.conf.TemplateInputPropertiesValidator;
import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.NotFoundException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
import org.apache.slider.core.exceptions.WaitTimeoutException;
import org.apache.slider.core.launch.AppMasterLauncher;
import org.apache.slider.core.launch.ClasspathConstructor;
import org.apache.slider.core.launch.CommandLineBuilder;
import org.apache.slider.core.launch.JavaCommandLineBuilder;
import org.apache.slider.core.launch.LaunchedApplication;
import org.apache.slider.core.launch.RunningApplication;
import org.apache.slider.core.launch.SerializedApplicationReport;
import org.apache.slider.core.main.RunService;
import org.apache.slider.core.persist.ApplicationReportSerDeser;
import org.apache.slider.core.persist.ConfPersister;
import org.apache.slider.core.persist.LockAcquireFailedException;
import org.apache.slider.core.registry.SliderRegistryUtils;
import org.apache.slider.core.registry.YarnAppListClient;
import org.apache.slider.core.registry.docstore.ConfigFormat;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
import org.apache.slider.core.registry.docstore.PublishedExports;
import org.apache.slider.core.registry.docstore.PublishedExportsOutputter;
import org.apache.slider.core.registry.docstore.PublishedExportsSet;
import org.apache.slider.core.registry.retrieve.RegistryRetriever;
import org.apache.slider.core.zk.BlockingZKWatcher;
import org.apache.slider.core.zk.ZKIntegration;
import org.apache.slider.core.zk.ZKPathBuilder;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.SliderProviderFactory;
import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.rpc.RpcBinder;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.io.StringWriter;
import java.io.Writer;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.regex.Pattern;

import static org.apache.slider.common.params.SliderActions.*;

/**
 * Client service for Slider
 */

public class SliderClient extends AbstractSliderLaunchedService
        implements RunService, SliderExitCodes, SliderKeys, ErrorStrings, SliderClientAPI {
    private static final Logger log = LoggerFactory.getLogger(SliderClient.class);

    private ClientArgs serviceArgs;
    public ApplicationId applicationId;

    private String deployedClusterName;
    /**
     * Cluster opaerations against the deployed cluster -will be null
     * if no bonding has yet taken place
     */
    private SliderClusterOperations sliderClusterOperations;

    protected SliderFileSystem sliderFileSystem;

    /**
     * Yarn client service
     */
    private SliderYarnClientImpl yarnClient;
    private YarnAppListClient YarnAppListClient;
    private AggregateConf launchedInstanceDefinition;

    /**
     * The YARN registry service
     */
    private RegistryOperations registryOperations;

    /**
     * Constructor
     */
    public SliderClient() {
        super("Slider Client");
        new HdfsConfiguration();
        new YarnConfiguration();
    }

    /**
     * This is called <i>Before serviceInit is called</i>
     * @param config the initial configuration build up by the
     * service launcher.
     * @param args argument list list of arguments passed to the command line
     * after any launcher-specific commands have been stripped.
     * @return the post-binding configuration to pass to the <code>init()</code>
     * operation.
     * @throws Exception
     */
    @Override
    public Configuration bindArgs(Configuration config, String... args) throws Exception {
        config = super.bindArgs(config, args);
        serviceArgs = new ClientArgs(args);
        serviceArgs.parse();
        // yarn-ify
        YarnConfiguration yarnConfiguration = new YarnConfiguration(config);
        return SliderUtils.patchConfiguration(yarnConfiguration);
    }

    @Override
    protected void serviceInit(Configuration conf) throws Exception {
        Configuration clientConf = SliderUtils.loadClientConfigurationResource();
        ConfigHelper.mergeConfigurations(conf, clientConf, CLIENT_RESOURCE, true);
        serviceArgs.applyDefinitions(conf);
        serviceArgs.applyFileSystemBinding(conf);
        // init security with our conf
        if (SliderUtils.isHadoopClusterSecure(conf)) {
            SliderUtils.forceLogin();
            SliderUtils.initProcessSecurity(conf);
        }
        AbstractActionArgs coreAction = serviceArgs.getCoreAction();
        if (coreAction.getHadoopServicesRequired()) {
            initHadoopBinding();
        }
        super.serviceInit(conf);
    }

    /**
     * this is where the work is done.
     * @return the exit code
     * @throws Throwable anything that went wrong
     */
    /* JDK7
        
      @Override
      public int runService() throws Throwable {
        
        // choose the action
        String action = serviceArgs.getAction();
        int exitCode = EXIT_SUCCESS;
        String clusterName = serviceArgs.getClusterName();
        // actions
        switch (action) {
          case ACTION_BUILD:
    exitCode = actionBuild(clusterName, serviceArgs.getActionBuildArgs());
    break;
          case ACTION_UPDATE:
    exitCode = actionUpdate(clusterName, serviceArgs.getActionUpdateArgs());
    break;
          case ACTION_CREATE:
    exitCode = actionCreate(clusterName, serviceArgs.getActionCreateArgs());
    break;
          case ACTION_FREEZE:
    exitCode = actionFreeze(clusterName, serviceArgs.getActionFreezeArgs());
    break;
          case ACTION_THAW:
    exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs());
    break;
          case ACTION_DESTROY:
    exitCode = actionDestroy(clusterName);
    break;
          case ACTION_EXISTS:
    exitCode = actionExists(clusterName,
        serviceArgs.getActionExistsArgs().live);
    break;
          case ACTION_FLEX:
    exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs());
    break;
          case ACTION_GETCONF:
    exitCode =
        actionGetConf(clusterName, serviceArgs.getActionGetConfArgs());
    break;
          case ACTION_HELP:
          case ACTION_USAGE:
    log.info(serviceArgs.usage());
    break;
          case ACTION_KILL_CONTAINER:
    exitCode = actionKillContainer(clusterName,
        serviceArgs.getActionKillContainerArgs());
    break;
          case ACTION_AM_SUICIDE:
    exitCode = actionAmSuicide(clusterName,
        serviceArgs.getActionAMSuicideArgs());
    break;
          case ACTION_LIST:
    exitCode = actionList(clusterName, serviceArgs.getActionListArgs());
    break;
          case ACTION_REGISTRY:
    exitCode = actionRegistry(
        serviceArgs.getActionRegistryArgs());
    break;
          case ACTION_STATUS:
    exitCode = actionStatus(clusterName,
        serviceArgs.getActionStatusArgs());
    break;
          case ACTION_VERSION:
    exitCode = actionVersion();
    break;
          default:
    throw new SliderException(EXIT_UNIMPLEMENTED,
        "Unimplemented: " + action);
        }
        
        return exitCode;
      }
        
    */

    /**
     * Launched service execution. This runs {@link #exec()}
     * then catches some exceptions and converts them to exit codes
     * @return an exit code
     * @throws Throwable
     */
    @Override
    public int runService() throws Throwable {
        try {
            return exec();
        } catch (FileNotFoundException nfe) {
            throw new NotFoundException(nfe, nfe.toString());
        } catch (PathNotFoundException nfe) {
            throw new NotFoundException(nfe, nfe.toString());
        }
    }

    /**
     * Execute the command line
     * @return an exit code
     * @throws Throwable on a failure
     */
    public int exec() throws Throwable {

        // choose the action
        String action = serviceArgs.getAction();

        int exitCode = EXIT_SUCCESS;
        String clusterName = serviceArgs.getClusterName();
        // actions
        if (ACTION_INSTALL_PACKAGE.equals(action)) {
            exitCode = actionInstallPkg(serviceArgs.getActionInstallPackageArgs());
        } else if (ACTION_INSTALL_KEYTAB.equals(action)) {
            exitCode = actionInstallKeytab(serviceArgs.getActionInstallKeytabArgs());
        } else if (ACTION_BUILD.equals(action)) {
            exitCode = actionBuild(clusterName, serviceArgs.getActionBuildArgs());
        } else if (ACTION_CREATE.equals(action)) {
            exitCode = actionCreate(clusterName, serviceArgs.getActionCreateArgs());
        } else if (ACTION_FREEZE.equals(action)) {
            exitCode = actionFreeze(clusterName, serviceArgs.getActionFreezeArgs());
        } else if (ACTION_THAW.equals(action)) {
            exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs());
        } else if (ACTION_DESTROY.equals(action)) {
            exitCode = actionDestroy(clusterName);
        } else if (ACTION_DIAGNOSTICS.equals(action)) {
            exitCode = actionDiagnostic(serviceArgs.getActionDiagnosticArgs());
        } else if (ACTION_EXISTS.equals(action)) {
            exitCode = actionExists(clusterName, serviceArgs.getActionExistsArgs());
        } else if (ACTION_FLEX.equals(action)) {
            exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs());
        } else if (ACTION_HELP.equals(action)) {
            log.info(serviceArgs.usage());
        } else if (ACTION_KILL_CONTAINER.equals(action)) {
            exitCode = actionKillContainer(clusterName, serviceArgs.getActionKillContainerArgs());
        } else if (ACTION_AM_SUICIDE.equals(action)) {
            exitCode = actionAmSuicide(clusterName, serviceArgs.getActionAMSuicideArgs());
        } else if (ACTION_LIST.equals(action)) {
            exitCode = actionList(clusterName, serviceArgs.getActionListArgs());
        } else if (ACTION_LOOKUP.equals(action)) {
            exitCode = actionLookup(serviceArgs.getActionLookupArgs());
        } else if (ACTION_REGISTRY.equals(action)) {
            exitCode = actionRegistry(serviceArgs.getActionRegistryArgs());
        } else if (ACTION_RESOLVE.equals(action)) {
            exitCode = actionResolve(serviceArgs.getActionResolveArgs());
        } else if (ACTION_STATUS.equals(action)) {
            exitCode = actionStatus(clusterName, serviceArgs.getActionStatusArgs());
        } else if (ACTION_UPDATE.equals(action)) {
            exitCode = actionUpdate(clusterName, serviceArgs.getActionUpdateArgs());
        } else if (ACTION_VERSION.equals(action)) {
            exitCode = actionVersion();
        } else if (SliderUtils.isUnset(action)) {
            throw new SliderException(EXIT_USAGE, serviceArgs.usage());
        } else {
            throw new SliderException(EXIT_UNIMPLEMENTED, "Unimplemented: " + action);
        }

        return exitCode;
    }

    /**
       * Perform everything needed to init the hadoop binding.
       * This assumes that the service is already  in inited or started state
       * @throws IOException
       * @throws SliderException
       */
    protected void initHadoopBinding() throws IOException, SliderException {
        // validate the client
        SliderUtils.validateSliderClientEnvironment(null);
        //create the YARN client
        yarnClient = new SliderYarnClientImpl();
        yarnClient.init(getConfig());
        if (getServiceState() == STATE.STARTED) {
            yarnClient.start();
        }
        addService(yarnClient);
        YarnAppListClient = new YarnAppListClient(yarnClient, getUsername(), getConfig());
        // create the filesystem
        sliderFileSystem = new SliderFileSystem(getConfig());
    }

    /**
     * Delete the zookeeper node associated with the calling user and the cluster
     * TODO: YARN registry operations
     **/
    @VisibleForTesting
    public boolean deleteZookeeperNode(String clusterName) throws YarnException, IOException {
        String user = getUsername();
        String zkPath = ZKIntegration.mkClusterPath(user, clusterName);
        Exception e = null;
        try {
            Configuration config = getConfig();
            ZKIntegration client = getZkClient(clusterName, user);
            if (client != null) {
                if (client.exists(zkPath)) {
                    log.info("Deleting zookeeper path {}", zkPath);
                }
                client.deleteRecursive(zkPath);
                return true;
            }
        } catch (InterruptedException ignored) {
            e = ignored;
        } catch (KeeperException ignored) {
            e = ignored;
        } catch (BadConfigException ignored) {
            e = ignored;
        }
        if (e != null) {
            log.debug("Unable to recursively delete zk node {}", zkPath);
            log.debug("Reason: ", e);
        }

        return false;
    }

    /**
     * Create the zookeeper node associated with the calling user and the cluster
     */
    @VisibleForTesting
    public String createZookeeperNode(String clusterName, Boolean nameOnly) throws YarnException, IOException {
        String user = getUsername();
        String zkPath = ZKIntegration.mkClusterPath(user, clusterName);
        if (nameOnly) {
            return zkPath;
        }
        Configuration config = getConfig();
        ZKIntegration client = getZkClient(clusterName, user);
        if (client != null) {
            try {
                client.createPath(zkPath, "", ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
                return zkPath;
            } catch (InterruptedException e) {
                log.warn("Unable to create default zk node {}", zkPath, e);
            } catch (KeeperException e) {
                log.warn("Unable to create default zk node {}", zkPath, e);
            }
        }

        return null;
    }

    /**
     * Gets a zookeeper client, returns null if it cannot connect to zookeeper
     **/
    protected ZKIntegration getZkClient(String clusterName, String user) throws YarnException {
        String registryQuorum = lookupZKQuorum();
        ZKIntegration client = null;
        try {
            BlockingZKWatcher watcher = new BlockingZKWatcher();
            client = ZKIntegration.newInstance(registryQuorum, user, clusterName, true, false, watcher);
            client.init();
            watcher.waitForZKConnection(2 * 1000);
        } catch (InterruptedException e) {
            client = null;
            log.warn("Unable to connect to zookeeper quorum {}", registryQuorum, e);
        } catch (IOException e) {
            log.warn("Unable to connect to zookeeper quorum {}", registryQuorum, e);
        }
        return client;
    }

    @Override
    public int actionDestroy(String clustername) throws YarnException, IOException {
        // verify that a live cluster isn't there
        SliderUtils.validateClusterName(clustername);
        //no=op, it is now mandatory. 
        verifyBindingsDefined();
        verifyNoLiveClusters(clustername, "Destroy");

        // create the directory path
        Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
        // delete the directory;
        FileSystem fs = sliderFileSystem.getFileSystem();
        boolean exists = fs.exists(clusterDirectory);
        if (exists) {
            log.debug("Application Instance {} found at {}: destroying", clustername, clusterDirectory);
            boolean deleted = fs.delete(clusterDirectory, true);
            if (!deleted) {
                log.warn("Filesystem returned false from delete() operation");
            }

            if (!deleteZookeeperNode(clustername)) {
                log.warn("Unable to perform node cleanup in Zookeeper.");
            }

            if (fs.exists(clusterDirectory)) {
                log.warn("Failed to delete {}", clusterDirectory);
            }

        } else {
            log.debug("Application Instance {} already destroyed", clustername);
        }

        // rm the registry entry do not let this block the destroy operations
        String registryPath = SliderRegistryUtils.registryPathForInstance(clustername);
        try {
            getRegistryOperations().delete(registryPath, true);
        } catch (IOException e) {
            log.warn("Error deleting registry entry {}: {} ", registryPath, e, e);
        } catch (SliderException e) {
            log.warn("Error binding to registry {} ", e, e);
        }

        List<ApplicationReport> instances = findAllLiveInstances(clustername);
        // detect any race leading to cluster creation during the check/destroy process
        // and report a problem.
        if (!instances.isEmpty()) {
            throw new SliderException(EXIT_APPLICATION_IN_USE,
                    clustername + ": " + E_DESTROY_CREATE_RACE_CONDITION + " :" + instances.get(0));
        }
        log.info("Destroyed cluster {}", clustername);
        return EXIT_SUCCESS;
    }

    @Override
    public int actionAmSuicide(String clustername, ActionAMSuicideArgs args) throws YarnException, IOException {
        SliderClusterOperations clusterOperations = createClusterOperations(clustername);
        clusterOperations.amSuicide(args.message, args.exitcode, args.waittime);
        return EXIT_SUCCESS;
    }

    @Override
    public AbstractClientProvider createClientProvider(String provider) throws SliderException {
        SliderProviderFactory factory = SliderProviderFactory.createSliderProviderFactory(provider);
        return factory.createClientProvider();
    }

    /**
     * Create the cluster -saving the arguments to a specification file first
     * @param clustername cluster name
     * @return the status code
     * @throws YarnException Yarn problems
     * @throws IOException other problems
     * @throws BadCommandArgumentsException bad arguments.
     */
    public int actionCreate(String clustername, ActionCreateArgs createArgs) throws YarnException, IOException {

        actionBuild(clustername, createArgs);
        Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
        AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(clustername, clusterDirectory);
        try {
            checkForCredentials(getConfig(), instanceDefinition.getAppConf());
        } catch (IOException e) {
            sliderFileSystem.getFileSystem().delete(clusterDirectory, true);
            throw e;
        }
        return startCluster(clustername, createArgs);
    }

    private void checkForCredentials(Configuration conf, ConfTree tree) throws IOException {
        if (tree.credentials == null || tree.credentials.size() == 0) {
            log.info("No credentials requested");
            return;
        }

        for (Entry<String, List<String>> cred : tree.credentials.entrySet()) {
            String provider = cred.getKey();
            List<String> aliases = cred.getValue();
            if (aliases == null || aliases.size() == 0) {
                continue;
            }
            Configuration c = new Configuration(conf);
            c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
            CredentialProvider credentialProvider = CredentialProviderFactory.getProviders(c).get(0);
            Set<String> existingAliases = new HashSet<String>(credentialProvider.getAliases());
            for (String alias : aliases) {
                if (!existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) {
                    throw new IOException("Specified credentials have not been " + "initialized in provider "
                            + provider + ": " + alias);
                }
            }
        }
    }

    @Override
    public int actionBuild(String clustername, AbstractClusterBuildingActionArgs buildInfo)
            throws YarnException, IOException {

        buildInstanceDefinition(clustername, buildInfo, false, false);
        return EXIT_SUCCESS;
    }

    @Override
    public int actionInstallKeytab(ActionInstallKeytabArgs installKeytabInfo) throws YarnException, IOException {

        Path srcFile = null;
        if (StringUtils.isEmpty(installKeytabInfo.folder)) {
            throw new BadCommandArgumentsException(
                    "A valid destination keytab sub-folder name is required (e.g. 'security').\n"
                            + CommonArgs.usage(serviceArgs, ACTION_INSTALL_KEYTAB));
        }

        if (StringUtils.isEmpty(installKeytabInfo.keytabUri)) {
            throw new BadCommandArgumentsException("A valid local keytab location is required.");
        } else {
            File keytabFile = new File(installKeytabInfo.keytabUri);
            if (!keytabFile.exists() || keytabFile.isDirectory()) {
                throw new BadCommandArgumentsException(
                        "Unable to access supplied keytab file at " + keytabFile.getAbsolutePath());
            } else {
                srcFile = new Path(keytabFile.toURI());
            }
        }

        Path pkgPath = sliderFileSystem.buildKeytabInstallationDirPath(installKeytabInfo.folder);
        sliderFileSystem.getFileSystem().mkdirs(pkgPath);
        sliderFileSystem.getFileSystem().setPermission(pkgPath,
                new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));

        Path fileInFs = new Path(pkgPath, srcFile.getName());
        log.info("Installing keytab {} at {} and overwrite is {}.", srcFile, fileInFs, installKeytabInfo.overwrite);
        if (sliderFileSystem.getFileSystem().exists(fileInFs) && !installKeytabInfo.overwrite) {
            throw new BadCommandArgumentsException(
                    "Keytab exists at " + fileInFs.toUri().toString() + ". Use --overwrite to overwrite.");
        }

        sliderFileSystem.getFileSystem().copyFromLocalFile(false, installKeytabInfo.overwrite, srcFile, fileInFs);
        sliderFileSystem.getFileSystem().setPermission(fileInFs,
                new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE));

        return EXIT_SUCCESS;
    }

    @Override
    public int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) throws YarnException, IOException {

        Path srcFile = null;
        if (StringUtils.isEmpty(installPkgInfo.name)) {
            throw new BadCommandArgumentsException("A valid application type name is required (e.g. HBASE).\n"
                    + CommonArgs.usage(serviceArgs, ACTION_INSTALL_PACKAGE));
        }

        if (StringUtils.isEmpty(installPkgInfo.packageURI)) {
            throw new BadCommandArgumentsException("A valid application package location required.");
        } else {
            File pkgFile = new File(installPkgInfo.packageURI);
            if (!pkgFile.exists() || pkgFile.isDirectory()) {
                throw new BadCommandArgumentsException(
                        "Unable to access supplied pkg file at " + pkgFile.getAbsolutePath());
            } else {
                srcFile = new Path(pkgFile.toURI());
            }
        }

        Path pkgPath = sliderFileSystem.buildPackageDirPath(installPkgInfo.name);
        sliderFileSystem.getFileSystem().mkdirs(pkgPath);

        Path fileInFs = new Path(pkgPath, srcFile.getName());
        log.info("Installing package {} at {} and overwrite is {}.", srcFile, fileInFs, installPkgInfo.replacePkg);
        if (sliderFileSystem.getFileSystem().exists(fileInFs) && !installPkgInfo.replacePkg) {
            throw new BadCommandArgumentsException(
                    "Pkg exists at " + fileInFs.toUri().toString() + ". Use --replacepkg to overwrite.");
        }

        sliderFileSystem.getFileSystem().copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs);
        return EXIT_SUCCESS;
    }

    @Override
    public int actionUpdate(String clustername, AbstractClusterBuildingActionArgs buildInfo)
            throws YarnException, IOException {
        buildInstanceDefinition(clustername, buildInfo, true, true);
        return EXIT_SUCCESS;
    }

    /**
     * Build up the AggregateConfiguration for an application instance then
     * persists it
     * @param clustername name of the cluster
     * @param buildInfo the arguments needed to build the cluster
     * @param overwrite true if existing cluster directory can be overwritten
     * @param liveClusterAllowed true if live cluster can be modified
     * @throws YarnException
     * @throws IOException
     */

    public void buildInstanceDefinition(String clustername, AbstractClusterBuildingActionArgs buildInfo,
            boolean overwrite, boolean liveClusterAllowed) throws YarnException, IOException {
        // verify that a live cluster isn't there
        SliderUtils.validateClusterName(clustername);
        verifyBindingsDefined();
        if (!liveClusterAllowed) {
            verifyNoLiveClusters(clustername, "Create");
        }

        Configuration conf = getConfig();
        String registryQuorum = lookupZKQuorum();

        Path appconfdir = buildInfo.getConfdir();
        // Provider
        String providerName = buildInfo.getProvider();
        requireArgumentSet(Arguments.ARG_PROVIDER, providerName);
        log.debug("Provider is {}", providerName);
        SliderAMClientProvider sliderAM = new SliderAMClientProvider(conf);
        AbstractClientProvider provider = createClientProvider(providerName);
        InstanceBuilder builder = new InstanceBuilder(sliderFileSystem, getConfig(), clustername);

        AggregateConf instanceDefinition = new AggregateConf();
        ConfTreeOperations appConf = instanceDefinition.getAppConfOperations();
        ConfTreeOperations resources = instanceDefinition.getResourceOperations();
        ConfTreeOperations internal = instanceDefinition.getInternalOperations();
        //initial definition is set by the providers 
        sliderAM.prepareInstanceConfiguration(instanceDefinition);
        provider.prepareInstanceConfiguration(instanceDefinition);

        //load in any specified on the command line
        if (buildInfo.resources != null) {
            try {
                resources.mergeFile(buildInfo.resources, new ResourcesInputPropertiesValidator());

            } catch (IOException e) {
                throw new BadConfigException(e, "incorrect argument to %s: \"%s\" : %s ", Arguments.ARG_RESOURCES,
                        buildInfo.resources, e.toString());
            }
        }
        if (buildInfo.template != null) {
            try {
                appConf.mergeFile(buildInfo.template, new TemplateInputPropertiesValidator());
            } catch (IOException e) {
                throw new BadConfigException(e, "incorrect argument to %s: \"%s\" : %s ", Arguments.ARG_TEMPLATE,
                        buildInfo.template, e.toString());
            }
        }

        //get the command line options
        ConfTree cmdLineAppOptions = buildInfo.buildAppOptionsConfTree();
        ConfTree cmdLineResourceOptions = buildInfo.buildResourceOptionsConfTree();

        appConf.merge(cmdLineAppOptions);

        // put the role counts into the resources file
        Map<String, String> argsRoleMap = buildInfo.getComponentMap();
        for (Map.Entry<String, String> roleEntry : argsRoleMap.entrySet()) {
            String count = roleEntry.getValue();
            String key = roleEntry.getKey();
            log.debug("{} => {}", key, count);
            resources.getOrAddComponent(key).put(ResourceKeys.COMPONENT_INSTANCES, count);
        }

        //all CLI role options
        Map<String, Map<String, String>> appOptionMap = buildInfo.getCompOptionMap();
        appConf.mergeComponents(appOptionMap);

        //internal picks up core. values only
        internal.propagateGlobalKeys(appConf, "slider.");
        internal.propagateGlobalKeys(appConf, "internal.");

        //copy over role. and yarn. values ONLY to the resources
        if (PROPAGATE_RESOURCE_OPTION) {
            resources.propagateGlobalKeys(appConf, "component.");
            resources.propagateGlobalKeys(appConf, "role.");
            resources.propagateGlobalKeys(appConf, "yarn.");
            resources.mergeComponentsPrefix(appOptionMap, "component.", true);
            resources.mergeComponentsPrefix(appOptionMap, "yarn.", true);
            resources.mergeComponentsPrefix(appOptionMap, "role.", true);
        }

        // resource component args
        appConf.merge(cmdLineResourceOptions);
        resources.merge(cmdLineResourceOptions);
        resources.mergeComponents(buildInfo.getResourceCompOptionMap());

        builder.init(providerName, instanceDefinition);
        builder.propagateFilename();
        builder.propagatePrincipals();
        builder.setImageDetailsIfAvailable(buildInfo.getImage(), buildInfo.getAppHomeDir());
        builder.setQueue(buildInfo.queue);

        String quorum = buildInfo.getZKhosts();
        if (SliderUtils.isUnset(quorum)) {
            quorum = registryQuorum;
        }
        if (isUnset(quorum)) {
            throw new BadConfigException("No Zookeeper quorum defined");
        }
        ZKPathBuilder zkPaths = new ZKPathBuilder(getAppName(), getUsername(), clustername, registryQuorum, quorum);
        String zookeeperRoot = buildInfo.getAppZKPath();

        if (isSet(zookeeperRoot)) {
            zkPaths.setAppPath(zookeeperRoot);
        } else {
            String createDefaultZkNode = appConf.getGlobalOptions().getOption(AgentKeys.CREATE_DEF_ZK_NODE,
                    "false");
            if (createDefaultZkNode.equals("true")) {
                String defaultZKPath = createZookeeperNode(clustername, false);
                log.debug("ZK node created for application instance: {}", defaultZKPath);
                if (defaultZKPath != null) {
                    zkPaths.setAppPath(defaultZKPath);
                }
            } else {
                // create AppPath if default is being used
                String defaultZKPath = createZookeeperNode(clustername, true);
                log.debug("ZK node assigned to application instance: {}", defaultZKPath);
                zkPaths.setAppPath(defaultZKPath);
            }
        }

        builder.addZKBinding(zkPaths);

        //then propagate any package URI
        if (buildInfo.packageURI != null) {
            appConf.set(AgentKeys.PACKAGE_PATH, buildInfo.packageURI);
        }

        propagatePythonExecutable(conf, instanceDefinition);

        // make any substitutions needed at this stage
        replaceTokens(appConf.getConfTree(), getUsername(), clustername);

        // providers to validate what there is
        AggregateConf instanceDescription = builder.getInstanceDescription();
        validateInstanceDefinition(sliderAM, instanceDescription, sliderFileSystem);
        validateInstanceDefinition(provider, instanceDescription, sliderFileSystem);
        try {
            persistInstanceDefinition(overwrite, appconfdir, builder);
        } catch (LockAcquireFailedException e) {
            log.warn("Failed to get a Lock on {} : {}", builder, e);
            throw new BadClusterStateException("Failed to save " + clustername + ": " + e);
        }
    }

    protected void persistInstanceDefinition(boolean overwrite, Path appconfdir, InstanceBuilder builder)
            throws IOException, SliderException, LockAcquireFailedException {
        builder.persist(appconfdir, overwrite);
    }

    @VisibleForTesting
    public static void replaceTokens(ConfTree conf, String userName, String clusterName) throws IOException {
        Map<String, String> newglobal = new HashMap<String, String>();
        for (Entry<String, String> entry : conf.global.entrySet()) {
            newglobal.put(entry.getKey(), replaceTokens(entry.getValue(), userName, clusterName));
        }
        conf.global.putAll(newglobal);

        Map<String, List<String>> newcred = new HashMap<String, List<String>>();
        for (Entry<String, List<String>> entry : conf.credentials.entrySet()) {
            List<String> resultList = new ArrayList<String>();
            for (String v : entry.getValue()) {
                resultList.add(replaceTokens(v, userName, clusterName));
            }
            newcred.put(replaceTokens(entry.getKey(), userName, clusterName), resultList);
        }
        conf.credentials.clear();
        conf.credentials.putAll(newcred);
    }

    private static String replaceTokens(String s, String userName, String clusterName) throws IOException {
        return s.replaceAll(Pattern.quote("${USER}"), userName).replaceAll(Pattern.quote("${USER_NAME}"), userName)
                .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName);
    }

    public FsPermission getClusterDirectoryPermissions(Configuration conf) {
        String clusterDirPermsOct = conf.get(CLUSTER_DIRECTORY_PERMISSIONS, DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
        return new FsPermission(clusterDirPermsOct);
    }

    /**
     * Verify that the Resource Manager is configured (on a non-HA cluster).
     * with a useful error message
     * @throws BadCommandArgumentsException the exception raised on an invalid config
     */
    public void verifyBindingsDefined() throws BadCommandArgumentsException {
        InetSocketAddress rmAddr = SliderUtils.getRmAddress(getConfig());
        if (!getConfig().getBoolean(YarnConfiguration.RM_HA_ENABLED, false)
                && !SliderUtils.isAddressDefined(rmAddr)) {
            throw new BadCommandArgumentsException("No valid Resource Manager address provided in the argument "
                    + Arguments.ARG_MANAGER + " or the configuration property " + YarnConfiguration.RM_ADDRESS
                    + " value :" + rmAddr);
        }
    }

    /**
     * Load and start a cluster specification.
     * This assumes that all validation of args and cluster state
     * have already taken place
     *
     * @param clustername name of the cluster.
     * @param launchArgs launch arguments
     * @return the exit code
     * @throws YarnException
     * @throws IOException
     */
    private int startCluster(String clustername, LaunchArgsAccessor launchArgs) throws YarnException, IOException {
        Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
        AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(clustername, clusterDirectory);

        LaunchedApplication launchedApplication = launchApplication(clustername, clusterDirectory,
                instanceDefinition, serviceArgs.isDebug());
        applicationId = launchedApplication.getApplicationId();

        if (launchArgs.getOutputFile() != null) {
            // output file has been requested. Get the app report and serialize it
            ApplicationReport report = launchedApplication.getApplicationReport();
            SerializedApplicationReport sar = new SerializedApplicationReport(report);
            sar.submitTime = System.currentTimeMillis();
            ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser();
            serDeser.save(sar, launchArgs.getOutputFile());
        }
        int waittime = launchArgs.getWaittime();
        if (waittime > 0) {
            return waitForAppRunning(launchedApplication, waittime, waittime);
        } else {
            // no waiting
            return EXIT_SUCCESS;
        }
    }

    /**
     * Load the instance definition. It is not resolved at this point
     * @param name cluster name
     * @param clusterDirectory cluster dir
     * @return the loaded configuration
     * @throws IOException
     * @throws SliderException
     * @throws UnknownApplicationInstanceException if the file is not found
     */
    public AggregateConf loadInstanceDefinitionUnresolved(String name, Path clusterDirectory)
            throws IOException, SliderException {

        try {
            AggregateConf definition = InstanceIO.loadInstanceDefinitionUnresolved(sliderFileSystem,
                    clusterDirectory);
            definition.setName(name);
            return definition;
        } catch (FileNotFoundException e) {
            throw UnknownApplicationInstanceException.unknownInstance(name, e);
        }
    }

    /**
    * Load the instance definition. 
    * @param name cluster name
    * @param resolved flag to indicate the cluster should be resolved
    * @return the loaded configuration
    * @throws IOException IO problems
    * @throws SliderException slider explicit issues
    * @throws UnknownApplicationInstanceException if the file is not found
    */
    public AggregateConf loadInstanceDefinition(String name, boolean resolved) throws IOException, SliderException {

        Path clusterDirectory = sliderFileSystem.buildClusterDirPath(name);
        AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(name, clusterDirectory);
        if (resolved) {
            instanceDefinition.resolve();
        }
        return instanceDefinition;

    }

    /**
     *
     * @param clustername name of the cluster
     * @param clusterDirectory cluster dir
     * @param instanceDefinition the instance definition
     * @param debugAM enable debug AM options
     * @return the launched application
     * @throws YarnException
     * @throws IOException
     */
    public LaunchedApplication launchApplication(String clustername, Path clusterDirectory,
            AggregateConf instanceDefinition, boolean debugAM) throws YarnException, IOException {

        deployedClusterName = clustername;
        SliderUtils.validateClusterName(clustername);
        verifyNoLiveClusters(clustername, "Launch");
        Configuration config = getConfig();
        lookupZKQuorum();
        boolean clusterSecure = SliderUtils.isHadoopClusterSecure(config);
        //create the Slider AM provider -this helps set up the AM
        SliderAMClientProvider sliderAM = new SliderAMClientProvider(config);

        instanceDefinition.resolve();
        launchedInstanceDefinition = instanceDefinition;

        ConfTreeOperations internalOperations = instanceDefinition.getInternalOperations();
        MapOperations internalOptions = internalOperations.getGlobalOptions();
        ConfTreeOperations resourceOperations = instanceDefinition.getResourceOperations();
        ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations();
        Path generatedConfDirPath = createPathThatMustExist(
                internalOptions.getMandatoryOption(InternalKeys.INTERNAL_GENERATED_CONF_PATH));
        Path snapshotConfPath = createPathThatMustExist(
                internalOptions.getMandatoryOption(InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH));

        // cluster Provider
        AbstractClientProvider provider = createClientProvider(
                internalOptions.getMandatoryOption(InternalKeys.INTERNAL_PROVIDER_NAME));
        // make sure the conf dir is valid;

        if (log.isDebugEnabled()) {
            log.debug(instanceDefinition.toString());
        }
        MapOperations sliderAMResourceComponent = resourceOperations.getOrAddComponent(SliderKeys.COMPONENT_AM);
        MapOperations resourceGlobalOptions = resourceOperations.getGlobalOptions();

        // add the tags if available
        Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem,
                appOperations.getGlobalOptions().get(AgentKeys.APP_DEF));
        AppMasterLauncher amLauncher = new AppMasterLauncher(clustername, SliderKeys.APP_TYPE, config,
                sliderFileSystem, yarnClient, clusterSecure, sliderAMResourceComponent, resourceGlobalOptions,
                applicationTags);

        ApplicationId appId = amLauncher.getApplicationId();
        // set the application name;
        amLauncher.setKeepContainersOverRestarts(true);

        int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0);
        amLauncher.setMaxAppAttempts(maxAppAttempts);

        sliderFileSystem.purgeAppInstanceTempFiles(clustername);
        Path tempPath = sliderFileSystem.createAppInstanceTempPath(clustername, appId.toString() + "/am");
        String libdir = "lib";
        Path libPath = new Path(tempPath, libdir);
        sliderFileSystem.getFileSystem().mkdirs(libPath);
        log.debug("FS={}, tempPath={}, libdir={}", sliderFileSystem.toString(), tempPath, libPath);
        // set local resources for the application master
        // local files or archives as needed
        // In this scenario, the jar file for the application master is part of the local resources
        Map<String, LocalResource> localResources = amLauncher.getLocalResources();

        // look for the configuration directory named on the command line
        boolean hasServerLog4jProperties = false;
        Path remoteConfPath = null;
        String relativeConfDir = null;
        String confdirProp = System.getProperty(SliderKeys.PROPERTY_CONF_DIR);
        if (confdirProp == null || confdirProp.isEmpty()) {
            log.debug("No local configuration directory provided as system property");
        } else {
            File confDir = new File(confdirProp);
            if (!confDir.exists()) {
                throw new BadConfigException(E_CONFIGURATION_DIRECTORY_NOT_FOUND, confDir);
            }
            Path localConfDirPath = SliderUtils.createLocalPath(confDir);
            remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR);
            log.debug("Slider configuration directory is {}; remote to be {}", localConfDirPath, remoteConfPath);
            SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null);

            File log4jserver = new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
            hasServerLog4jProperties = log4jserver.isFile();
        }
        // the assumption here is that minimr cluster => this is a test run
        // and the classpath can look after itself

        boolean usingMiniMRCluster = getUsingMiniMRCluster();
        if (!usingMiniMRCluster) {

            log.debug("Destination is not a MiniYARNCluster -copying full classpath");

            // insert conf dir first
            if (remoteConfPath != null) {
                relativeConfDir = SliderKeys.SUBMITTED_CONF_DIR;
                Map<String, LocalResource> submittedConfDir = sliderFileSystem.submitDirectory(remoteConfPath,
                        relativeConfDir);
                SliderUtils.mergeMaps(localResources, submittedConfDir);
            }
        }
        // build up the configuration 
        // IMPORTANT: it is only after this call that site configurations
        // will be valid.

        propagatePrincipals(config, instanceDefinition);
        // validate security data

        /*
            // turned off until tested
            SecurityConfiguration securityConfiguration =
                new SecurityConfiguration(config,
        instanceDefinition, clustername);
                
        */
        Configuration clientConfExtras = new Configuration(false);
        // then build up the generated path.
        FsPermission clusterPerms = getClusterDirectoryPermissions(config);
        SliderUtils.copyDirectory(config, snapshotConfPath, generatedConfDirPath, clusterPerms);

        // standard AM resources
        sliderAM.prepareAMAndConfigForLaunch(sliderFileSystem, config, amLauncher, instanceDefinition,
                snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath, usingMiniMRCluster);
        //add provider-specific resources
        provider.prepareAMAndConfigForLaunch(sliderFileSystem, config, amLauncher, instanceDefinition,
                snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath, usingMiniMRCluster);

        // now that the site config is fully generated, the provider gets
        // to do a quick review of them.
        log.debug("Preflight validation of cluster configuration");

        sliderAM.preflightValidateClusterConfiguration(sliderFileSystem, clustername, config, instanceDefinition,
                clusterDirectory, generatedConfDirPath, clusterSecure);

        provider.preflightValidateClusterConfiguration(sliderFileSystem, clustername, config, instanceDefinition,
                clusterDirectory, generatedConfDirPath, clusterSecure);

        // TODO: consider supporting apps that don't have an image path
        Path imagePath = SliderUtils.extractImagePath(sliderFileSystem, internalOptions);
        if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) {
            log.debug("Registered image path {}", imagePath);
        }

        // build the environment
        amLauncher.putEnv(SliderUtils.buildEnvMap(sliderAMResourceComponent));
        ClasspathConstructor classpath = SliderUtils.buildClasspath(relativeConfDir, libdir, getConfig(),
                usingMiniMRCluster);
        amLauncher.setClasspath(classpath);
        if (log.isDebugEnabled()) {
            log.debug("AM classpath={}", classpath);
            log.debug("Environment Map:\n{}", SliderUtils.stringifyMap(amLauncher.getEnv()));
            log.debug("Files in lib path\n{}", sliderFileSystem.listFSDir(libPath));
        }

        // rm address

        InetSocketAddress rmSchedulerAddress;
        try {
            rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(config);
        } catch (IllegalArgumentException e) {
            throw new BadConfigException("%s Address invalid: %s", YarnConfiguration.RM_SCHEDULER_ADDRESS,
                    config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS));

        }
        String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress);

        JavaCommandLineBuilder commandLine = new JavaCommandLineBuilder();
        // insert any JVM options);
        sliderAM.addJVMOptions(instanceDefinition, commandLine);
        // enable asserts if the text option is set
        commandLine.enableJavaAssertions();

        // if the conf dir has a log4j-server.properties, switch to that
        if (hasServerLog4jProperties) {
            commandLine.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME);
            commandLine.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
        }

        // add the AM sevice entry point
        commandLine.add(SliderAppMaster.SERVICE_CLASSNAME);

        // create action and the cluster name
        commandLine.add(ACTION_CREATE, clustername);

        // debug
        if (debugAM) {
            commandLine.add(Arguments.ARG_DEBUG);
        }

        // set the cluster directory path
        commandLine.add(Arguments.ARG_CLUSTER_URI, clusterDirectory.toUri());

        if (!isUnset(rmAddr)) {
            commandLine.add(Arguments.ARG_RM_ADDR, rmAddr);
        }

        if (serviceArgs.getFilesystemBinding() != null) {
            commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemBinding());
        }

        /**
         * pass the registry binding
         */
        addConfOptionToCLI(commandLine, config, REGISTRY_PATH, DEFAULT_REGISTRY_PATH);
        addMandatoryConfOptionToCLI(commandLine, config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM);

        if (clusterSecure) {
            // if the cluster is secure, make sure that
            // the relevant security settings go over
            /*
                  addConfOptionToCLI(commandLine, config, KEY_SECURITY);
            */
            addConfOptionToCLI(commandLine, config, DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
        }
        // write out the path output
        commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM);

        String cmdStr = commandLine.build();
        log.debug("Completed setting up app master command {}", cmdStr);

        amLauncher.addCommandLine(commandLine);

        // the Slider AM gets to configure the AM requirements, not the custom provider
        sliderAM.prepareAMResourceRequirements(sliderAMResourceComponent, amLauncher.getResource());

        // Set the priority for the application master

        int amPriority = config.getInt(KEY_YARN_QUEUE_PRIORITY, DEFAULT_YARN_QUEUE_PRIORITY);

        amLauncher.setPriority(amPriority);

        // Set the queue to which this application is to be submitted in the RM
        // Queue for App master
        String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE);
        String suppliedQueue = internalOperations.getGlobalOptions().get(InternalKeys.INTERNAL_QUEUE);
        if (!SliderUtils.isUnset(suppliedQueue)) {
            amQueue = suppliedQueue;
            log.info("Using queue {} for the application instance.", amQueue);
        }

        if (amQueue != null) {
            amLauncher.setQueue(amQueue);
        }

        // submit the application
        LaunchedApplication launchedApplication = amLauncher.submitApplication();
        return launchedApplication;
    }

    private void propagatePythonExecutable(Configuration config, AggregateConf instanceDefinition) {
        String pythonExec = config.get(SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH);
        if (pythonExec != null) {
            instanceDefinition.getAppConfOperations().getGlobalOptions()
                    .putIfUnset(SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH, pythonExec);
        }
    }

    /**
     * Wait for the launched app to be accepted in the time  
     * and, optionally running.
     * <p>
     * If the application
     *
     * @param launchedApplication application
     * @param acceptWaitMillis time in millis to wait for accept
     * @param runWaitMillis time in millis to wait for the app to be running.
     * May be null, in which case no wait takes place
     * @return exit code: success
     * @throws YarnException
     * @throws IOException
     */
    public int waitForAppRunning(LaunchedApplication launchedApplication, int acceptWaitMillis, int runWaitMillis)
            throws YarnException, IOException {
        assert launchedApplication != null;
        int exitCode;
        // wait for the submit state to be reached
        ApplicationReport report = launchedApplication.monitorAppToState(YarnApplicationState.ACCEPTED,
                new Duration(acceptWaitMillis));

        // may have failed, so check that
        if (SliderUtils.hasAppFinished(report)) {
            exitCode = buildExitCode(report);
        } else {
            // exit unless there is a wait

            if (runWaitMillis != 0) {
                // waiting for state to change
                Duration duration = new Duration(runWaitMillis * 1000);
                duration.start();
                report = launchedApplication.monitorAppToState(YarnApplicationState.RUNNING, duration);
                if (report != null && report.getYarnApplicationState() == YarnApplicationState.RUNNING) {
                    exitCode = EXIT_SUCCESS;
                } else {
                    exitCode = buildExitCode(report);
                }
            } else {
                exitCode = EXIT_SUCCESS;
            }
        }
        return exitCode;
    }

    /**
     * Propagate any critical principals from the current site config down to the HBase one.
     * @param config config to read from
     * @param clusterSpec cluster spec
     */
    private void propagatePrincipals(Configuration config, AggregateConf clusterSpec) {
        String dfsPrincipal = config.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
        if (dfsPrincipal != null) {
            String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX
                    + DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
            clusterSpec.getAppConfOperations().getGlobalOptions().putIfUnset(siteDfsPrincipal, dfsPrincipal);
        }
    }

    private boolean addConfOptionToCLI(CommandLineBuilder cmdLine, Configuration conf, String key) {
        String val = conf.get(key);
        return defineIfSet(cmdLine, key, val);
    }

    private String addConfOptionToCLI(CommandLineBuilder cmdLine, Configuration conf, String key, String defVal) {
        String val = conf.get(key, defVal);
        define(cmdLine, key, val);
        return val;
    }

    /**
     * Add a <code>-D key=val</code> command to the CLI
     * @param cmdLine command line
     * @param key key
     * @param val value
     */
    private void define(CommandLineBuilder cmdLine, String key, String val) {
        Preconditions.checkArgument(key != null, "null key");
        Preconditions.checkArgument(val != null, "null value");
        cmdLine.add(Arguments.ARG_DEFINE, key + "=" + val);
    }

    /**
     * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
     * is not null
     * @param cmdLine command line
     * @param key key
     * @param val value
     */
    private boolean defineIfSet(CommandLineBuilder cmdLine, String key, String val) {
        Preconditions.checkArgument(key != null, "null key");
        if (val != null) {
            define(cmdLine, key, val);
            return true;
        } else {
            return false;
        }
    }

    private void addMandatoryConfOptionToCLI(CommandLineBuilder cmdLine, Configuration conf, String key)
            throws BadConfigException {
        if (!addConfOptionToCLI(cmdLine, conf, key)) {
            throw new BadConfigException("Missing configuration option: " + key);
        }
    }

    /**
     * Create a path that must exist in the cluster fs
     * @param uri uri to create
     * @return the path
     * @throws FileNotFoundException if the path does not exist
     */
    public Path createPathThatMustExist(String uri) throws SliderException, IOException {
        return sliderFileSystem.createPathThatMustExist(uri);
    }

    /**
     * verify that a live cluster isn't there
     * @param clustername cluster name
     * @param action
     * @throws SliderException with exit code EXIT_CLUSTER_LIVE
     * if a cluster of that name is either live or starting up.
     */
    public void verifyNoLiveClusters(String clustername, String action) throws IOException, YarnException {
        List<ApplicationReport> existing = findAllLiveInstances(clustername);

        if (!existing.isEmpty()) {
            throw new SliderException(EXIT_APPLICATION_IN_USE,
                    action + " failed for " + clustername + ": " + E_CLUSTER_RUNNING + " :" + existing.get(0));
        }
    }

    public String getUsername() throws IOException {
        return RegistryUtils.currentUser();
    }

    /**
     * Get the name of any deployed cluster
     * @return the cluster name
     */
    public String getDeployedClusterName() {
        return deployedClusterName;
    }

    @VisibleForTesting
    public void setDeployedClusterName(String deployedClusterName) {
        this.deployedClusterName = deployedClusterName;
    }

    /**
     * ask if the client is using a mini MR cluster
     * @return true if they are
     */
    private boolean getUsingMiniMRCluster() {
        return getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false);
    }

    /**
     * Get the application name used in the zookeeper root paths
     * @return an application-specific path in ZK
     */
    private String getAppName() {
        return "slider";
    }

    /**
     * Wait for the app to start running (or go past that state)
     * @param duration time to wait
     * @return the app report; null if the duration turned out
     * @throws YarnException YARN or app issues
     * @throws IOException IO problems
     */
    @VisibleForTesting
    public ApplicationReport monitorAppToRunning(Duration duration) throws YarnException, IOException {
        return monitorAppToState(YarnApplicationState.RUNNING, duration);
    }

    /**
     * Build an exit code for an application from its report.
     * If the report parameter is null, its interpreted as a timeout
     * @param report report application report
     * @return the exit code
     * @throws IOException
     * @throws YarnException
     */
    private int buildExitCode(ApplicationReport report) throws IOException, YarnException {
        if (null == report) {
            return EXIT_TIMED_OUT;
        }

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        switch (state) {
        case FINISHED:
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                log.info("Application has completed successfully");
                return EXIT_SUCCESS;
            } else {
                log.info("Application finished unsuccessfully."
                        + "YarnState = {}, DSFinalStatus = {} Breaking monitoring loop", state, dsStatus);
                return EXIT_YARN_SERVICE_FINISHED_WITH_ERROR;
            }

        case KILLED:
            log.info("Application did not finish. YarnState={}, DSFinalStatus={}", state, dsStatus);
            return EXIT_YARN_SERVICE_KILLED;

        case FAILED:
            log.info("Application Failed. YarnState={}, DSFinalStatus={}", state, dsStatus);
            return EXIT_YARN_SERVICE_FAILED;

        default:
            //not in any of these states
            return EXIT_SUCCESS;
        }
    }

    /**
     * Monitor the submitted application for reaching the requested state.
     * Will also report if the app reaches a later state (failed, killed, etc)
     * Kill application if duration!= null & time expires. 
     * Prerequisite: the applicatin was launched.
     * @param desiredState desired state.
     * @param duration how long to wait -must be more than 0
     * @return the application report -null on a timeout
     * @throws YarnException
     * @throws IOException
     */
    @VisibleForTesting
    public ApplicationReport monitorAppToState(YarnApplicationState desiredState, Duration duration)
            throws YarnException, IOException {
        LaunchedApplication launchedApplication = new LaunchedApplication(applicationId, yarnClient);
        return launchedApplication.monitorAppToState(desiredState, duration);
    }

    @Override
    public ApplicationReport getApplicationReport() throws IOException, YarnException {
        return getApplicationReport(applicationId);
    }

    @Override
    public boolean forceKillApplication(String reason) throws YarnException, IOException {
        if (applicationId != null) {
            new LaunchedApplication(applicationId, yarnClient).forceKill(reason);
            return true;
        }
        return false;
    }

    /**
     * List Slider instances belonging to a specific user. This will include
     * failed and killed instances; there may be duplicates
     * @param user user: "" means all users, null means "default"
     * @return a possibly empty list of Slider AMs
     */
    @VisibleForTesting
    public List<ApplicationReport> listSliderInstances(String user) throws YarnException, IOException {
        return YarnAppListClient.listInstances(user);
    }

    /**
     * A basic list action to list live instances
     * @param clustername cluster name
     * @return success if the listing was considered successful
     * @throws IOException
     * @throws YarnException
     */
    public int actionList(String clustername) throws IOException, YarnException {
        ActionListArgs args = new ActionListArgs();
        args.live = true;
        return actionList(clustername, args);
    }

    /**
     * Implement the list action.
     * @param clustername List out specific instance name
     * @param args Action list arguments
     * @return 0 if one or more entries were listed
     * @throws IOException
     * @throws YarnException
     * @throws UnknownApplicationInstanceException if a specific instance
     * was named but it was not found
     */
    @Override
    @VisibleForTesting
    public int actionList(String clustername, ActionListArgs args) throws IOException, YarnException {
        verifyBindingsDefined();

        boolean live = args.live;
        String state = args.state;
        boolean verbose = args.verbose;

        if (live && !state.isEmpty()) {
            throw new BadCommandArgumentsException(
                    Arguments.ARG_LIVE + " and " + Arguments.ARG_STATE + " are exclusive");
        }
        // flag to indicate only services in a specific state are to be listed
        boolean listOnlyInState = live || !state.isEmpty();

        YarnApplicationState min, max;
        if (live) {
            min = YarnApplicationState.NEW;
            max = YarnApplicationState.RUNNING;
        } else if (!state.isEmpty()) {
            YarnApplicationState stateVal = extractYarnApplicationState(state);
            min = max = stateVal;
        } else {
            min = YarnApplicationState.NEW;
            max = YarnApplicationState.KILLED;
        }
        // get the complete list of persistent instances
        Map<String, Path> persistentInstances = sliderFileSystem.listPersistentInstances();

        if (persistentInstances.isEmpty() && isUnset(clustername)) {
            // an empty listing is a success if no cluster was named
            log.debug("No application instances found");
            return EXIT_SUCCESS;
        }

        // and those the RM knows about
        List<ApplicationReport> instances = listSliderInstances(null);
        SliderUtils.sortApplicationsByMostRecent(instances);
        Map<String, ApplicationReport> reportMap = SliderUtils.buildApplicationReportMap(instances, min, max);
        log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}", persistentInstances.size(),
                instances.size(), min, max, reportMap.size());

        if (isSet(clustername)) {
            // only one instance is expected
            // resolve the persistent value
            Path persistent = persistentInstances.get(clustername);
            if (persistent == null) {
                throw unknownClusterException(clustername);
            }
            // create a new map with only that instance in it.
            // this restricts the output of results to this instance
            persistentInstances = new HashMap<String, Path>();
            persistentInstances.put(clustername, persistent);
        }

        // at this point there is either the entire list or a stripped down instance
        int listed = 0;

        for (String name : persistentInstances.keySet()) {
            ApplicationReport report = reportMap.get(name);
            if (!listOnlyInState || report != null) {
                // list the details if all were requested, or the filtering contained
                // a report
                listed++;
                String details = instanceDetailsToString(name, report, verbose);
                print(details);
            }
        }

        return listed > 0 ? EXIT_SUCCESS : EXIT_FALSE;
    }

    /**
     * Convert the instance details of an application to a string
     * @param name instance name
     * @param report the application report
     * @param verbose verbose output
     * @return a string
     */
    String instanceDetailsToString(String name, ApplicationReport report, boolean verbose) {
        // format strings
        String staticf = "%-30s";
        String reportedf = staticf + "  %10s  %-40s";
        String livef = reportedf + " %s";
        StringBuilder builder = new StringBuilder(200);
        if (report == null) {
            builder.append(String.format(staticf, name));
        } else {
            // there's a report to look at
            String appId = report.getApplicationId().toString();
            String state = report.getYarnApplicationState().toString();
            if (report.getYarnApplicationState() == YarnApplicationState.RUNNING) {
                // running: there's a URL
                builder.append(String.format(livef, name, state, appId, report.getTrackingUrl()));
            } else {
                builder.append(String.format(reportedf, name, state, appId));
            }
            if (verbose) {
                builder.append('\n');
                builder.append(SliderUtils.appReportToString(report, "\n  "));
            }
        }

        builder.append('\n');
        return builder.toString();
    }

    /**
     * Extract the state of a Yarn application --state argument
     * @param state state argument
     * @return the application state
     * @throws BadCommandArgumentsException if the argument did not match
     * any known state
     */
    private YarnApplicationState extractYarnApplicationState(String state) throws BadCommandArgumentsException {
        YarnApplicationState stateVal;
        try {
            stateVal = YarnApplicationState.valueOf(state.toUpperCase(Locale.ENGLISH));
        } catch (IllegalArgumentException e) {
            throw new BadCommandArgumentsException("Unknown state: " + state);

        }
        return stateVal;
    }

    /**
     * Is an application active: accepted or running
     * @param report the application report
     * @return true if it is running or scheduled to run.
     */
    public boolean isApplicationActive(ApplicationReport report) {
        return report.getYarnApplicationState() == YarnApplicationState.RUNNING
                || report.getYarnApplicationState() == YarnApplicationState.ACCEPTED;
    }

    /**
     * Implement the islive action: probe for a cluster of the given name existing
     * @return exit code
     */

    @Override
    @VisibleForTesting
    public int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException {
        verifyBindingsDefined();
        SliderUtils.validateClusterName(name);
        log.debug("actionFlex({})", name);
        Map<String, Integer> roleInstances = new HashMap<String, Integer>();
        Map<String, String> roleMap = args.getComponentMap();
        for (Map.Entry<String, String> roleEntry : roleMap.entrySet()) {
            String key = roleEntry.getKey();
            String val = roleEntry.getValue();
            try {
                roleInstances.put(key, Integer.valueOf(val));
            } catch (NumberFormatException e) {
                throw new BadCommandArgumentsException("Requested count of role %s" + " is not a number: \"%s\"",
                        key, val);
            }
        }
        return flex(name, roleInstances);
    }

    @Override
    public int actionExists(String name, boolean checkLive) throws YarnException, IOException {
        ActionExistsArgs args = new ActionExistsArgs();
        args.live = checkLive;
        return actionExists(name, args);
    }

    public int actionExists(String name, ActionExistsArgs args) throws YarnException, IOException {
        verifyBindingsDefined();
        SliderUtils.validateClusterName(name);
        boolean checkLive = args.live;
        log.debug("actionExists({}, {}, {})", name, checkLive, args.state);

        //initial probe for a cluster in the filesystem
        Path clusterDirectory = sliderFileSystem.buildClusterDirPath(name);
        if (!sliderFileSystem.getFileSystem().exists(clusterDirectory)) {
            throw unknownClusterException(name);
        }
        String state = args.state;
        if (!checkLive && SliderUtils.isUnset(state)) {
            log.info("Application {} exists", name);
            return EXIT_SUCCESS;
        }

        //test for liveness/state
        boolean inDesiredState = false;
        ApplicationReport instance;
        instance = findInstance(name);
        if (instance == null) {
            log.info("Application {} not running", name);
            return EXIT_FALSE;
        }
        if (checkLive) {
            // the app exists, check that it is not in any terminated state
            YarnApplicationState appstate = instance.getYarnApplicationState();
            log.debug(" current app state = {}", appstate);
            inDesiredState = appstate.ordinal() < YarnApplicationState.FINISHED.ordinal();
        } else {
            // scan for instance in single --state state
            List<ApplicationReport> userInstances = yarnClient.listInstances("");
            state = state.toUpperCase(Locale.ENGLISH);
            YarnApplicationState desiredState = extractYarnApplicationState(state);
            ApplicationReport foundInstance = yarnClient.findAppInInstanceList(userInstances, name, desiredState);
            if (foundInstance != null) {
                // found in selected state: success
                inDesiredState = true;
                // mark this as the instance to report
                instance = foundInstance;
            }
        }

        SliderUtils.OnDemandReportStringifier report = new SliderUtils.OnDemandReportStringifier(instance);
        if (!inDesiredState) {
            //cluster in the list of apps but not running
            log.info("Application {} found but is in wrong state {}", name, instance.getYarnApplicationState());
            log.debug("State {}", report);
            return EXIT_FALSE;
        } else {
            log.debug("Application instance is in desired state");
            log.info("Application {} is {}\n{}", name, instance.getYarnApplicationState(), report);
            return EXIT_SUCCESS;
        }
    }

    @Override
    public int actionKillContainer(String name, ActionKillContainerArgs args) throws YarnException, IOException {
        String id = args.id;
        if (SliderUtils.isUnset(id)) {
            throw new BadCommandArgumentsException("Missing container id");
        }
        log.info("killingContainer {}:{}", name, id);
        SliderClusterOperations clusterOps = new SliderClusterOperations(bondToCluster(name));
        try {
            clusterOps.killContainer(id);
        } catch (NoSuchNodeException e) {
            throw new BadClusterStateException("Container %s not found in cluster %s", id, name);
        }
        return EXIT_SUCCESS;
    }

    @Override
    public String actionEcho(String name, ActionEchoArgs args) throws YarnException, IOException {
        String message = args.message;
        if (message == null) {
            throw new BadCommandArgumentsException("missing message");
        }
        SliderClusterOperations clusterOps = new SliderClusterOperations(bondToCluster(name));
        return clusterOps.echo(message);
    }

    /**
     * Get at the service registry operations
     * @return registry client -valid after the service is inited.
     */
    public YarnAppListClient getYarnAppListClient() {
        return YarnAppListClient;
    }

    /**
     * Find an instance of an application belonging to the current user
     * @param appname application name
     * @return the app report or null if none is found
     * @throws YarnException YARN issues
     * @throws IOException IO problems
     */
    private ApplicationReport findInstance(String appname) throws YarnException, IOException {
        return YarnAppListClient.findInstance(appname);
    }

    private RunningApplication findApplication(String appname) throws YarnException, IOException {
        ApplicationReport applicationReport = findInstance(appname);
        return applicationReport != null ? new RunningApplication(yarnClient, applicationReport) : null;
    }

    /**
     * find all live instances of a specific app -if there is >1 in the cluster,
     * this returns them all. State should be running or less
     * @param appname application name
     * @return the list of all matching application instances
     */
    private List<ApplicationReport> findAllLiveInstances(String appname) throws YarnException, IOException {

        return YarnAppListClient.findAllLiveInstances(appname);
    }

    /**
     * Connect to a Slider AM
     * @param app application report providing the details on the application
     * @return an instance
     * @throws YarnException
     * @throws IOException
     */
    private SliderClusterProtocol connect(ApplicationReport app) throws YarnException, IOException {

        try {
            return RpcBinder.getProxy(getConfig(), yarnClient.getRmClient(), app, Constants.CONNECT_TIMEOUT,
                    Constants.RPC_TIMEOUT);
        } catch (InterruptedException e) {
            throw new SliderException(SliderExitCodes.EXIT_TIMED_OUT, e,
                    "Interrupted waiting for communications with the Slider AM");
        }
    }

    @Override
    @VisibleForTesting
    public int actionStatus(String clustername, ActionStatusArgs statusArgs) throws YarnException, IOException {
        verifyBindingsDefined();
        SliderUtils.validateClusterName(clustername);
        String outfile = statusArgs.getOutput();
        ClusterDescription status = getClusterDescription(clustername);
        String text = status.toJsonString();
        if (outfile == null) {
            log.info(text);
        } else {
            status.save(new File(outfile).getAbsoluteFile());
        }
        return EXIT_SUCCESS;
    }

    @Override
    public int actionVersion() {
        SliderVersionInfo.loadAndPrintVersionInfo(log);
        return EXIT_SUCCESS;
    }

    @Override
    public int actionFreeze(String clustername, ActionFreezeArgs freezeArgs) throws YarnException, IOException {
        verifyBindingsDefined();
        SliderUtils.validateClusterName(clustername);
        int waittime = freezeArgs.getWaittime();
        String text = freezeArgs.message;
        boolean forcekill = freezeArgs.force;
        log.debug("actionFreeze({}, reason={}, wait={}, force={})", clustername, text, waittime, forcekill);

        //is this actually a known cluster?
        sliderFileSystem.locateInstanceDefinition(clustername);
        ApplicationReport app = findInstance(clustername);
        if (app == null) {
            // exit early
            log.info("Cluster {} not running", clustername);
            // not an error to stop a stopped cluster
            return EXIT_SUCCESS;
        }
        log.debug("App to stop was found: {}:\n{}", clustername, new SliderUtils.OnDemandReportStringifier(app));
        if (app.getYarnApplicationState().ordinal() >= YarnApplicationState.FINISHED.ordinal()) {
            log.info("Cluster {} is a terminated state {}", clustername, app.getYarnApplicationState());
            return EXIT_SUCCESS;
        }

        // IPC request for a managed shutdown is only possible if the app is running.
        // so we need to force kill if the app is accepted or submitted
        if (!forcekill && app.getYarnApplicationState().ordinal() < YarnApplicationState.RUNNING.ordinal()) {
            log.info("Cluster {} is in a pre-running state {}. Force killing it", clustername,
                    app.getYarnApplicationState());
            forcekill = true;
        }

        LaunchedApplication application = new LaunchedApplication(yarnClient, app);
        applicationId = application.getApplicationId();

        if (forcekill) {
            // escalating to forced kill
            application.kill("Forced stop of " + clustername + ": " + text);
        } else {
            try {
                SliderClusterProtocol appMaster = connect(app);
                Messages.StopClusterRequestProto r = Messages.StopClusterRequestProto.newBuilder().setMessage(text)
                        .build();
                appMaster.stopCluster(r);

                log.debug("Cluster stop command issued");

            } catch (YarnException e) {
                log.warn("Exception while trying to terminate {}: {}", clustername, e);
                return EXIT_FALSE;
            } catch (IOException e) {
                log.warn("Exception while trying to terminate {}: {}", clustername, e);
                return EXIT_FALSE;
            }
        }

        //wait for completion. We don't currently return an exception during this process
        //as the stop operation has been issued, this is just YARN.
        try {
            if (waittime > 0) {
                ApplicationReport applicationReport = application.monitorAppToState(YarnApplicationState.FINISHED,
                        new Duration(waittime * 1000));
                if (applicationReport == null) {
                    log.info("application did not shut down in time");
                    return EXIT_FALSE;
                }
            }

            // JDK7    } catch (YarnException | IOException e) {
        } catch (YarnException e) {
            log.warn("Exception while waiting for the application {} to shut down: {}", clustername, e);
        } catch (IOException e) {
            log.warn("Exception while waiting for the application {} to shut down: {}", clustername, e);
        }

        return EXIT_SUCCESS;
    }

    @Override
    public int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException {
        SliderUtils.validateClusterName(clustername);
        verifyBindingsDefined();
        // see if it is actually running and bail out;
        verifyNoLiveClusters(clustername, "Start");

        //start the cluster
        return startCluster(clustername, thaw);
    }

    /**
     * Implement flexing
     * @param clustername name of the cluster
     * @param roleInstances map of new role instances
     * @return EXIT_SUCCESS if the #of nodes in a live cluster changed
     * @throws YarnException
     * @throws IOException
     */
    public int flex(String clustername, Map<String, Integer> roleInstances) throws YarnException, IOException {
        verifyBindingsDefined();
        SliderUtils.validateClusterName(clustername);
        Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
        AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(clustername, clusterDirectory);

        ConfTreeOperations resources = instanceDefinition.getResourceOperations();
        for (Map.Entry<String, Integer> entry : roleInstances.entrySet()) {
            String role = entry.getKey();
            int count = entry.getValue();
            resources.getOrAddComponent(role).put(ResourceKeys.COMPONENT_INSTANCES, Integer.toString(count));

            log.debug("Flexed cluster specification ( {} -> {}) : \n{}", role, count, resources);
        }
        SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig());
        AbstractClientProvider provider = createClientProvider(instanceDefinition.getInternalOperations()
                .getGlobalOptions().getMandatoryOption(InternalKeys.INTERNAL_PROVIDER_NAME));
        // slider provider to validate what there is
        validateInstanceDefinition(sliderAM, instanceDefinition, sliderFileSystem);
        validateInstanceDefinition(provider, instanceDefinition, sliderFileSystem);

        int exitCode = EXIT_FALSE;
        // save the specification
        try {
            InstanceIO.updateInstanceDefinition(sliderFileSystem, clusterDirectory, instanceDefinition);
        } catch (LockAcquireFailedException e) {
            // lock failure
            log.debug("Failed to lock dir {}", clusterDirectory, e);
            log.warn("Failed to save new resource definition to {} : {}", clusterDirectory, e);
        }

        // now see if it is actually running and tell it about the update if it is
        ApplicationReport instance = findInstance(clustername);
        if (instance != null) {
            log.info("Flexing running cluster");
            SliderClusterProtocol appMaster = connect(instance);
            SliderClusterOperations clusterOps = new SliderClusterOperations(appMaster);
            clusterOps.flex(instanceDefinition.getResources());
            log.info("application instance size updated");
            exitCode = EXIT_SUCCESS;
        } else {
            log.info("No running instance to update");
        }
        return exitCode;
    }

    /**
     * Validate an instance definition against a provider.
     * @param provider the provider performing the validation
     * @param instanceDefinition the instance definition
     * @throws SliderException if invalid.
     */
    protected void validateInstanceDefinition(AbstractClientProvider provider, AggregateConf instanceDefinition,
            SliderFileSystem fs) throws SliderException {
        try {
            provider.validateInstanceDefinition(instanceDefinition, fs);
        } catch (SliderException e) {
            //problem, reject it
            log.info("Error {} validating application instance definition ", e.getMessage());
            log.debug("Error validating application instance definition ", e);
            log.info(instanceDefinition.toString());
            throw e;
        }
    }

    /**
     * Load the persistent cluster description
     * @param clustername name of the cluster
     * @return the description in the filesystem
     * @throws IOException any problems loading -including a missing file
     */
    @VisibleForTesting
    public AggregateConf loadPersistedClusterDescription(String clustername)
            throws IOException, SliderException, LockAcquireFailedException {
        Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
        ConfPersister persister = new ConfPersister(sliderFileSystem, clusterDirectory);
        AggregateConf instanceDescription = new AggregateConf();
        persister.load(instanceDescription);
        return instanceDescription;
    }

    /**
     * Connect to a live cluster and get its current state
     * @param clustername the cluster name
     * @return its description
     */
    @VisibleForTesting
    public ClusterDescription getClusterDescription(String clustername) throws YarnException, IOException {
        SliderClusterOperations clusterOperations = createClusterOperations(clustername);
        return clusterOperations.getClusterDescription();
    }

    /**
     * Connect to the cluster and get its current state
     * @return its description
     */
    @VisibleForTesting
    public ClusterDescription getClusterDescription() throws YarnException, IOException {
        return getClusterDescription(getDeployedClusterName());
    }

    /**
     * List all node UUIDs in a role
     * @param role role name or "" for all
     * @return an array of UUID strings
     * @throws IOException
     * @throws YarnException
     */
    @VisibleForTesting
    public String[] listNodeUUIDsByRole(String role) throws IOException, YarnException {
        return createClusterOperations().listNodeUUIDsByRole(role);
    }

    /**
     * List all nodes in a role. This is a double round trip: once to list
     * the nodes in a role, another to get their details
     * @param role component/role to look for
     * @return an array of ContainerNode instances
     * @throws IOException
     * @throws YarnException
     */
    @VisibleForTesting
    public List<ClusterNode> listClusterNodesInRole(String role) throws IOException, YarnException {
        return createClusterOperations().listClusterNodesInRole(role);
    }

    /**
     * Get the details on a list of uuids
     * @param uuids uuids to ask for 
     * @return a possibly empty list of node details
     * @throws IOException
     * @throws YarnException
     */
    @VisibleForTesting
    public List<ClusterNode> listClusterNodes(String[] uuids) throws IOException, YarnException {

        if (uuids.length == 0) {
            // short cut on an empty list
            return new LinkedList<ClusterNode>();
        }
        return createClusterOperations().listClusterNodes(uuids);
    }

    /**
     * Get a node from the AM
     * @param uuid uuid of node
     * @return deserialized node
     * @throws IOException IO problems
     * @throws NoSuchNodeException if the node isn't found
     */
    @VisibleForTesting
    public ClusterNode getNode(String uuid) throws IOException, YarnException {
        return createClusterOperations().getNode(uuid);
    }

    /**
     * Get the instance definition from the far end
     */
    @VisibleForTesting
    public AggregateConf getLiveInstanceDefinition() throws IOException, YarnException {
        return createClusterOperations().getInstanceDefinition();
    }

    /**
     * Bond to a running cluster
     * @param clustername cluster name
     * @return the AM RPC client
     * @throws SliderException if the cluster is unkown
     */
    private SliderClusterProtocol bondToCluster(String clustername) throws YarnException, IOException {
        verifyBindingsDefined();
        if (clustername == null) {
            throw unknownClusterException("(undefined)");
        }
        ApplicationReport instance = findInstance(clustername);
        if (null == instance) {
            throw unknownClusterException(clustername);
        }
        return connect(instance);
    }

    /**
     * Create a cluster operations instance against a given cluster
     * @param clustername cluster name
     * @return a bonded cluster operations instance
     * @throws YarnException YARN issues
     * @throws IOException IO problems
     */
    private SliderClusterOperations createClusterOperations(String clustername) throws YarnException, IOException {
        SliderClusterProtocol sliderAM = bondToCluster(clustername);
        return new SliderClusterOperations(sliderAM);
    }

    /**
     * Create a cluster operations instance against the active cluster
     * -returning any previous created one if held.
     * @return a bonded cluster operations instance
     * @throws YarnException YARN issues
     * @throws IOException IO problems
     */
    public SliderClusterOperations createClusterOperations() throws YarnException, IOException {
        if (sliderClusterOperations == null) {
            sliderClusterOperations = createClusterOperations(getDeployedClusterName());
        }
        return sliderClusterOperations;
    }

    /**
     * Wait for an instance of a named role to be live (or past it in the lifecycle)
     * @param role role to look for
     * @param timeout time to wait
     * @return the state. If still in CREATED, the cluster didn't come up
     * in the time period. If LIVE, all is well. If >LIVE, it has shut for a reason
     * @throws IOException IO
     * @throws SliderException Slider
     * @throws WaitTimeoutException if the wait timed out
     */
    @VisibleForTesting
    public int waitForRoleInstanceLive(String role, long timeout)
            throws WaitTimeoutException, IOException, YarnException {
        return createClusterOperations().waitForRoleInstanceLive(role, timeout);
    }

    /**
     * Generate an exception for an unknown cluster
     * @param clustername cluster name
     * @return an exception with text and a relevant exit code
     */
    public UnknownApplicationInstanceException unknownClusterException(String clustername) {
        return UnknownApplicationInstanceException.unknownInstance(clustername);
    }

    @Override
    public String toString() {
        return "Slider Client in state " + getServiceState() + " and Slider Application Instance "
                + deployedClusterName;
    }

    /**
     * Get all YARN applications
     * @return a possibly empty list
     * @throws YarnException
     * @throws IOException
     */
    @VisibleForTesting
    public List<ApplicationReport> getApplications() throws YarnException, IOException {
        return yarnClient.getApplications();
    }

    @VisibleForTesting
    public ApplicationReport getApplicationReport(ApplicationId appId) throws YarnException, IOException {
        return new LaunchedApplication(appId, yarnClient).getApplicationReport();
    }

    /**
     * The configuration used for deployment (after resolution)
     * @return
     */
    @VisibleForTesting
    public AggregateConf getLaunchedInstanceDefinition() {
        return launchedInstanceDefinition;
    }

    @Override
    public int actionResolve(ActionResolveArgs args) throws YarnException, IOException {
        // as this is an API entry point, validate
        // the arguments
        args.validate();
        RegistryOperations operations = getRegistryOperations();
        String path = SliderRegistryUtils.resolvePath(args.path);
        ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal();
        try {
            if (args.list) {
                File destDir = args.destdir;
                if (destDir != null) {
                    destDir.mkdirs();
                }

                Map<String, ServiceRecord> recordMap;
                Map<String, RegistryPathStatus> znodes;
                try {
                    znodes = statChildren(registryOperations, path);
                    recordMap = extractServiceRecords(registryOperations, path, znodes.values());
                } catch (PathNotFoundException e) {
                    // treat the root directory as if if is always there

                    if ("/".equals(path)) {
                        znodes = new HashMap<String, RegistryPathStatus>(0);
                        recordMap = new HashMap<String, ServiceRecord>(0);
                    } else {
                        throw e;
                    }
                }
                // subtract all records from the znodes map to get pure directories
                log.info("Entries: {}", znodes.size());

                for (String name : znodes.keySet()) {
                    println("  " + name);
                }
                println("");

                log.info("Service records: {}", recordMap.size());
                for (Entry<String, ServiceRecord> recordEntry : recordMap.entrySet()) {
                    String name = recordEntry.getKey();
                    ServiceRecord instance = recordEntry.getValue();
                    String json = serviceRecordMarshal.toJson(instance);
                    if (destDir == null) {
                        println(name);
                        println(json);
                    } else {
                        String filename = RegistryPathUtils.lastPathEntry(name) + ".json";
                        File jsonFile = new File(destDir, filename);
                        SliderUtils.write(jsonFile, serviceRecordMarshal.toBytes(instance), true);
                    }
                }
            } else {
                // resolve single entry
                ServiceRecord instance = resolve(path);
                File outFile = args.out;
                if (args.destdir != null) {
                    outFile = new File(args.destdir, RegistryPathUtils.lastPathEntry(path));
                }
                if (outFile != null) {
                    SliderUtils.write(outFile, serviceRecordMarshal.toBytes(instance), true);
                } else {
                    println(serviceRecordMarshal.toJson(instance));
                }
            }
            //      TODO JDK7
        } catch (PathNotFoundException e) {
            // no record at this path
            throw new NotFoundException(e, path);
        } catch (NoRecordException e) {
            throw new NotFoundException(e, path);
        }
        return EXIT_SUCCESS;
    }

    @Override
    public int actionRegistry(ActionRegistryArgs registryArgs) throws YarnException, IOException {
        // as this is also a test entry point, validate
        // the arguments
        registryArgs.validate();
        try {
            if (registryArgs.list) {
                actionRegistryList(registryArgs);
            } else if (registryArgs.listConf) {
                // list the configurations
                actionRegistryListConfigsYarn(registryArgs);
            } else if (registryArgs.listExports) {
                // list the exports
                actionRegistryListExports(registryArgs);
            } else if (SliderUtils.isSet(registryArgs.getConf)) {
                // get a configuration
                PublishedConfiguration publishedConfiguration = actionRegistryGetConfig(registryArgs);
                outputConfig(publishedConfiguration, registryArgs);
            } else if (SliderUtils.isSet(registryArgs.getExport)) {
                // get a export group
                PublishedExports publishedExports = actionRegistryGetExport(registryArgs);
                outputExport(publishedExports, registryArgs);
            } else {
                // it's an unknown command
                log.info(CommonArgs.usage(serviceArgs, ACTION_DIAGNOSTICS));
                return EXIT_USAGE;
            }
            //      JDK7
        } catch (FileNotFoundException e) {
            log.info("{}", e);
            log.debug("{}", e, e);
            return EXIT_NOT_FOUND;
        } catch (PathNotFoundException e) {
            log.info("{}", e);
            log.debug("{}", e, e);
            return EXIT_NOT_FOUND;
        }
        return EXIT_SUCCESS;
    }

    /**
     * Registry operation
     *
     * @param registryArgs registry Arguments
     * @return the instances (for tests)
     * @throws YarnException YARN problems
     * @throws IOException Network or other problems
     */
    @VisibleForTesting
    public Collection<ServiceRecord> actionRegistryList(ActionRegistryArgs registryArgs)
            throws YarnException, IOException {
        String serviceType = registryArgs.serviceType;
        String name = registryArgs.name;
        RegistryOperations operations = getRegistryOperations();
        Collection<ServiceRecord> serviceRecords;
        if (StringUtils.isEmpty(name)) {
            String path = serviceclassPath(currentUser(), serviceType);

            try {
                Map<String, ServiceRecord> recordMap = listServiceRecords(operations, path);
                if (recordMap.isEmpty()) {
                    throw new UnknownApplicationInstanceException("No applications registered under " + path);
                }
                serviceRecords = recordMap.values();
            } catch (PathNotFoundException e) {
                throw new NotFoundException(path, e);
            }
        } else {
            ServiceRecord instance = lookupServiceRecord(registryArgs);
            serviceRecords = new ArrayList<ServiceRecord>(1);
            serviceRecords.add(instance);
        }

        for (ServiceRecord serviceRecord : serviceRecords) {
            logInstance(serviceRecord, registryArgs.verbose);
        }
        return serviceRecords;
    }

    @Override
    public int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) {
        try {
            if (diagnosticArgs.client) {
                actionDiagnosticClient(diagnosticArgs);
            } else if (diagnosticArgs.application) {
                actionDiagnosticApplication(diagnosticArgs);
            } else if (diagnosticArgs.yarn) {
                actionDiagnosticYarn(diagnosticArgs);
            } else if (diagnosticArgs.credentials) {
                actionDiagnosticCredentials();
            } else if (diagnosticArgs.all) {
                actionDiagnosticAll(diagnosticArgs);
            } else if (diagnosticArgs.level) {
                actionDiagnosticIntelligent(diagnosticArgs);
            } else {
                // it's an unknown option
                log.info(CommonArgs.usage(serviceArgs, ACTION_DIAGNOSTICS));
                return EXIT_USAGE;
            }
        } catch (Exception e) {
            log.error(e.toString());
            return EXIT_FALSE;
        }
        return EXIT_SUCCESS;
    }

    private void actionDiagnosticIntelligent(ActionDiagnosticArgs diagnosticArgs)
            throws YarnException, IOException, URISyntaxException {
        // not using member variable clustername because we want to place
        // application name after --application option and member variable
        // cluster name has to be put behind action
        String clusterName = diagnosticArgs.name;
        if (SliderUtils.isUnset(clusterName)) {
            throw new BadCommandArgumentsException("application name must be provided with --name option");
        }

        try {
            SliderUtils.validateClientConfigFile();
            log.info("Slider-client.xml is accessible");
        } catch (IOException e) {
            // we are catching exceptions here because those are indication of
            // validation result, and we need to print them here
            log.error("validation of slider-client.xml fails because: " + e.toString(), e);
            return;
        }
        SliderClusterOperations clusterOperations = createClusterOperations(clusterName);
        // cluster not found exceptions will be thrown upstream
        ClusterDescription clusterDescription = clusterOperations.getClusterDescription();
        log.info("Slider AppMaster is accessible");

        if (clusterDescription.state == ClusterDescription.STATE_LIVE) {
            AggregateConf instanceDefinition = clusterOperations.getInstanceDefinition();
            String imagePath = instanceDefinition.getInternalOperations()
                    .get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
            // if null, that means slider uploaded the agent tarball for the user
            // and we need to use where slider has put
            if (imagePath == null) {
                ApplicationReport appReport = findInstance(clusterName);
                Path path1 = sliderFileSystem.getTempPathForCluster(clusterName);

                Path subPath = new Path(path1, appReport.getApplicationId().toString() + "/agent");
                imagePath = subPath.toString();
            }
            try {
                SliderUtils.validateHDFSFile(sliderFileSystem, imagePath + "/" + AGENT_TAR);
                log.info("Slider agent package is properly installed");
            } catch (FileNotFoundException e) {
                log.error("can not find agent package: " + e.toString());
                return;
            } catch (IOException e) {
                log.error("can not open agent package: " + e.toString());
                return;
            }
            String pkgTarballPath = instanceDefinition.getAppConfOperations().getGlobalOptions()
                    .getMandatoryOption(AgentKeys.APP_DEF);
            try {
                SliderUtils.validateHDFSFile(sliderFileSystem, pkgTarballPath);
                log.info("Application package is properly installed");
            } catch (FileNotFoundException e) {
                log.error("can not find application package: {}", e);
                return;
            } catch (IOException e) {
                log.error("can not open application package: {} ", e);
                return;
            }
        }
    }

    private void actionDiagnosticAll(ActionDiagnosticArgs diagnosticArgs) throws IOException, YarnException {
        // assign application name from param to each sub diagnostic function
        actionDiagnosticClient(diagnosticArgs);
        actionDiagnosticApplication(diagnosticArgs);
        actionDiagnosticSlider(diagnosticArgs);
        actionDiagnosticYarn(diagnosticArgs);
        actionDiagnosticCredentials();
    }

    private void actionDiagnosticCredentials() throws BadConfigException, IOException {
        if (SliderUtils.isHadoopClusterSecure(SliderUtils.loadClientConfigurationResource())) {
            String credentialCacheFileDescription = null;
            try {
                credentialCacheFileDescription = SliderUtils.checkCredentialCacheFile();
            } catch (BadConfigException e) {
                log.error("The credential config is not valid: " + e.toString());
                throw e;
            } catch (IOException e) {
                log.error("Unable to read the credential file: " + e.toString());
                throw e;
            }
            log.info("Credential cache file for the current user: " + credentialCacheFileDescription);
        } else {
            log.info("the cluster is not in secure mode");
        }
    }

    private void actionDiagnosticYarn(ActionDiagnosticArgs diagnosticArgs) throws IOException, YarnException {
        JSONObject converter = null;
        log.info("the node in the YARN cluster has below state: ");
        List<NodeReport> yarnClusterInfo;
        try {
            yarnClusterInfo = yarnClient.getNodeReports(NodeState.RUNNING);
        } catch (YarnException e1) {
            log.error("Exception happened when fetching node report from the YARN cluster: " + e1.toString());
            throw e1;
        } catch (IOException e1) {
            log.error("Network problem happened when fetching node report YARN cluster: " + e1.toString());
            throw e1;
        }
        for (NodeReport nodeReport : yarnClusterInfo) {
            log.info(nodeReport.toString());
        }

        if (diagnosticArgs.verbose) {
            Writer configWriter = new StringWriter();
            try {
                Configuration.dumpConfiguration(yarnClient.getConfig(), configWriter);
            } catch (IOException e1) {
                log.error("Network problem happened when retrieving YARN config from YARN: " + e1.toString());
                throw e1;
            }
            try {
                converter = new JSONObject(configWriter.toString());
                log.info("the configuration of the YARN cluster is: " + converter.toString(2));

            } catch (JSONException e) {
                log.error("JSONException happened during parsing response from YARN: " + e.toString());
            }
        }
    }

    private void actionDiagnosticSlider(ActionDiagnosticArgs diagnosticArgs) throws YarnException, IOException {
        // not using member variable clustername because we want to place
        // application name after --application option and member variable
        // cluster name has to be put behind action
        String clusterName = diagnosticArgs.name;
        if (SliderUtils.isUnset(clusterName)) {
            throw new BadCommandArgumentsException("application name must be provided with --name option");
        }
        SliderClusterOperations clusterOperations;
        AggregateConf instanceDefinition = null;
        try {
            clusterOperations = createClusterOperations(clusterName);
            instanceDefinition = clusterOperations.getInstanceDefinition();
        } catch (YarnException e) {
            log.error("Exception happened when retrieving instance definition from YARN: " + e.toString());
            throw e;
        } catch (IOException e) {
            log.error("Network problem happened when retrieving instance definition from YARN: " + e.toString());
            throw e;
        }
        String imagePath = instanceDefinition.getInternalOperations()
                .get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
        // if null, it will be uploaded by Slider and thus at slider's path
        if (imagePath == null) {
            ApplicationReport appReport = findInstance(clusterName);
            Path path1 = sliderFileSystem.getTempPathForCluster(clusterName);
            Path subPath = new Path(path1, appReport.getApplicationId().toString() + "/agent");
            imagePath = subPath.toString();
        }
        log.info("The path of slider agent tarball on HDFS is: " + imagePath);
    }

    private void actionDiagnosticApplication(ActionDiagnosticArgs diagnosticArgs)
            throws YarnException, IOException {
        // not using member variable clustername because we want to place
        // application name after --application option and member variable
        // cluster name has to be put behind action
        String clusterName = diagnosticArgs.name;
        if (SliderUtils.isUnset(clusterName)) {
            throw new BadCommandArgumentsException("application name must be provided with --name option");
        }
        SliderClusterOperations clusterOperations;
        AggregateConf instanceDefinition = null;
        try {
            clusterOperations = createClusterOperations(clusterName);
            instanceDefinition = clusterOperations.getInstanceDefinition();
        } catch (YarnException e) {
            log.error("Exception happened when retrieving instance definition from YARN: " + e.toString());
            throw e;
        } catch (IOException e) {
            log.error("Network problem happened when retrieving instance definition from YARN: " + e.toString());
            throw e;
        }
        String clusterDir = instanceDefinition.getAppConfOperations().getGlobalOptions().get(AgentKeys.APP_ROOT);
        String pkgTarball = instanceDefinition.getAppConfOperations().getGlobalOptions().get(AgentKeys.APP_DEF);
        String runAsUser = instanceDefinition.getAppConfOperations().getGlobalOptions().get(AgentKeys.RUNAS_USER);

        log.info("The location of the cluster instance directory in HDFS is: " + clusterDir);
        log.info("The name of the application package tarball on HDFS is: " + pkgTarball);
        log.info("The runas user of the application in the cluster is: " + runAsUser);

        if (diagnosticArgs.verbose) {
            log.info("App config of the application: " + instanceDefinition.getAppConf().toJson());
            log.info("Resource config of the application: " + instanceDefinition.getResources().toJson());
        }
    }

    private void actionDiagnosticClient(ActionDiagnosticArgs diagnosticArgs) throws SliderException, IOException {
        try {
            String currentCommandPath = SliderUtils.getCurrentCommandPath();
            SliderVersionInfo.loadAndPrintVersionInfo(log);
            String clientConfigPath = SliderUtils.getClientConfigPath();
            String jdkInfo = SliderUtils.getJDKInfo();
            println("The slider command path: %s", currentCommandPath);
            println("The slider-client.xml used by current running command path: %s", clientConfigPath);
            println(jdkInfo);

            // security info
            Configuration config = getConfig();
            if (SliderUtils.isHadoopClusterSecure(config)) {
                println("Hadoop Cluster is secure");
                println("Login user is %s", UserGroupInformation.getLoginUser());
                println("Current user is %s", UserGroupInformation.getCurrentUser());

            } else {
                println("Hadoop Cluster is insecure");
            }

            // verbose?
            if (diagnosticArgs.verbose) {
                // do the environment
                Map<String, String> env = System.getenv();
                Set<String> envList = ConfigHelper.sortedConfigKeys(env.entrySet());
                StringBuilder builder = new StringBuilder("Environment variables:\n");
                for (String key : envList) {
                    builder.append(key).append("=").append(env.get(key)).append("\n");
                }
                println(builder.toString());

                // Java properties
                builder = new StringBuilder("JVM Properties\n");
                Map<String, String> props = SliderUtils.sortedMap(SliderUtils.toMap(System.getProperties()));
                for (Entry<String, String> entry : props.entrySet()) {
                    builder.append(entry.getKey()).append("=").append(entry.getValue()).append("\n");
                }

                println(builder.toString());

                // then the config
                println("Slider client configuration:\n" + ConfigHelper.dumpConfigToString(config));

            }

            SliderUtils.validateSliderClientEnvironment(log);
        } catch (SliderException e) {
            log.error(e.toString());
            throw e;
        } catch (IOException e) {
            log.error(e.toString());
            throw e;
        }

    }

    /**
     * Log a service record instance
     * @param instance record
     * @param verbose verbose logging of all external endpoints
     */
    private void logInstance(ServiceRecord instance, boolean verbose) {
        if (!verbose) {
            log.info("{}", instance.get(YarnRegistryAttributes.YARN_ID, ""));
        } else {
            log.info("{}: ", instance.get(YarnRegistryAttributes.YARN_ID, ""));
            logEndpoints(instance);
        }
    }

    /**
     * Log the external endpoints of a service record
     * @param instance service record instance
     */
    private void logEndpoints(ServiceRecord instance) {
        List<Endpoint> endpoints = instance.external;
        for (Endpoint endpoint : endpoints) {
            log.info(endpoint.toString());
        }
    }

    /**
      * list configs available for an instance
      *
      * @param registryArgs registry Arguments
      * @throws YarnException YARN problems
      * @throws IOException Network or other problems
      */
    public void actionRegistryListConfigsYarn(ActionRegistryArgs registryArgs) throws YarnException, IOException {

        ServiceRecord instance = lookupServiceRecord(registryArgs);

        RegistryRetriever retriever = new RegistryRetriever(instance);
        PublishedConfigSet configurations = retriever.getConfigurations(!registryArgs.internal);
        PrintStream out = null;
        try {
            if (registryArgs.out != null) {
                out = new PrintStream(new FileOutputStream(registryArgs.out));
            } else {
                out = System.out;
            }
            for (String configName : configurations.keys()) {
                if (!registryArgs.verbose) {
                    out.println(configName);
                } else {
                    PublishedConfiguration published = configurations.get(configName);
                    out.printf("%s: %s\n", configName, published.description);
                }
            }
        } finally {
            if (registryArgs.out != null && out != null) {
                out.flush();
                out.close();
            }
        }
    }

    /**
     * list exports available for an instance
     *
     * @param registryArgs registry Arguments
     * @throws YarnException YARN problems
     * @throws IOException Network or other problems
     */
    public void actionRegistryListExports(ActionRegistryArgs registryArgs) throws YarnException, IOException {
        ServiceRecord instance = lookupServiceRecord(registryArgs);

        RegistryRetriever retriever = new RegistryRetriever(instance);
        PublishedExportsSet exports = retriever.getExports(!registryArgs.internal);
        PrintStream out = null;
        boolean streaming = false;
        try {
            if (registryArgs.out != null) {
                out = new PrintStream(new FileOutputStream(registryArgs.out));
                streaming = true;
                log.debug("Saving output to {}", registryArgs.out);
            } else {
                out = System.out;
            }
            log.debug("Number of exports: {}", exports.keys().size());
            for (String exportName : exports.keys()) {
                if (streaming) {
                    log.debug(exportName);
                }
                if (!registryArgs.verbose) {
                    out.println(exportName);
                } else {
                    PublishedExports published = exports.get(exportName);
                    out.printf("%s: %s\n", exportName, published.description);
                }
            }
        } finally {
            if (streaming) {
                out.flush();
                out.close();
            }
        }
    }

    /**
     * list configs available for an instance
     *
     * @param registryArgs registry Arguments
     * @throws YarnException YARN problems
     * @throws IOException Network or other problems
     * @throws FileNotFoundException if the config is not found
     */
    @VisibleForTesting
    public PublishedConfiguration actionRegistryGetConfig(ActionRegistryArgs registryArgs)
            throws YarnException, IOException {
        ServiceRecord instance = lookupServiceRecord(registryArgs);

        RegistryRetriever retriever = new RegistryRetriever(instance);
        boolean external = !registryArgs.internal;
        PublishedConfigSet configurations = retriever.getConfigurations(external);

        PublishedConfiguration published = retriever.retrieveConfiguration(configurations, registryArgs.getConf,
                external);
        return published;
    }

    /**
     * get a specific export group
     *
     * @param registryArgs registry Arguments
     *
     * @throws YarnException         YARN problems
     * @throws IOException           Network or other problems
     * @throws FileNotFoundException if the config is not found
     */
    @VisibleForTesting
    public PublishedExports actionRegistryGetExport(ActionRegistryArgs registryArgs)
            throws YarnException, IOException {
        ServiceRecord instance = lookupServiceRecord(registryArgs);

        RegistryRetriever retriever = new RegistryRetriever(instance);
        boolean external = !registryArgs.internal;
        PublishedExportsSet exports = retriever.getExports(external);

        PublishedExports published = retriever.retrieveExports(exports, registryArgs.getExport, external);
        return published;
    }

    /**
     * write out the config. If a destination is provided and that dir is a
     * directory, the entry is written to it with the name provided + extension,
     * else it is printed to standard out.
     * @param published published config
     * @param registryArgs registry Arguments
     * @throws BadCommandArgumentsException
     * @throws IOException
     */
    private void outputConfig(PublishedConfiguration published, ActionRegistryArgs registryArgs)
            throws BadCommandArgumentsException, IOException {
        // decide whether or not to print
        String entry = registryArgs.getConf;
        String format = registryArgs.format;
        ConfigFormat configFormat = ConfigFormat.resolve(format);
        if (configFormat == null) {
            throw new BadCommandArgumentsException("Unknown/Unsupported format %s ", format);
        }
        PublishedConfigurationOutputter outputter = PublishedConfigurationOutputter.createOutputter(configFormat,
                published);
        boolean print = registryArgs.out == null;
        if (!print) {
            File outputPath = registryArgs.out;
            if (outputPath.isDirectory()) {
                // creating it under a directory
                outputPath = new File(outputPath, entry + "." + format);
            }
            log.debug("Destination path: {}", outputPath);
            outputter.save(outputPath);
        } else {
            print(outputter.asString());
        }

    }

    /**
     * write out the config
     * @param published
     * @param registryArgs
     * @throws BadCommandArgumentsException
     * @throws IOException
     */
    private void outputExport(PublishedExports published, ActionRegistryArgs registryArgs)
            throws BadCommandArgumentsException, IOException {
        // decide whether or not to print
        String entry = registryArgs.getExport;
        String format = ConfigFormat.JSON.toString();
        ConfigFormat configFormat = ConfigFormat.resolve(format);
        if (configFormat == null || configFormat != ConfigFormat.JSON) {
            throw new BadCommandArgumentsException("Unknown/Unsupported format %s . Only JSON is supported.",
                    format);
        }

        PublishedExportsOutputter outputter = PublishedExportsOutputter.createOutputter(configFormat, published);
        boolean print = registryArgs.out == null;
        if (!print) {
            File destFile;
            destFile = registryArgs.out;
            if (destFile.isDirectory()) {
                // creating it under a directory
                destFile = new File(destFile, entry + "." + format);
            }
            log.info("Destination path: {}", destFile);
            outputter.save(destFile);
        } else {
            print(outputter.asString());
        }
    }

    /**
     * Look up an instance
     * @return instance data
     * @throws SliderException other failures
     * @throws IOException IO problems or wrapped exceptions
     */
    private ServiceRecord lookupServiceRecord(ActionRegistryArgs registryArgs) throws SliderException, IOException {
        String user;
        if (StringUtils.isNotEmpty(registryArgs.user)) {
            user = RegistryPathUtils.encodeForRegistry(registryArgs.user);
        } else {
            user = currentUser();
        }

        String path = servicePath(user, registryArgs.serviceType, registryArgs.name);
        return resolve(path);
    }

    /**
     * Look up a service record of the current user
     * @param serviceType service type
     * @param id instance ID
     * @return instance data
     * @throws UnknownApplicationInstanceException no path or service record
     * at the end of the path
     * @throws SliderException other failures
     * @throws IOException IO problems or wrapped exceptions
     */
    public ServiceRecord lookupServiceRecord(String serviceType, String id) throws IOException, SliderException {
        String path = servicePath(currentUser(), serviceType, id);
        return resolve(path);
    }

    /**
     * 
     * Look up an instance
     * @param path path
     * @return instance data
     * @throws NotFoundException no path/no service record
     * at the end of the path
     * @throws SliderException other failures
     * @throws IOException IO problems or wrapped exceptions
     */
    public ServiceRecord resolve(String path) throws IOException, SliderException {
        try {
            return getRegistryOperations().resolve(path);
        } catch (PathNotFoundException e) {
            throw new NotFoundException(e.getPath().toString(), e);
        } catch (NoRecordException e) {
            throw new NotFoundException(e.getPath().toString(), e);
        }
    }

    /**
     * List instances in the registry for the current user
     * @return a list of slider registry instances
     * @throws IOException Any IO problem ... including no path in the registry
     * to slider service classes for this user
     * @throws SliderException other failures
     */

    public Map<String, ServiceRecord> listRegistryInstances() throws IOException, SliderException {
        Map<String, ServiceRecord> recordMap = listServiceRecords(getRegistryOperations(),
                serviceclassPath(currentUser(), SliderKeys.APP_TYPE));
        return recordMap;
    }

    /**
     * List instances in the registry
     * @return the instance IDs
     * @throws IOException
     * @throws YarnException
     */
    public List<String> listRegisteredSliderInstances() throws IOException, YarnException {
        try {
            Map<String, ServiceRecord> recordMap = listServiceRecords(getRegistryOperations(),
                    serviceclassPath(currentUser(), SliderKeys.APP_TYPE));
            return new ArrayList<String>(recordMap.keySet());
            /// JDK7    } catch (YarnException | IOException e) {
        } catch (IOException e) {
            throw e;
        } catch (YarnException e) {
            throw e;
        } catch (Exception e) {
            throw new IOException(e);
        }
    }

    /**
     * Start the registry if it is not there yet
     * @return the registry service
     * @throws SliderException
     * @throws IOException
     */
    private synchronized RegistryOperations maybeStartYarnRegistry() throws SliderException, IOException {

        if (registryOperations == null) {
            registryOperations = startRegistryOperationsService();
        }
        return registryOperations;
    }

    @Override
    public RegistryOperations getRegistryOperations() throws SliderException, IOException {
        return maybeStartYarnRegistry();
    }

    /**
     * Output to standard out/stderr (implementation specific detail)
     * @param src source
     */
    @SuppressWarnings("UseOfSystemOutOrSystemErr")
    private static void print(CharSequence src) {
        System.out.append(src);
    }

    /**
     * Output to standard out/stderr with a newline after
     * @param message message
     */
    private static void println(String message) {
        print(message);
        print("\n");
    }

    /**
     * Output to standard out/stderr with a newline after, formatted
     * @param message message
     * @param args arguments for string formatting
     */
    private static void println(String message, Object... args) {
        print(String.format(message, args));
        print("\n");
    }

    /**
     * Implement the lookup action.
     * @param args Action arguments
     * @return 0 if the entry was found
     * @throws IOException
     * @throws YarnException
     * @throws UnknownApplicationInstanceException if a specific instance
     * was named but it was not found
     */
    @VisibleForTesting
    public int actionLookup(ActionLookupArgs args) throws IOException, YarnException {
        verifyBindingsDefined();
        try {
            ApplicationId id = ConverterUtils.toApplicationId(args.id);
            ApplicationReport report = yarnClient.getApplicationReport(id);
            SerializedApplicationReport sar = new SerializedApplicationReport(report);
            ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser();
            if (args.outputFile != null) {
                serDeser.save(sar, args.outputFile);
            } else {
                println(serDeser.toJson(sar));
            }
        } catch (IllegalArgumentException e) {
            throw new BadCommandArgumentsException(e, "%s : %s", args, e);
        } catch (ApplicationAttemptNotFoundException notFound) {
            throw new NotFoundException(notFound, notFound.toString());
        } catch (ApplicationNotFoundException notFound) {
            throw new NotFoundException(notFound, notFound.toString());
        }
        return EXIT_SUCCESS;
    }
}