org.wso2.appcloud.provisioning.runtime.KubernetesRuntimeProvisioningService.java Source code

Java tutorial

Introduction

Here is the source code for org.wso2.appcloud.provisioning.runtime.KubernetesRuntimeProvisioningService.java

Source

/*
* Copyright (c) 2016, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.wso2.appcloud.provisioning.runtime;

import io.fabric8.kubernetes.api.KubernetesHelper;
import io.fabric8.kubernetes.api.model.*;
import io.fabric8.kubernetes.api.model.extensions.*;
import io.fabric8.kubernetes.client.AutoAdaptableKubernetesClient;
import io.fabric8.kubernetes.client.KubernetesClientException;
import io.fabric8.kubernetes.client.dsl.LogWatch;
import io.fabric8.kubernetes.client.dsl.PrettyLoggable;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.json.JSONArray;
import org.json.JSONObject;
import org.wso2.appcloud.provisioning.runtime.Utils.KubernetesProvisioningUtils;
import org.wso2.appcloud.provisioning.runtime.beans.*;
import org.wso2.appcloud.provisioning.runtime.beans.Container;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.*;

/**
 * This class will implement the runtime provisioning service specific to Kubernetes.
 */
public class KubernetesRuntimeProvisioningService implements RuntimeProvisioningService {

    private static final Log log = LogFactory.getLog(KubernetesRuntimeProvisioningService.class);
    private ApplicationContext applicationContext;
    private Namespace namespace;
    private ResourceQuotaLimit resourceQuotaLimit;

    public KubernetesRuntimeProvisioningService(ApplicationContext applicationContext,
            ResourceQuotaLimit resourceQuotaLimit) {
        this.applicationContext = applicationContext;
        this.namespace = KubernetesProvisioningUtils.getNameSpace(applicationContext);
        this.resourceQuotaLimit = resourceQuotaLimit;

        //Creating namespace in kubernetes if not available
        AutoAdaptableKubernetesClient kubernetesClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        NamespaceList namespaceList = kubernetesClient.namespaces().list();
        boolean isNamespaceExists = false;
        for (Namespace ns : namespaceList.getItems()) {
            if (ns.getMetadata().getName().equals(this.namespace.getMetadata().getName())) {
                isNamespaceExists = true;
                if (log.isDebugEnabled()) {
                    log.debug("Namespace found: " + ns.getMetadata().getName());
                }
                break;
            }
        }
        if (!isNamespaceExists) {
            if (log.isDebugEnabled()) {
                log.debug("Namespace not available hence creating namespace: " + namespace.getMetadata().getName());
            }
            kubernetesClient.namespaces().create(namespace);
        }
    }

    public KubernetesRuntimeProvisioningService(ApplicationContext applicationContext) {
        this.applicationContext = applicationContext;
        this.namespace = KubernetesProvisioningUtils.getNameSpace(applicationContext);

        //Creating namespace in kubernetes if not available
        AutoAdaptableKubernetesClient kubernetesClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        NamespaceList namespaceList = kubernetesClient.namespaces().list();
        boolean isNamespaceExists = false;
        for (Namespace ns : namespaceList.getItems()) {
            if (ns.getMetadata().getName().equals(this.namespace.getMetadata().getName())) {
                isNamespaceExists = true;
                if (log.isDebugEnabled()) {
                    log.debug("Namespace found: " + ns.getMetadata().getName());
                }
                break;
            }
        }
        if (!isNamespaceExists) {
            if (log.isDebugEnabled()) {
                log.debug("Namespace not available hence creating namespace: " + namespace.getMetadata().getName());
            }
            kubernetesClient.namespaces().create(namespace);
        }
    }

    @Override
    public void setApplicationContext(ApplicationContext applicationContext) throws RuntimeProvisioningException {

    }

    @Override
    public void createOrganization(TenantInfo tenantInfo) throws RuntimeProvisioningException {
        AutoAdaptableKubernetesClient kubernetesClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        kubernetesClient.namespaces().create(this.namespace);
        kubernetesClient.close();
    }

    @Override
    public void updateOrganization(TenantInfo tenantInfo) throws RuntimeProvisioningException {

    }

    @Override
    public void deleteOrganization(TenantInfo tenantInfo) throws RuntimeProvisioningException {
        AutoAdaptableKubernetesClient kubernetesClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        kubernetesClient.namespaces().delete(this.namespace);
        kubernetesClient.close();
    }

    @Override
    public void archiveOrganization(TenantInfo tenantInfo) throws RuntimeProvisioningException {

    }

    /**
     * Create Kubernetes Deployment and set of services according to the deployment configuration.
     * @param config deployment configuration
     * @return list of created service names
     * @throws RuntimeProvisioningException
     */
    @Override
    public List<String> deployApplication(DeploymentConfig config) throws RuntimeProvisioningException {

        AutoAdaptableKubernetesClient kubClient = null;
        List<Container> containers = config.getContainers();
        ArrayList<io.fabric8.kubernetes.api.model.Container> kubContainerList = new ArrayList<>();
        List<String> serviceNameList = new ArrayList<>();
        String cpuLimitInt = resourceQuotaLimit.getCpuLimit();
        String cpuLimit = cpuLimitInt.concat("m");
        String cpuRequest = resourceQuotaLimit.getCpuRequest().concat("m");
        String memoryLimitInt = resourceQuotaLimit.getMemoryLimit();
        String memoryLimit = memoryLimitInt.concat("Mi");
        String memoryRequest = resourceQuotaLimit.getMemoryRequest().concat("Mi");

        try {
            //Deployment creation
            for (Container container : containers) {
                io.fabric8.kubernetes.api.model.Container kubContainer = new io.fabric8.kubernetes.api.model.Container();
                kubContainer.setName(container.getContainerName());
                kubContainer.setImage(container.getBaseImageName() + ":" + container.getBaseImageVersion());
                kubContainer.setImagePullPolicy(KubernetesPovisioningConstants.IMAGE_PULL_POLICY_ALWAYS);

                ResourceRequirementsBuilder resourceRequirementsBuilder = new ResourceRequirementsBuilder();
                ResourceRequirements resourceRequirement = resourceRequirementsBuilder
                        .addToLimits("cpu", new Quantity(cpuLimit)).addToRequests("cpu", new Quantity(cpuRequest))
                        .addToLimits("memory", new Quantity(memoryLimit))
                        .addToRequests("memory", new Quantity(memoryRequest)).build();
                kubContainer.setResources(resourceRequirement);

                //Checking whether the container is including volume mounts
                if (container.getVolumeMounts() != null) {
                    kubContainer.setVolumeMounts(container.getVolumeMounts());
                }

                List<ContainerPort> containerPorts = new ArrayList<>();
                List<ServiceProxy> serviceProxies = container.getServiceProxies();
                if (serviceProxies != null && serviceProxies.size() > 0) {
                    for (ServiceProxy serviceProxy : serviceProxies) {
                        ContainerPort kubContainerPort = new ContainerPortBuilder()
                                .withContainerPort(serviceProxy.getServiceBackendPort()).build();
                        containerPorts.add(kubContainerPort);
                    }
                }
                kubContainer.setPorts(containerPorts);
                if (container.getEnvVariables() != null) {
                    List<EnvVar> envVarList = new ArrayList<>();
                    for (Map.Entry envVarEntry : container.getEnvVariables().entrySet()) {
                        EnvVar envVar = new EnvVarBuilder().withName((String) envVarEntry.getKey())
                                .withValue((String) envVarEntry.getValue()).build();
                        envVarList.add(envVar);
                    }
                    kubContainer.setEnv(envVarList);
                }
                kubContainerList.add(kubContainer);
            }

            PodSpec podSpec = new PodSpecBuilder().withContainers(kubContainerList).withVolumes(config.getSecrets())
                    .build();

            PodTemplateSpec podTemplateSpec = new PodTemplateSpecBuilder()
                    .withMetadata(new ObjectMetaBuilder()
                            .withLabels(KubernetesProvisioningUtils.getLableMap(applicationContext)).build())
                    .withSpec(podSpec).build();

            DeploymentSpec deploymentSpec = new DeploymentSpecBuilder().withReplicas(config.getReplicas())
                    .withTemplate(podTemplateSpec).build();

            Deployment deployment = new DeploymentBuilder().withKind(KubernetesPovisioningConstants.KIND_DEPLOYMENT)
                    .withMetadata(
                            new ObjectMetaBuilder().withName(config.getDeploymentName().toLowerCase()).build())
                    .withSpec(deploymentSpec).build();

            //Service creation
            List<Service> serviceList = new ArrayList<>();
            for (Container container : containers) {
                List<ServiceProxy> serviceProxies = container.getServiceProxies();
                for (ServiceProxy serviceProxy : serviceProxies) {
                    Service service = getService(serviceProxy);
                    serviceList.add(service);
                }
            }

            kubClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();

            //create a new deployement with servies
            kubClient.inNamespace(namespace.getMetadata().getName()).extensions().deployments().create(deployment);
            for (Service service : serviceList) {
                kubClient.inNamespace(namespace.getMetadata().getName()).services().create(service);
                serviceNameList.add(service.getMetadata().getName());
            }

        } catch (KubernetesClientException e) {
            String msg = "Error while creating Deployment : " + config.getDeploymentName();
            log.error(msg, e);
            throw new RuntimeProvisioningException(msg, e);
        } finally {
            if (kubClient != null) {
                kubClient.close();
            }
        }
        return serviceNameList;
    }

    /**
     * Get service with service port and service specification.
     * @param serviceProxy service information
     * @return K8s service
     */
    private Service getService(ServiceProxy serviceProxy) {
        Map<String, String> annotationMap = new HashMap<String, String>();
        annotationMap.put(KubernetesPovisioningConstants.ANNOTATION_KEY_HOST, serviceProxy.getAppHostURL());
        //Check whether service is https and enable ssl term
        if (serviceProxy.getServicePort() == KubernetesPovisioningConstants.HTTPS_SERVICE_PORT) {
            annotationMap.put(KubernetesPovisioningConstants.ANNOTATION_KEY_SSL_TERM,
                    KubernetesPovisioningConstants.ANNOTATION_VALUE_SSL_TERM);
        }
        ServicePort servicePorts = new ServicePortBuilder().withName(serviceProxy.getServiceName())
                .withProtocol(serviceProxy.getServiceProtocol()).withPort(serviceProxy.getServicePort())
                .withTargetPort(new IntOrString(serviceProxy.getServiceBackendPort())).build();
        ServiceSpec serviceSpec = new ServiceSpecBuilder()
                .withSelector(KubernetesProvisioningUtils.getLableMap(applicationContext)).withPorts(servicePorts)
                .withSessionAffinity(KubernetesPovisioningConstants.SERVICE_SESSION_AFFINITY_MODE).build();

        //Add tenantDomain to label map for the service
        Map<String, String> labelMap = KubernetesProvisioningUtils.getLableMap(applicationContext);
        labelMap.put("tenantDomain", applicationContext.getTenantInfo().getTenantDomain());
        labelMap.put("exposure-level", applicationContext.getExposureLevel());

        //Deployment Unique service name is built using deployment name and the service name.
        String serviceName = serviceProxy.getServiceName();
        return new ServiceBuilder().withKind(KubernetesPovisioningConstants.KIND_SERVICE).withSpec(serviceSpec)
                .withMetadata(new ObjectMetaBuilder().withName(serviceName.toLowerCase()).withLabels(labelMap)
                        .withAnnotations(annotationMap).build())
                .build();
    }

    @Override
    public boolean getDeploymentStatus(DeploymentConfig config) throws RuntimeProvisioningException {

        AutoAdaptableKubernetesClient kubClient = null;
        DeploymentStatus deploymentStatus = kubClient.inNamespace(namespace.getMetadata().getName()).extensions()
                .deployments().withName(config.getDeploymentName()).get().getStatus();
        //Assuming AF does not do zero replica deployments
        if (deploymentStatus.getReplicas() > 0) {
            return true;
        }
        return false;
    }

    @Override
    public DeploymentLogStream streamRuntimeLogs() throws RuntimeProvisioningException {

        DeploymentLogStream deploymentLogStream = new DeploymentLogStream();
        Map<String, BufferedReader> logOutPut = new HashMap<>();
        AutoAdaptableKubernetesClient kubernetesClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        PodList podList = KubernetesProvisioningUtils.getPods(applicationContext);
        if (podList != null) {
            try {
                int podCounter = 1;
                Map<String, LogWatch> watches = new HashMap<>();
                for (Pod pod : podList.getItems()) {
                    for (io.fabric8.kubernetes.api.model.Container container : KubernetesHelper
                            .getContainers(pod)) {
                        String logWatchKey = container.getName();
                        if (log.isDebugEnabled()) {
                            log.debug("Streaming logs in pod : " + pod.getMetadata().getName() + "-"
                                    + container.getName());
                        }
                        LogWatch logs = kubernetesClient.pods().inNamespace(namespace.getMetadata().getName())
                                .withName(pod.getMetadata().getName()).inContainer(container.getName()).watchLog();

                        //logStream should close by after the streaming done in front end
                        //you can use closeLogStream() method in DeploymentStreamLogs
                        BufferedReader logStream = new BufferedReader(new InputStreamReader(logs.getOutput()));
                        logOutPut.put("Replica-" + podCounter + "-" + container.getName(), logStream);
                        deploymentLogStream.setDeploymentLogs(logOutPut);
                        watches.put(logWatchKey, logs);
                    }
                    podCounter++;
                }
                deploymentLogStream.setWatches(watches);
            } catch (KubernetesClientException e) {
                log.error("Error while streaming runtime logs for application : " + applicationContext.getId()
                        + " tenant domain : " + applicationContext.getTenantInfo().getTenantDomain(), e);
                throw new RuntimeProvisioningException(
                        "Error while streaming runtime logs for application : " + applicationContext.getId()
                                + " tenant domain : " + applicationContext.getTenantInfo().getTenantDomain(),
                        e);
            }
        } else {
            log.error("Pod list returned as null for application : " + applicationContext.getId()
                    + " tenant domain : " + applicationContext.getTenantInfo().getTenantDomain());
            throw new RuntimeProvisioningException(
                    "Pod list returned as null for application : " + applicationContext.getId()
                            + " tenant domain : " + applicationContext.getTenantInfo().getTenantDomain());
        }
        return deploymentLogStream;
    }

    @Override
    public DeploymentLogs getRuntimeLogs(LogQuery query) throws RuntimeProvisioningException {

        AutoAdaptableKubernetesClient kubernetesClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        DeploymentLogs deploymentLogs = new DeploymentLogs();
        Map<String, String> logOutPut = new HashMap<>();
        PrettyLoggable prettyLoggable;
        PrettyLoggable prettyLoggablePrev = null;
        PodList podList = KubernetesProvisioningUtils.getPods(applicationContext);
        if (podList != null) {
            try {
                int podCounter = 1;
                for (Pod pod : podList.getItems()) {
                    for (io.fabric8.kubernetes.api.model.Container container : KubernetesHelper
                            .getContainers(pod)) {
                        //Get logs from last pod if restart count > 0
                        if (pod.getStatus().getContainerStatuses().size() > 0
                                && pod.getStatus().getContainerStatuses().get(0).getRestartCount() > 0) {
                            prettyLoggablePrev = kubernetesClient.pods()
                                    .inNamespace(namespace.getMetadata().getName())
                                    .withName(pod.getMetadata().getName()).terminated();
                        }
                        //Get logs from current pod
                        if (query == null || (query.getDurationInHours() < 0 && query.getTailingLines() < 0)) {
                            prettyLoggable = kubernetesClient.pods().inNamespace(namespace.getMetadata().getName())
                                    .withName(pod.getMetadata().getName()).inContainer(container.getName());
                        } else if (query.getDurationInHours() < 0 && query.getTailingLines() > 0) {
                            prettyLoggable = kubernetesClient.pods().inNamespace(namespace.getMetadata().getName())
                                    .withName(pod.getMetadata().getName()).inContainer(container.getName())
                                    .tailingLines(query.getTailingLines());
                        } else if (query.getDurationInHours() > 0 && query.getTailingLines() < 0) {
                            prettyLoggable = kubernetesClient.pods().inNamespace(namespace.getMetadata().getName())
                                    .withName(pod.getMetadata().getName()).inContainer(container.getName())
                                    .sinceSeconds(query.getDurationInHours() * 3600);
                        } else if (query.getDurationInHours() > 0 && query.getTailingLines() > 0) {
                            prettyLoggable = kubernetesClient.pods().inNamespace(namespace.getMetadata().getName())
                                    .withName(pod.getMetadata().getName()).inContainer(container.getName())
                                    .sinceSeconds(query.getDurationInHours() * 3600)
                                    .tailingLines(query.getTailingLines());
                        } else {
                            log.error("Error in log query while getting snapshot logs of application : "
                                    + applicationContext.getId() + " tenant domain : "
                                    + applicationContext.getTenantInfo().getTenantDomain());
                            throw new RuntimeProvisioningException(
                                    "Error in log query while getting snapshot logs of application : "
                                            + applicationContext.getId() + " tenant domain : "
                                            + applicationContext.getTenantInfo().getTenantDomain());
                        }
                        if (log.isDebugEnabled()) {
                            log.debug("Retrieving logs in pod : " + pod.getMetadata().getName() + "-"
                                    + container.getName());
                        }
                        String logs = "";
                        if (prettyLoggablePrev != null) {
                            logs = (String) prettyLoggablePrev.getLog(true);
                        }
                        logs += (String) prettyLoggable.getLog(true);
                        logOutPut.put("Replica_" + podCounter + "_" + pod.getMetadata().getName(), logs);
                        deploymentLogs.setDeploymentLogs(logOutPut);
                    }
                    podCounter++;
                }
            } catch (KubernetesClientException e) {
                log.error("Error while getting snapshot logs for application : " + applicationContext.getId()
                        + " tenant domain : " + applicationContext.getTenantInfo().getTenantDomain(), e);
                throw new RuntimeProvisioningException(
                        "Error while getting snapshot logs for application : " + applicationContext.getId()
                                + " tenant domain : " + applicationContext.getTenantInfo().getTenantDomain(),
                        e);
            }
            return deploymentLogs;
        } else {
            log.error("Pod list returned as null for application : " + applicationContext.getId()
                    + " tenant domain : " + applicationContext.getTenantInfo().getTenantDomain());
            throw new RuntimeProvisioningException(
                    "Pod list returned as null for application : " + applicationContext.getId()
                            + " tenant domain : " + applicationContext.getTenantInfo().getTenantDomain());
        }
    }

    /**
     * Set runtime properties to kubernetes environment.
     *
     * @param runtimeProperties runtime properties
     * @param deploymentConfig  includes deployment related details
     * @throws RuntimeProvisioningException
     */
    @Override
    public void setRuntimeProperties(List<RuntimeProperty> runtimeProperties, DeploymentConfig deploymentConfig)
            throws RuntimeProvisioningException {

        runtimeProperties.addAll(getRuntimeProperties());

        //list of secretes
        List secrets = new ArrayList();

        //list of env variables
        HashMap<String, String> envVariables = new HashMap<>();

        //create a instance of kubernetes client to invoke service call
        AutoAdaptableKubernetesClient kubernetesClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();

        List<VolumeMount> volumeMounts = new ArrayList<>();

        for (RuntimeProperty runtimeProperty : runtimeProperties) {
            switch (runtimeProperty.getPropertyType()) {
            case SENSITIVE:
                if (log.isDebugEnabled()) {
                    String message = "Creating property type secret for the application:"
                            + applicationContext.getId() + " for the tenant domain:"
                            + applicationContext.getTenantInfo().getTenantDomain();
                    log.debug(message);
                }

                Secret currentSecret = kubernetesClient.secrets().inNamespace(namespace.getMetadata().getName())
                        .withName(runtimeProperty.getName()).get();

                //if secrete exists then replace the same secrete, otherwise create a new secrete
                if (currentSecret != null) {
                    kubernetesClient.secrets().inNamespace(namespace.getMetadata().getName())
                            .withName(runtimeProperty.getName()).replace(currentSecret);
                } else {
                    Secret secret = new SecretBuilder().withKind(KubernetesPovisioningConstants.KIND_SECRETS)
                            .withNewMetadata().withNamespace(namespace.getMetadata().getName())
                            .withLabels(KubernetesProvisioningUtils.getLableMap(applicationContext))
                            .withName(runtimeProperty.getName()).endMetadata()
                            .withData(runtimeProperty.getProperties()).build();

                    kubernetesClient.secrets().create(secret);
                }

                Volume volume = new VolumeBuilder().withName(runtimeProperty.getName()).withNewSecret()
                        .withSecretName(runtimeProperty.getName()).endSecret().build();

                secrets.add(volume);

                //create volume mount for the secretes
                VolumeMount volumeMount = new VolumeMountBuilder().withName(runtimeProperty.getName())
                        .withMountPath(KubernetesPovisioningConstants.VOLUME_MOUNT_PATH + runtimeProperty.getName())
                        .withReadOnly(true).build();

                volumeMounts.add(volumeMount);

                break;
            case ENVIRONMENT:
                if (log.isDebugEnabled()) {
                    String message = "Creating property type environment for the application:"
                            + applicationContext.getId() + " for the tenant domain : "
                            + applicationContext.getTenantInfo().getTenantDomain();
                    log.debug(message);
                }

                envVariables.putAll(runtimeProperty.getProperties());

                break;
            default:
                String message = "Runtime property type : " + runtimeProperty.getPropertyType() + " not supported.";

                throw new IllegalArgumentException(message);

            }
        }

        //Initially assume first container is the application and set volume mounts
        deploymentConfig.getContainers().get(0).setVolumeMounts(volumeMounts);

        //Set secretes to a pod
        deploymentConfig.setSecrets(secrets);

        //Initially assume first container is the application
        deploymentConfig.getContainers().get(0).setEnvVariables(envVariables);

        //Call deploy application to redeploy application with runtime properties
        deployApplication(deploymentConfig);

    }

    /**
     * Update runtime properties already defined in the application.
     *
     * @param runtimeProperties list of runtime properties
     * @param deploymentConfig  includes deployment related details
     * @throws RuntimeProvisioningException
     */
    @Override
    public void updateRuntimeProperties(List<RuntimeProperty> runtimeProperties, DeploymentConfig deploymentConfig)
            throws RuntimeProvisioningException {

        List secrets = new ArrayList();
        Container container = new Container();

        for (RuntimeProperty runtimeProperty : runtimeProperties) {
            switch (runtimeProperty.getPropertyType()) {
            case SENSITIVE:
                if (log.isDebugEnabled()) {
                    String message = "Updating property type secret for the application:"
                            + applicationContext.getId() + " for the tenant domain:"
                            + applicationContext.getTenantInfo().getTenantDomain();

                    log.debug(message);
                }
                Secret secret = new SecretBuilder().withKind(KubernetesPovisioningConstants.KIND_SECRETS)
                        .withNewMetadata().withNamespace(namespace.getMetadata().getNamespace())
                        .withName(runtimeProperty.getName()).endMetadata().withData(runtimeProperty.getProperties())
                        .build();

                AutoAdaptableKubernetesClient kubernetesClient = KubernetesProvisioningUtils
                        .getFabric8KubernetesClient();
                kubernetesClient.secrets().inNamespace(namespace.getMetadata().getName())
                        .withName(runtimeProperty.getName()).replace(secret);

                Volume volume = new VolumeBuilder().withName(KubernetesPovisioningConstants.VOLUME_MOUNT)
                        .withNewSecret().withSecretName(runtimeProperty.getName()).endSecret().build();

                secrets.add(volume);

                break;
            case ENVIRONMENT:
                if (log.isDebugEnabled()) {
                    String message = "Updating property type environment for the application:"
                            + applicationContext.getId() + " for the tenant domain:"
                            + applicationContext.getTenantInfo().getTenantDomain();

                    log.debug(message);
                }

                //updating environment variables for container
                container.setEnvVariables(runtimeProperty.getProperties());

                break;
            default:
                String message = "Runtime property type is not support, property type:"
                        + runtimeProperty.getPropertyType();

                throw new IllegalArgumentException(message);
            }
        }

        List<Container> containers = new ArrayList<>();
        containers.add(container);
        deploymentConfig.setContainers(containers);

        //call application deployment to re deploy the application with update variables
        deployApplication(deploymentConfig);
    }

    /**
     * Provide runtime properties for given application context.
     *
     * @return list of runtime properties
     * @throws RuntimeProvisioningException
     */
    @Override
    public List<RuntimeProperty> getRuntimeProperties() throws RuntimeProvisioningException {

        AutoAdaptableKubernetesClient kubernetesClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        SecretList secretList = kubernetesClient.secrets().inNamespace(namespace.getMetadata().getName())
                .withLabels(KubernetesProvisioningUtils.getLableMap(applicationContext)).list();

        List<RuntimeProperty> runtimeProperties = new ArrayList<>();

        for (Secret secret : secretList.getItems()) {
            RuntimeProperty sensitiveRuntimeProperty = new RuntimeProperty();
            sensitiveRuntimeProperty.setPropertyType(RuntimeProperty.PropertyType.SENSITIVE);
            sensitiveRuntimeProperty.setName(secret.getMetadata().getName());
            sensitiveRuntimeProperty.setProperties(secret.getData());
            runtimeProperties.add(sensitiveRuntimeProperty);

        }

        PodList podList = KubernetesProvisioningUtils.getPods(applicationContext);

        HashMap<String, String> data = new HashMap<>();

        for (Pod pod : podList.getItems()) {
            //get only first container from the container list
            List<EnvVar> envVarList = pod.getSpec().getContainers().get(0).getEnv();

            for (EnvVar envVar : envVarList) {
                data.put(envVar.getName(), envVar.getValue());
            }

            RuntimeProperty environmentVariable = new RuntimeProperty();
            environmentVariable.setPropertyType(RuntimeProperty.PropertyType.ENVIRONMENT);
            environmentVariable.setProperties(data);
            runtimeProperties.add(environmentVariable);
        }

        return runtimeProperties;

    }

    /**
     * add a set of custom domains for an application version.
     * This will create an ingress for each domain and each service.
     *
     * @param domains set of domains
     * @throws RuntimeProvisioningException
     */
    @Override
    public boolean addCustomDomain(Set<String> domains) throws RuntimeProvisioningException {

        AutoAdaptableKubernetesClient kubClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        ServiceList serviceList = KubernetesProvisioningUtils.getServices(applicationContext);
        Ingress createdIng;
        boolean created = false;

        for (String domain : domains) {
            for (Service service : serviceList.getItems()) {
                Ingress ing = new IngressBuilder().withKind(KubernetesPovisioningConstants.KIND_INGRESS)
                        .withNewMetadata().withName(KubernetesProvisioningUtils.createIngressMetaName(domain))
                        .withNamespace(namespace.getMetadata().getName()).endMetadata().withNewSpec().withRules()
                        .addNewRule().withHost(domain).withNewHttp().withPaths().addNewPath().withNewBackend()
                        .withServiceName(service.getMetadata().getName()).withServicePort(new IntOrString(80))
                        .endBackend().endPath().endHttp().endRule().endSpec().build();

                createdIng = kubClient.extensions().ingress().inNamespace(namespace.getMetadata().getName())
                        .create(ing);
                if (createdIng != null && KubernetesProvisioningUtils.createIngressMetaName(domain)
                        .equals(createdIng.getMetadata().getName())) {
                    created = true;
                    log.info("Kubernetes ingress : " + ing + "created for service : "
                            + service.getMetadata().getName());
                } else {
                    created = false;
                    log.error("Error occured while creating Kubernetes ingress : " + ing + "for service : "
                            + service.getMetadata().getName());
                }
            }
        }

        return created;
    }

    /**
     * update a certain domain by replacing the ingresses created for related services with new ingresses.
     *
     * @param oldDomain old domain name to be changed
     * @param newDomain new domain name to be changed to
     * @throws RuntimeProvisioningException
     */
    @Override
    public boolean updateCustomDomain(String oldDomain, String newDomain) throws RuntimeProvisioningException {

        AutoAdaptableKubernetesClient kubClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();

        ServiceList serviceList = KubernetesProvisioningUtils.getServices(applicationContext);
        boolean deleted = false;
        boolean updated = false;
        Ingress createdIng;

        for (Service service : serviceList.getItems()) {

            String oldIngName = KubernetesProvisioningUtils.createIngressMetaName(oldDomain);

            String newIngName = KubernetesProvisioningUtils.createIngressMetaName(newDomain);
            Ingress oldIng = new IngressBuilder().withKind(KubernetesPovisioningConstants.KIND_INGRESS)
                    .withNewMetadata().withName(oldIngName).withNamespace(namespace.getMetadata().getName())
                    .endMetadata().build();

            Ingress newIng = new IngressBuilder().withKind(KubernetesPovisioningConstants.KIND_INGRESS)
                    .withNewMetadata().withName(newIngName).withNamespace(namespace.getMetadata().getName())
                    .endMetadata().withNewSpec().addNewRule().withHost(newDomain).withNewHttp().addNewPath()
                    .withNewBackend().withServiceName(service.getMetadata().getName())
                    .withServicePort(new IntOrString(80)).endBackend().endPath().endHttp().endRule().endSpec()
                    .build();

            deleted = kubClient.extensions().ingress().inNamespace(namespace.getMetadata().getName())
                    .delete(oldIng);
            if (deleted) {
                createdIng = kubClient.extensions().ingress().inNamespace(namespace.getMetadata().getName())
                        .create(newIng);
                if (createdIng != null && newIngName.equals(createdIng.getMetadata().getName())) {
                    updated = true;
                } else {
                    updated = false;
                    log.error("Error occured while creating Kubernetes ingress : " + newIng + "for service : "
                            + service.getMetadata().getName());
                }

            } else {
                log.error("Error occured while deleting Kubernetes ingress : " + oldIng + "for service : "
                        + service.getMetadata().getName());
            }
        }
        return updated;
    }

    /**
     * get a set of custom domains for a particular applicaiton context.
     *
     * @return set of domains
     * @throws RuntimeProvisioningException
     */
    @Override
    public Set<String> getCustomDomains() throws RuntimeProvisioningException {

        AutoAdaptableKubernetesClient kubClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        Set<String> domains = new HashSet<>();

        IngressList ingressList = kubClient.extensions().ingress().inNamespace(namespace.getMetadata().getName())
                .list();
        for (Ingress ingress : ingressList.getItems()) {
            domains.add(ingress.getSpec().getRules().get(0).getHost());
        }
        return domains;
    }

    /**
     * delete a custom domain and delete the ingresses created for related services.
     *
     * @param domain domain name
     * @throws RuntimeProvisioningException
     */
    @Override
    public boolean deleteCustomDomain(String domain) throws RuntimeProvisioningException {
        AutoAdaptableKubernetesClient kubClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();

        ServiceList serviceList = KubernetesProvisioningUtils.getServices(applicationContext);
        boolean deleted = false;

        for (Service service : serviceList.getItems()) {
            String ingName = KubernetesProvisioningUtils.createIngressMetaName(domain);
            Ingress ing = new IngressBuilder().withKind(KubernetesPovisioningConstants.KIND_INGRESS)
                    .withNewMetadata().withName(ingName).withNamespace(namespace.getMetadata().getName())
                    .endMetadata().build();

            deleted = kubClient.extensions().ingress().inNamespace(namespace.getMetadata().getName()).delete(ing);
            if (!deleted) {
                log.error("Error occurred while deleting Kubernetes ingress : " + ingName + "for service : "
                        + service.getMetadata().getName());
            }
        }
        return deleted;
    }

    @Override
    public boolean createDeploymentUrl(String environmentUrl) throws RuntimeProvisioningException {

        AutoAdaptableKubernetesClient kubClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        ServiceList serviceList = KubernetesProvisioningUtils.getServices(applicationContext);
        if (log.isDebugEnabled()) {
            log.debug("Deployment service List size: " + serviceList.getItems().size());
        }
        Ingress createdIng;
        boolean created = false;
        String ingressPathStr = KubernetesProvisioningUtils.getDeploymentPath(applicationContext);
        if (log.isDebugEnabled()) {
            log.debug("Ingress path: " + ingressPathStr);
        }

        HTTPIngressPath ingressPath = new HTTPIngressPath(new IngressBackend(),
                KubernetesPovisioningConstants.DEFAULT_INGRESS_PATH);

        for (Service service : serviceList.getItems()) {
            if (log.isDebugEnabled()) {
                log.debug("Ingress creating for service: " + service.getMetadata().getName());
            }
            String ingressName = environmentUrl + "-" + service.getMetadata().getName();
            Ingress ing = new IngressBuilder().withKind(KubernetesPovisioningConstants.KIND_INGRESS)
                    .withNewMetadata().withName(KubernetesProvisioningUtils.createIngressMetaName(ingressName))
                    .withNamespace(namespace.getMetadata().getName())
                    .withLabels(KubernetesProvisioningUtils.getLableMap(applicationContext)).endMetadata()
                    .withNewSpec().withRules().addNewRule().withHost(environmentUrl).withNewHttp().withPaths()
                    .addNewPathLike(ingressPath).withNewBackend().withServiceName(service.getMetadata().getName())
                    .withServicePort(new IntOrString(service.getSpec().getPorts().get(0).getPort())).endBackend()
                    .endPath().endHttp().endRule().endSpec().build();

            createdIng = kubClient.extensions().ingress().inNamespace(namespace.getMetadata().getName())
                    .create(ing);

            if (createdIng != null && KubernetesProvisioningUtils.createIngressMetaName(ingressName)
                    .equals(createdIng.getMetadata().getName())) {
                created = true;
                if (log.isDebugEnabled()) {
                    log.debug("Kubernetes ingress : " + ing + "created for service : "
                            + service.getMetadata().getName());
                }

            } else {
                created = false;
                log.error("Error occurred while creating Kubernetes ingress : " + ing + "for service : "
                        + service.getMetadata().getName());
            }
        }
        return created;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public boolean deleteDeployment() throws RuntimeProvisioningException {
        try {
            deleteK8sKind(KubernetesPovisioningConstants.KIND_DEPLOYMENT);
            KubernetesProvisioningUtils.waitForDeploymentToGetDeleted(applicationContext);
            deleteK8sKind(KubernetesPovisioningConstants.KIND_REPLICATION_CONTROLLER);
            KubernetesProvisioningUtils.waitForRCToGetDeleted(applicationContext);
            deleteK8sKind(KubernetesPovisioningConstants.KIND_POD);
            KubernetesProvisioningUtils.waitForPodToGetDeleted(applicationContext);
            deleteK8sKind(KubernetesPovisioningConstants.KIND_INGRESS);
            KubernetesProvisioningUtils.waitForIngressesToGetDeleted(applicationContext);
            deleteK8sKind(KubernetesPovisioningConstants.KIND_SECRETS);
            KubernetesProvisioningUtils.waitForSecretToGetDeleted(applicationContext);
            deleteK8sKind(KubernetesPovisioningConstants.KIND_SERVICE);
            KubernetesProvisioningUtils.waitForServiceToGetDeleted(applicationContext);
            return true;
        } catch (RuntimeProvisioningException e) {
            return false;
        }
    }

    /**
     * Delete K8s object for given kind with labels.
     *
     * @param k8sKind k8s object type
     */
    public void deleteK8sKind(String k8sKind) throws RuntimeProvisioningException {
        AutoAdaptableKubernetesClient kubernetesClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        String namespace = this.namespace.getMetadata().getName();
        Map<String, String> labels = KubernetesProvisioningUtils.getDeleteLables(applicationContext);

        try {
            switch (k8sKind) {
            case KubernetesPovisioningConstants.KIND_REPLICATION_CONTROLLER:
                kubernetesClient.replicationControllers().inNamespace(namespace).withLabels(labels).delete();
                break;
            case KubernetesPovisioningConstants.KIND_DEPLOYMENT:
                kubernetesClient.extensions().deployments().inNamespace(namespace).withLabels(labels).delete();
                break;
            case KubernetesPovisioningConstants.KIND_POD:
                kubernetesClient.pods().inNamespace(namespace).withLabels(labels).delete();
                break;
            case KubernetesPovisioningConstants.KIND_INGRESS:
                kubernetesClient.extensions().ingress().inNamespace(namespace).withLabels(labels).delete();
                break;
            case KubernetesPovisioningConstants.KIND_SECRETS:
                kubernetesClient.secrets().inNamespace(namespace).withLabels(labels).delete();
                break;
            case KubernetesPovisioningConstants.KIND_SERVICE:
                kubernetesClient.services().inNamespace(namespace).withLabels(labels).delete();
                break;
            default:
                String message = "The kubernetes kind : " + k8sKind + " deletion is not supported";
                throw new IllegalArgumentException(message);
            }
        } catch (KubernetesClientException e) {
            String message = "Error while deleting kubernetes kind : " + k8sKind + " from deployment";
            log.error(message, e);
            throw new RuntimeProvisioningException(message, e);
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void deleteK8sKindByName(String k8sKind, String name) throws RuntimeProvisioningException {
        AutoAdaptableKubernetesClient kubernetesClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        String namespace = this.namespace.getMetadata().getName();

        if (log.isDebugEnabled()) {
            log.debug("Kubernetes kind : " + k8sKind + ", object name : " + name);
        }

        try {
            switch (k8sKind) {
            case KubernetesPovisioningConstants.KIND_REPLICATION_CONTROLLER:
                kubernetesClient.replicationControllers().inNamespace(namespace).withName(name).delete();
                break;
            case KubernetesPovisioningConstants.KIND_DEPLOYMENT:
                kubernetesClient.extensions().deployments().inNamespace(namespace).withName(name).delete();
                break;
            case KubernetesPovisioningConstants.KIND_POD:
                kubernetesClient.pods().inNamespace(namespace).withName(name).delete();
                break;
            case KubernetesPovisioningConstants.KIND_INGRESS:
                kubernetesClient.extensions().ingress().inNamespace(namespace).withName(name).delete();
                break;
            case KubernetesPovisioningConstants.KIND_SECRETS:
                kubernetesClient.secrets().inNamespace(namespace).withName(name).delete();
                break;
            case KubernetesPovisioningConstants.KIND_SERVICE:
                kubernetesClient.services().inNamespace(namespace).withName(name).delete();
                break;
            default:
                String message = "The kubernetes kind : " + k8sKind + " deletion is not supported";
                throw new IllegalArgumentException(message);
            }
        } catch (KubernetesClientException e) {
            String message = "Error while deleting kubernetes kind : " + k8sKind + " with name : " + name
                    + " from deployment";
            log.error(message, e);
            throw new RuntimeProvisioningException(message, e);
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void createService(ServiceProxy serviceProxy) throws RuntimeProvisioningException {
        Service service = getService(serviceProxy);
        String namespace = this.namespace.getMetadata().getName();

        try {
            AutoAdaptableKubernetesClient kubClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
            kubClient.inNamespace(namespace).services().create(service);
        } catch (KubernetesClientException e) {
            String message = "Error while creating kubernetes kind service with namespace : " + namespace;
            log.error(message, e);
            throw new RuntimeProvisioningException(message, e);
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public Map<String, String> getPodRestartCounts() throws RuntimeProvisioningException {
        Map<String, String> podRestartCounts = new HashMap<>();
        PodList podList = KubernetesProvisioningUtils.getPods(applicationContext);
        if (podList != null) {
            int podCounter = 0;
            for (Pod pod : podList.getItems()) {
                if (pod.getStatus().getContainerStatuses().size() > 0) {
                    podRestartCounts.put(String.valueOf(podCounter),
                            String.valueOf(pod.getStatus().getContainerStatuses().get(0).getRestartCount()));
                } else {
                    //In case query is done before the pod get fully created,
                    //Restart count wont be available so returning 0
                    podRestartCounts.put(String.valueOf(podCounter), String.valueOf(0));
                }
                podCounter++;
            }
            return podRestartCounts;
        } else {
            String message = "Could not find a pod associated with pod for application : "
                    + applicationContext.getId() + ", version : " + applicationContext.getVersion();
            log.error(message);
            throw new RuntimeProvisioningException(message);
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public JSONArray getReplicaInfo() throws RuntimeProvisioningException {
        PodList podList = KubernetesProvisioningUtils.getPods(applicationContext);
        if (podList != null) {
            JSONArray jsonArray = new JSONArray();
            for (Pod pod : podList.getItems()) {
                if (pod.getStatus().getContainerStatuses().size() > 0) {
                    JSONObject jsonObject = new JSONObject();
                    jsonObject.put("podName", pod.getMetadata().getName());
                    jsonObject.put("restartCount", pod.getStatus().getContainerStatuses().get(0).getRestartCount());
                    jsonArray.put(jsonObject);
                } else {
                    //In case query is done before the pod get fully created,
                    //Restart count wont be available so returning 0
                    //Pod name return as empty
                    JSONObject jsonObject = new JSONObject();
                    jsonObject.put("podName", "");
                    jsonObject.put("restartCount", String.valueOf(0));
                    jsonArray.put(jsonObject);
                }
            }
            return jsonArray;
        } else {
            String message = "Could not find a pod associated with pod for application : "
                    + applicationContext.getId() + ", version : " + applicationContext.getVersion();
            log.error(message);
            throw new RuntimeProvisioningException(message);
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void changeExposureLevelInServices(String serviceName, String exposureLevel, String lbHost)
            throws RuntimeProvisioningException {
        AutoAdaptableKubernetesClient kubClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        kubClient.services().inNamespace(this.namespace.getMetadata().getName()).withName(serviceName).edit()
                .editMetadata().addToLabels("exposure-level", exposureLevel).endMetadata().done();

        kubClient.services().inNamespace(this.namespace.getMetadata().getName()).withName(serviceName).edit()
                .editMetadata().addToAnnotations("serviceloadbalancer/lb.host", lbHost).endMetadata().done();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void updateKubernetesServiceWithLabel(String serviceName, String labelKey, String labelValue)
            throws RuntimeProvisioningException {
        String namespace = this.namespace.getMetadata().getName();
        try {
            AutoAdaptableKubernetesClient kubClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
            kubClient.services().inNamespace(namespace).withName(serviceName).edit().editMetadata()
                    .addToLabels(labelKey, labelValue).endMetadata().done();
        } catch (KubernetesClientException e) {
            String message = "Error while adding label to kubernetes kind service with service name: " + serviceName
                    + " in namespace: " + namespace;
            log.error(message, e);
            throw new RuntimeProvisioningException(message, e);
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void createDeploymentAutoScalePolicy(String name) throws RuntimeProvisioningException {
        AutoAdaptableKubernetesClient kubernetesClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        String namespace = this.namespace.getMetadata().getName();

        HorizontalPodAutoscaler horizontalPodAutoscaler = new HorizontalPodAutoscalerBuilder().withNewMetadata()
                .withNamespace(namespace).withName(name).endMetadata().withNewSpec().withMinReplicas(1)
                .withMaxReplicas(5).withNewCpuUtilization(50).withNewScaleRef().withName(name)
                .withKind(KubernetesPovisioningConstants.KIND_DEPLOYMENT).endScaleRef().endSpec().build();

        kubernetesClient.extensions().horizontalPodAutoscalers().inNamespace(namespace)
                .create(horizontalPodAutoscaler);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void scaleDeployment(String name, int replicaCount) throws RuntimeProvisioningException {
        AutoAdaptableKubernetesClient kubernetesClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        String namespace = this.namespace.getMetadata().getName();

        kubernetesClient.extensions().deployments().inNamespace(namespace).withName(name).scale(replicaCount);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public int getReplicasForDeployment(String name) throws RuntimeProvisioningException {
        AutoAdaptableKubernetesClient kubernetesClient = KubernetesProvisioningUtils.getFabric8KubernetesClient();
        String namespace = this.namespace.getMetadata().getName();
        int count = kubernetesClient.extensions().deployments().inNamespace(namespace).withName(name).get()
                .getSpec().getReplicas();
        log.info("Replica count for deployment:" + name + ":" + count);
        return count;
    }
}