io.fabric8.kubernetes.pipeline.devops.ApplyStepExecution.java Source code

Java tutorial

Introduction

Here is the source code for io.fabric8.kubernetes.pipeline.devops.ApplyStepExecution.java

Source

/*
 * Copyright (C) 2015 Original Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.fabric8.kubernetes.pipeline.devops;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import hudson.AbortException;
import hudson.EnvVars;
import hudson.FilePath;
import hudson.model.TaskListener;
import io.fabric8.devops.ProjectConfig;
import io.fabric8.devops.ProjectConfigs;
import io.fabric8.devops.ProjectRepositories;
import io.fabric8.kubernetes.api.Controller;
import io.fabric8.kubernetes.api.KubernetesHelper;
import io.fabric8.kubernetes.api.ServiceNames;
import io.fabric8.kubernetes.api.model.*;
import io.fabric8.kubernetes.api.model.extensions.Deployment;
import io.fabric8.kubernetes.api.model.extensions.ReplicaSet;
import io.fabric8.kubernetes.client.DefaultKubernetesClient;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.internal.HasMetadataComparator;
import io.fabric8.kubernetes.pipeline.devops.elasticsearch.DeploymentEventDTO;
import io.fabric8.kubernetes.pipeline.devops.elasticsearch.ElasticsearchClient;
import io.fabric8.kubernetes.pipeline.devops.git.GitInfoCallback;
import io.fabric8.openshift.api.model.*;
import io.fabric8.openshift.client.OpenShiftClient;
import io.fabric8.utils.Strings;
import io.fabric8.utils.Systems;
import io.fabric8.utils.URLUtils;
import io.fabric8.workflow.core.Constants;
import io.fabric8.kubernetes.pipeline.devops.elasticsearch.JsonUtils;
import io.fabric8.kubernetes.pipeline.devops.git.GitConfig;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.jenkinsci.plugins.gitclient.Git;
import org.jenkinsci.plugins.gitclient.GitClient;
import org.jenkinsci.plugins.workflow.steps.*;

import static io.fabric8.utils.PropertiesHelper.toMap;

import javax.inject.Inject;
import java.io.*;
import java.net.URL;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class ApplyStepExecution extends AbstractSynchronousStepExecution<String> {

    @Inject
    private transient ApplyStep step;

    @StepContextParameter
    private transient TaskListener listener;

    @StepContextParameter
    private transient FilePath workspace;

    @StepContextParameter
    transient EnvVars env;

    private KubernetesClient kubernetes;

    @Override
    public String run() throws Exception {

        String environment = step.getEnvironment();
        String environmentName = step.getEnvironmentName();
        if (StringUtils.isBlank(environmentName)) {
            environmentName = createDefaultEnvironmentName(environment);
        }

        if (StringUtils.isBlank(environment)) {
            throw new AbortException("Supply target environment");
        }

        try (KubernetesClient kubernetes = getKubernetes()) {
            Controller controller = new Controller(kubernetes);
            controller.setThrowExceptionOnError(true);
            controller.setRecreateMode(false);
            controller.setAllowCreate(step.getCreateNewResources());
            controller.setServicesOnlyMode(step.getServicesOnly());
            controller.setIgnoreServiceMode(step.getIgnoreServices());
            controller.setIgnoreRunningOAuthClients(step.getIgnoreRunningOAuthClients());
            controller.setProcessTemplatesLocally(step.getProcessTemplatesLocally());
            controller
                    .setDeletePodsOnReplicationControllerUpdate(step.getDeletePodsOnReplicationControllerUpdate());
            controller.setRollingUpgrade(step.getRollingUpgrades());
            controller.setRollingUpgradePreserveScale(step.getRollingUpgradePreserveScale());

            String resources = getResources();

            Object dto;
            if (resources.startsWith("{")) {
                dto = KubernetesHelper.loadJson(resources);
            } else {
                dto = KubernetesHelper.loadYaml(resources.getBytes(), KubernetesResource.class);
            }

            if (dto == null) {
                throw new AbortException("Cannot load kubernetes json: " + resources);
            }
            // lets check we have created the namespace
            controller.applyNamespace(environment);
            controller.setNamespace(environment);

            // TODO do we need this?
            String fileName = "pipeline.json";

            Set<KubernetesList> kubeConfigs = new LinkedHashSet<>();

            Set<HasMetadata> entities = new TreeSet<>(new HasMetadataComparator());
            for (KubernetesList c : kubeConfigs) {
                entities.addAll(c.getItems());
            }

            entities.addAll(KubernetesHelper.toItemList(dto));

            addEnvironmentAnnotations(entities);

            String registry = getRegistry();
            if (Strings.isNotBlank(registry)) {
                listener.getLogger().println("Adapting resources to use pull images from registry: " + registry);
                addRegistryToImageNameIfNotPresent(entities, registry);
            }

            listener.getLogger().println("About to apply resource" + entities);
            //Apply all items
            for (HasMetadata entity : entities) {
                if (entity instanceof Pod) {
                    Pod pod = (Pod) entity;
                    controller.applyPod(pod, fileName);

                    String event = getDeploymentEventJson(entity.getKind(), environment, environmentName);
                    ElasticsearchClient.createEvent(event, ElasticsearchClient.DEPLOYMENT, listener);

                } else if (entity instanceof Service) {
                    Service service = (Service) entity;
                    controller.applyService(service, fileName);
                } else if (entity instanceof ReplicationController) {
                    ReplicationController replicationController = (ReplicationController) entity;
                    controller.applyReplicationController(replicationController, fileName);

                    String event = getDeploymentEventJson(entity.getKind(), environment, environmentName);
                    ElasticsearchClient.createEvent(event, ElasticsearchClient.DEPLOYMENT, listener);

                } else if (entity instanceof ReplicaSet) {
                    controller.apply(entity, fileName);

                    String event = getDeploymentEventJson(entity.getKind(), environment, environmentName);
                    ElasticsearchClient.createEvent(event, ElasticsearchClient.DEPLOYMENT, listener);

                } else if (entity instanceof Deployment) {
                    controller.apply(entity, fileName);

                    String event = getDeploymentEventJson(entity.getKind(), environment, environmentName);
                    ElasticsearchClient.createEvent(event, ElasticsearchClient.DEPLOYMENT, listener);

                } else if (entity instanceof DeploymentConfig) {
                    controller.apply(entity, fileName);

                    String event = getDeploymentEventJson(entity.getKind(), environment, environmentName);
                    ElasticsearchClient.createEvent(event, ElasticsearchClient.DEPLOYMENT, listener);

                } else if (entity != null) {
                    controller.apply(entity, fileName);
                }
            }
        } catch (Exception e) {
            String stacktrace = ExceptionUtils.getStackTrace(e);
            throw new AbortException("Error during kubernetes apply: " + stacktrace);
        }
        return "OK";
    }

    private String getResources() throws AbortException {
        if (Strings.isNotBlank(step.getFile())) {
            return step.getFile();
        } else {
            if (kubernetes.isAdaptable(OpenShiftClient.class)) {
                try {
                    return readFile("target/classes/META-INF/fabric8/openshift.yml");
                } catch (AbortException e) {
                    // lets not worry yet if we can't find the yaml as we may be using the old f-m-p
                    listener.getLogger().println("no openshift.yml found");
                }

            } else {
                try {
                    return readFile("target/classes/META-INF/fabric8/kubernetes.yml");
                } catch (AbortException e) {
                    // lets not worry yet if we can't find the yaml as we may be using the old f-m-p
                    listener.getLogger().println("no kubernetes.yml found");
                }
            }

            try {
                return readFile("target/classes/kubernetes.json");
            } catch (AbortException e) {
                throw new AbortException(
                        "No resources passed into the step using the 'file' argument or found in the default target folder generated by the fabric8-maven-plugin");
            }
        }
    }

    private String createDefaultEnvironmentName(String environment) {
        if (StringUtils.isBlank(environment)) {
            return environment;
        }
        String[] split = environment.split("-");
        String answer = environment;
        if (split != null && split.length > 0) {
            answer = split[split.length - 1];
        }
        return StringUtils.capitalize(answer);
    }

    private String getRegistry() {
        if (Strings.isNullOrBlank(step.getRegistry())) {
            if (Strings.isNotBlank(env.get(Constants.FABRIC8_DOCKER_REGISTRY_SERVICE_HOST))) {
                return env.get(Constants.FABRIC8_DOCKER_REGISTRY_SERVICE_HOST) + ":"
                        + env.get(Constants.FABRIC8_DOCKER_REGISTRY_SERVICE_PORT);
            }
            return null;
        } else {
            return step.getRegistry();
        }
    }

    /**
     * Should we try to create a route for the given service?
     * <p>
     * By default lets ignore the kubernetes services and any service which does not expose ports 80 and 443
     *
     * @return true if we should create an OpenShift Route for this service.
     */
    protected static boolean shouldCreateRouteForService(Service service, String id, TaskListener listener) {
        if ("kubernetes".equals(id) || "kubernetes-ro".equals(id)) {
            return false;
        }
        Set<Integer> ports = KubernetesHelper.getPorts(service);
        if (ports.size() == 1) {
            return true;
        } else {
            listener.getLogger().println("Not generating route for service " + id
                    + " as only single port services are supported. Has ports: " + ports);
            return false;
        }
    }

    private Object applyTemplates(Template template, KubernetesClient kubernetes, Controller controller,
            String fileName, String namespace) throws Exception {
        KubernetesHelper.setNamespace(template, namespace);
        // TODO do we need to override template values during a CD pipeline?
        //overrideTemplateParameters(template);
        return controller.applyTemplate(template, fileName);
    }

    private void addEnvironmentAnnotations(Iterable<HasMetadata> items) throws Exception {
        if (items != null) {
            for (HasMetadata item : items) {
                if (item instanceof KubernetesList) {
                    KubernetesList list = (KubernetesList) item;
                    addEnvironmentAnnotations(list.getItems());
                } else if (item instanceof Template) {
                    Template template = (Template) item;
                    addEnvironmentAnnotations(template.getObjects());
                } else if (item instanceof ReplicationController) {
                    addEnvironmentAnnotations(item);
                } else if (item instanceof DeploymentConfig) {
                    addEnvironmentAnnotations(item);
                }
            }
        }
    }

    public void addRegistryToImageNameIfNotPresent(Iterable<HasMetadata> items, String registry) throws Exception {
        if (items != null) {
            for (HasMetadata item : items) {
                if (item instanceof KubernetesList) {
                    KubernetesList list = (KubernetesList) item;
                    addRegistryToImageNameIfNotPresent(list.getItems(), registry);
                } else if (item instanceof Template) {
                    Template template = (Template) item;
                    addRegistryToImageNameIfNotPresent(template.getObjects(), registry);
                } else if (item instanceof Pod) {
                    List<Container> containers = ((Pod) item).getSpec().getContainers();
                    prefixRegistryIfNotPresent(containers, registry);

                } else if (item instanceof ReplicationController) {
                    List<Container> containers = ((ReplicationController) item).getSpec().getTemplate().getSpec()
                            .getContainers();
                    prefixRegistryIfNotPresent(containers, registry);

                } else if (item instanceof ReplicaSet) {
                    List<Container> containers = ((ReplicaSet) item).getSpec().getTemplate().getSpec()
                            .getContainers();
                    prefixRegistryIfNotPresent(containers, registry);

                } else if (item instanceof DeploymentConfig) {
                    List<Container> containers = ((DeploymentConfig) item).getSpec().getTemplate().getSpec()
                            .getContainers();
                    prefixRegistryIfNotPresent(containers, registry);

                } else if (item instanceof Deployment) {
                    List<Container> containers = ((Deployment) item).getSpec().getTemplate().getSpec()
                            .getContainers();
                    prefixRegistryIfNotPresent(containers, registry);
                }

            }
        }
    }

    private void prefixRegistryIfNotPresent(List<Container> containers, String registry) {
        for (Container container : containers) {
            if (!hasRegistry(container.getImage())) {
                container.setImage(registry + "/" + container.getImage());
            }
        }
    }

    /**
     * Checks to see if there's a registry name already provided in the image name
     *
     * Code influenced from <a href="https://github.com/rhuss/docker-maven-plugin/blob/master/src/main/java/org/jolokia/docker/maven/util/ImageName.java">docker-maven-plugin</a>
     * @param imageName
     * @return true if the image name contains a registry
     */
    public static boolean hasRegistry(String imageName) {
        if (imageName == null) {
            throw new NullPointerException("Image name must not be null");
        }
        Pattern tagPattern = Pattern.compile("^(.+?)(?::([^:/]+))?$");
        Matcher matcher = tagPattern.matcher(imageName);
        if (!matcher.matches()) {
            throw new IllegalArgumentException(imageName + " is not a proper image name ([registry/][repo][:port]");
        }

        String rest = matcher.group(1);
        String[] parts = rest.split("\\s*/\\s*");
        String part = parts[0];

        return part.contains(".") || part.contains(":");
    }

    protected void addEnvironmentAnnotations(HasMetadata resource) throws AbortException {
        Map<String, String> mapEnvVarToAnnotation = new HashMap<>();
        String resourceName = "environmentAnnotations.properties";
        URL url = getClass().getResource(resourceName);
        if (url == null) {
            throw new AbortException("Cannot find resource `" + resourceName + "` on the classpath!");
        }
        addPropertiesFileToMap(url, mapEnvVarToAnnotation);
        //TODO add this in and support for non java projects
        //addPropertiesFileToMap(this.environmentVariableToAnnotationsFile, mapEnvVarToAnnotation);
        addPropertiesFileToMap(new File("./src/main/fabric8/environemntToAnnotations.properties"),
                mapEnvVarToAnnotation);
        Map<String, String> annotations = KubernetesHelper.getOrCreateAnnotations(resource);
        Set<Map.Entry<String, String>> entries = mapEnvVarToAnnotation.entrySet();
        for (Map.Entry<String, String> entry : entries) {
            String envVar = entry.getKey();
            String annotation = entry.getValue();
            if (Strings.isNotBlank(envVar) && Strings.isNotBlank(annotation)) {
                String value = Systems.getEnvVarOrSystemProperty(envVar);
                if (Strings.isNullOrBlank(value)) {
                    value = tryDefaultAnnotationEnvVar(envVar);
                }
                if (Strings.isNotBlank(value)) {
                    String oldValue = annotations.get(annotation);
                    if (Strings.isNotBlank(oldValue)) {
                        listener.getLogger()
                                .println("Not adding annotation `" + annotation + "` to "
                                        + KubernetesHelper.getKind(resource) + " "
                                        + KubernetesHelper.getName(resource) + " with value `" + value
                                        + "` as there is already an annotation value of `" + oldValue + "`");
                    } else {
                        annotations.put(annotation, value);
                    }
                }
            }
        }
    }

    /**
     * Tries to default some environment variables if they are not already defined.
     * <p>
     * This can happen if using Jenkins Workflow which doens't seem to define BUILD_URL or GIT_URL for example
     *
     * @return the value of the environment variable name if it can be found or calculated
     */
    protected String tryDefaultAnnotationEnvVar(String envVarName) throws AbortException {

        ProjectConfig projectConfig = getProjectConfig();
        if (projectConfig != null) {
            GitConfig gitConfig = getGitConfig();
            String repoName = projectConfig.getBuildName();
            String userEnvVar = "JENKINS_GOGS_USER";
            String username = env.get(userEnvVar);

            if (io.fabric8.utils.Objects.equal("BUILD_URL", envVarName)) {
                String jobUrl = projectConfig.getLink("Job");
                if (Strings.isNullOrBlank(jobUrl)) {
                    String name = projectConfig.getBuildName();
                    if (Strings.isNullOrBlank(name)) {
                        // lets try deduce the jenkins build name we'll generate
                        if (Strings.isNotBlank(repoName)) {
                            name = repoName;
                            if (Strings.isNotBlank(username)) {
                                name = ProjectRepositories.createBuildName(username, repoName);
                            } else {
                                listener.getLogger().println(
                                        "Cannot auto-default BUILD_URL as there is no environment variable `"
                                                + userEnvVar + "` defined so we can't guess the Jenkins build URL");
                            }
                        }
                    }
                    if (Strings.isNotBlank(name)) {
                        String jenkinsUrl = KubernetesHelper.getServiceURLInCurrentNamespace(getKubernetes(),
                                ServiceNames.JENKINS, "http", null, true);
                        jobUrl = URLUtils.pathJoin(jenkinsUrl, "/job", name);

                    }
                }
                if (Strings.isNotBlank(jobUrl)) {
                    String buildId = env.get("BUILD_ID");
                    if (Strings.isNotBlank(buildId)) {
                        jobUrl = URLUtils.pathJoin(jobUrl, buildId);
                    } else {
                        listener.getLogger().println(
                                "Cannot find BUILD_ID to create a specific jenkins build URL. So using: " + jobUrl);
                    }
                }
                return jobUrl;
            } else if (io.fabric8.utils.Objects.equal("GIT_URL", envVarName)) {
                String gitUrl = projectConfig.getLinks().get("Git");
                if (Strings.isNullOrBlank(gitUrl)) {
                    listener.getLogger().println("No Job link found in fabric8.yml so we cannot set the GIT_URL");
                } else {
                    if (gitUrl.endsWith(".git")) {
                        gitUrl = gitUrl.substring(0, gitUrl.length() - 4);
                    }
                    String gitCommitId = gitConfig.getCommit();
                    if (Strings.isNotBlank(gitCommitId)) {
                        gitUrl = URLUtils.pathJoin(gitUrl, "commit", gitCommitId);
                    }
                    return gitUrl;
                }

            } else if (io.fabric8.utils.Objects.equal("GIT_COMMIT", envVarName)) {
                String gitCommit = gitConfig.getCommit();
                if (Strings.isNullOrBlank(gitCommit)) {
                    listener.getLogger().println("No git commit found in git.yml so we cannot set the GIT_COMMIT");
                }
                return gitCommit;

            } else if (io.fabric8.utils.Objects.equal("GIT_BRANCH", envVarName)) {
                String gitBranch = gitConfig.getBranch();
                if (Strings.isNullOrBlank(gitBranch)) {
                    listener.getLogger().println("No git branch found in git.yml so we cannot set the GIT_BRANCH");
                }
                return gitBranch;

            }
        } else {
            listener.getLogger().println("No fabric8.yml so unable to add environment pod annotations");
        }
        return null;
    }

    protected static void addPropertiesFileToMap(File file, Map<String, String> answer) throws AbortException {
        if (file != null && file.isFile() && file.exists()) {
            try (FileInputStream in = new FileInputStream(file)) {
                Properties properties = new Properties();
                properties.load(in);
                Map<String, String> map = toMap(properties);
                answer.putAll(map);
            } catch (IOException e) {
                throw new AbortException("Failed to load properties file: " + file + ". " + e);
            }
        }
    }

    protected static void addPropertiesFileToMap(URL url, Map<String, String> answer) throws AbortException {
        if (url != null) {
            try (InputStream in = url.openStream()) {
                Properties properties = new Properties();
                properties.load(in);
                Map<String, String> map = toMap(properties);
                answer.putAll(map);
            } catch (IOException e) {
                throw new AbortException("Failed to load properties URL: " + url + ". " + e);
            }
        }
    }

    public GitConfig getGitConfig() throws AbortException {
        try {
            GitClient client = Git.with(listener, env).in(workspace).getClient();
            return client.withRepository(new GitInfoCallback(listener));
        } catch (Exception e) {
            throw new AbortException("Error getting git config " + e);
        }
    }

    public ProjectConfig getProjectConfig() throws AbortException {
        try {
            return ProjectConfigs.parseProjectConfig(readFile("fabric8.yml"));
        } catch (Exception e) {
            // its fine if no fabric8.yml file is found
            return null;
        }
    }

    private String readFile(String fileName) throws AbortException {
        try {
            InputStream is = workspace.child(fileName).read();
            try {
                return IOUtils.toString(is, Charsets.UTF_8);
            } finally {
                is.close();
            }
        } catch (Exception e) {
            throw new AbortException("Unable to read file " + fileName + ". " + e);
        }
    }

    public String getDeploymentEventJson(String resource, String environment, String environmentName)
            throws IOException, InterruptedException {
        DeploymentEventDTO event = new DeploymentEventDTO();

        GitConfig config = getGitConfig();

        event.setAuthor(config.getAuthor());
        event.setCommit(config.getCommit());
        event.setNamespace(environment);
        event.setEnvironment(environmentName);
        event.setApp(env.get("JOB_NAME"));
        event.setResource(resource);
        event.setVersion(env.get("VERSION"));

        ObjectMapper mapper = JsonUtils.createObjectMapper();
        return mapper.writeValueAsString(event);
    }

    public KubernetesClient getKubernetes() {
        if (kubernetes == null) {
            kubernetes = new DefaultKubernetesClient();
        }
        return kubernetes;
    }
}