org.ballerinalang.containers.docker.impl.DefaultBallerinaDockerClient.java Source code

Java tutorial

Introduction

Here is the source code for org.ballerinalang.containers.docker.impl.DefaultBallerinaDockerClient.java

Source

/*
 * Copyright (c) 2017, WSO2 Inc. (http://wso2.com) All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package org.ballerinalang.containers.docker.impl;

import io.fabric8.docker.api.model.Image;
import io.fabric8.docker.api.model.ImageDelete;
import io.fabric8.docker.client.Config;
import io.fabric8.docker.client.ConfigBuilder;
import io.fabric8.docker.client.DockerClient;
import io.fabric8.docker.client.DockerClientException;
import io.fabric8.docker.dsl.EventListener;
import io.fabric8.docker.dsl.OutputHandle;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.ballerinalang.containers.Constants;
import org.ballerinalang.containers.docker.BallerinaDockerClient;
import org.ballerinalang.containers.docker.exception.BallerinaDockerClientException;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.CountDownLatch;

/**
 * Default implementation of the {@link BallerinaDockerClient}.
 */
public final class DefaultBallerinaDockerClient implements BallerinaDockerClient {

    private static final String BALLERINA_VERSION_SYSTEM_PROP = "ballerina.version";
    private static final String TMPDIR_SYSTEM_PROP = "java.io.tmpdir";
    private static final String DOCKER_CLIENT_TMPDIR_SYSTEM_PROP = "tmp.dir";
    private static final String PATH_FILES = "files";
    private static final String PATH_DOCKER_IMAGE_ROOT = "/docker/image/";
    private static final String PATH_DOCKERFILE_NAME = "Dockerfile";
    private static final String PATH_TEMP_DOCKERFILE_CONTEXT_PREFIX = "ballerina-docker-";
    private static final String PATH_BAL_FILE_EXT = ".bal";
    private static final String ENV_SVC_MODE = "SVC_MODE";
    private static final String ENV_FILE_MODE = "FILE_MODE";
    private static final String LOCAL_DOCKER_DAEMON_SOCKET = "unix:///var/run/docker.sock";

    private final CountDownLatch buildDone = new CountDownLatch(1);
    // Cannot depend on buildErrors because Fabric8 seems to be randomly adding "Failed:" errors even
    // when the build completed successfully.
    //    private final List<String> buildErrors = new ArrayList<>();
    private String buildError;

    /**
     * {@inheritDoc}
     */
    @Override
    public String createServiceImage(String packageName, String dockerEnv, List<Path> bPackagePaths,
            String imageName, String imageVersion)
            throws BallerinaDockerClientException, IOException, InterruptedException {

        return createImageFromPackage(packageName, dockerEnv, bPackagePaths, true, imageName, imageVersion);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public String createServiceImage(String serviceName, String dockerEnv, String ballerinaConfig, String imageName,
            String imageVersion) throws InterruptedException, BallerinaDockerClientException, IOException {

        return createImageFromSingleConfig(serviceName, dockerEnv, ballerinaConfig, true, imageName, imageVersion);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public String createMainImage(String packageName, String dockerEnv, List<Path> bPackagePaths, String imageName,
            String imageVersion) throws BallerinaDockerClientException, IOException, InterruptedException {

        return createImageFromPackage(packageName, dockerEnv, bPackagePaths, false, imageName, imageVersion);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public String createMainImage(String packageName, String dockerEnv, Path bPackagePath, String imageName,
            String imageVersion) throws BallerinaDockerClientException, IOException, InterruptedException {

        return createImageFromSingleFile(packageName, dockerEnv, bPackagePath, imageName, imageVersion);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public String createMainImage(String mainPackageName, String dockerEnv, String ballerinaConfig,
            String imageName, String imageVersion)
            throws InterruptedException, BallerinaDockerClientException, IOException {

        return createImageFromSingleConfig(mainPackageName, dockerEnv, ballerinaConfig, false, imageName,
                imageVersion);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public boolean deleteImage(String packageName, String dockerEnv, String imageName, String imageVersion)
            throws BallerinaDockerClientException {

        // TODO: should not be able to delete arbitrary images.
        imageName = getImageName(packageName, imageName, imageVersion);
        List<ImageDelete> imageDeleteList;

        try {
            imageDeleteList = getDockerClient(dockerEnv).image().withName(imageName).delete().force()
                    .andPrune(false);

        } catch (DockerClientException e) {
            if (e.getMessage().contains("No such image")) {
                return false;
            }
            throw e;
        }

        return imageDeleteList.size() != 0;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public String getImage(String imageName, String dockerEnv) {
        DockerClient client = getDockerClient(dockerEnv);
        List<Image> images = client.image().list().filter(imageName).endImages();

        // No images found?
        if (images == null || images.size() < 1) {
            return null;
        }

        for (Image image : images) {
            // Invalid tags found?
            if (image.getRepoTags() == null) {
                continue;
            }

            String currentImageName = image.getRepoTags().get(0);
            if (currentImageName.equals(imageName + ":" + Constants.IMAGE_VERSION_LATEST)
                    || currentImageName.equals(imageName)) {
                return currentImageName;
            }
        }

        return null;
    }

    /*
    Create a Docker image from a given set of Ballerina packages.
     */
    private String createImageFromPackage(String packageName, String dockerEnv, List<Path> bPackagePaths,
            boolean isService, String imageName, String imageVersion)
            throws BallerinaDockerClientException, IOException, InterruptedException {

        if (bPackagePaths == null || bPackagePaths.size() == 0) {
            throw new BallerinaDockerClientException("Invalid Ballerina package(s)");
        }

        for (Path bPackage : bPackagePaths) {
            if (!Files.exists(bPackage)) {
                throw new BallerinaDockerClientException(
                        "Cannot find Ballerina Package file: " + bPackage.toString());
            }

            if (isService && !FilenameUtils.getExtension(bPackage.toString()).equalsIgnoreCase("bsz")) {
                throw new BallerinaDockerClientException(
                        "Invalid Ballerina package archive. " + "Service packages should be of \"bsz\" type.");
            }

            if (!isService && !FilenameUtils.getExtension(bPackage.toString()).equalsIgnoreCase("bmz")) {
                throw new BallerinaDockerClientException(
                        "Invalid Ballerina package archive. " + "Main packages should be of \"bmz\" type.");
            }
        }

        imageName = getImageName(packageName, imageName, imageVersion);

        // 1. Create a tmp docker context
        Path tmpDir = prepTempDockerfileContext();

        // 2. Copy Ballerina packages
        for (Path bPackage : bPackagePaths) {
            Files.copy(bPackage, Paths.get(
                    tmpDir.toString() + File.separator + PATH_FILES + File.separator + bPackage.toFile().getName()),
                    StandardCopyOption.REPLACE_EXISTING);
        }

        // 3. Create a docker image from the temp context
        String timestamp = new SimpleDateFormat("yyyy-MM-dd'T'h:m:ssXX").format(new Date());
        String buildArgs = "{\"" + ENV_SVC_MODE + "\":\"" + String.valueOf(isService) + "\", " + "\"BUILD_DATE\":\""
                + timestamp + "\"}";
        buildImage(dockerEnv, imageName, tmpDir, buildArgs);

        // 4. Cleanup
        cleanupTempDockerfileContext(tmpDir);

        return getImage(imageName, dockerEnv);
    }

    /*
    Create a Docker image from a give Ballerina configuration.
     */
    private String createImageFromSingleConfig(String serviceName, String dockerEnv, String ballerinaConfig,
            boolean isService, String imageName, String imageVersion)
            throws BallerinaDockerClientException, IOException, InterruptedException {

        imageName = getImageName(serviceName, imageName, imageVersion);

        // 1. Create a tmp docker context
        Path tmpDir = prepTempDockerfileContext();

        // 2. Create a .bal file inside context/files
        Path ballerinaFile = Files.createFile(
                Paths.get(tmpDir + File.separator + PATH_FILES + File.separator + serviceName + PATH_BAL_FILE_EXT));

        Files.write(ballerinaFile, ballerinaConfig.getBytes(Charset.defaultCharset()), StandardOpenOption.CREATE,
                StandardOpenOption.TRUNCATE_EXISTING);

        // 3. Create a docker image from the temp context
        String timestamp = new SimpleDateFormat("yyyy-MM-dd'T'h:m:ssXX").format(new Date());
        String buildArgs = "{\"" + ENV_SVC_MODE + "\":\"" + String.valueOf(isService) + "\", " + "\""
                + ENV_FILE_MODE + "\":\"true\", \"BUILD_DATE\":\"" + timestamp + "\"}";
        buildImage(dockerEnv, imageName, tmpDir, buildArgs);

        // 4. Cleanup
        cleanupTempDockerfileContext(tmpDir);

        return getImage(imageName, dockerEnv);
    }

    /**
     * Create a Docker image from a give Ballerina configuration in bal or balx.
     */
    private String createImageFromSingleFile(String serviceName, String dockerEnv, Path ballerinaConfig,
            String imageName, String imageVersion)
            throws BallerinaDockerClientException, IOException, InterruptedException {

        imageName = getImageName(serviceName, imageName, imageVersion);

        if (!Files.exists(ballerinaConfig)) {
            throw new BallerinaDockerClientException("Cannot find Ballerina file: " + ballerinaConfig.toString());
        }

        if (!FilenameUtils.getExtension(ballerinaConfig.toString()).equalsIgnoreCase("bal")
                && !FilenameUtils.getExtension(ballerinaConfig.toString()).equalsIgnoreCase("balx")) {
            throw new BallerinaDockerClientException(
                    "Invalid Ballerina file. " + "Ballerina files should be of \"bal\" | \"balx\" type.");
        }

        // 1. Create a tmp docker context
        Path tmpDir = prepTempDockerfileContext();

        // 2. Copy a .bal or .balx file inside context/files
        Files.copy(ballerinaConfig, Paths.get(tmpDir.toString() + File.separator + PATH_FILES + File.separator
                + ballerinaConfig.toFile().getName()), StandardCopyOption.REPLACE_EXISTING);

        // 3. Create a docker image from the temp context
        String timestamp = new SimpleDateFormat("yyyy-MM-dd'T'h:m:ssXX").format(new Date());
        String buildArgs = "{\"BUILD_DATE\":\"" + timestamp + "\"}";
        buildImage(dockerEnv, imageName, tmpDir, buildArgs);

        // 4. Cleanup
        cleanupTempDockerfileContext(tmpDir);

        return getImage(imageName, dockerEnv);
    }

    /**
     * Generate the image name from the given parameters.
     *
     * @param packageName  The Ballerina Package name.
     * @param imageName    The given image name. This can be null, in which case the package name is used as the
     *                     image name.
     * @param imageVersion The given image version. If the imageName is not null, this should not be null. If imageName
     *                     is null, this value is ignored and "latest" is used as the image version.
     * @return The image name derived from the above information.
     * @throws BallerinaDockerClientException If any of the required parameters are not found.
     */
    private String getImageName(String packageName, String imageName, String imageVersion)
            throws BallerinaDockerClientException {

        if (StringUtils.isEmpty(packageName)) {
            throw new BallerinaDockerClientException("Package name should not be null or empty.");
        }

        if (imageName == null) {
            imageName = packageName.toLowerCase(Locale.getDefault()) + ":" + Constants.IMAGE_VERSION_LATEST;
        } else {
            if (imageVersion == null) {
                throw new BallerinaDockerClientException(
                        "Image version cannot be null when Image name is specified.");
            }

            imageName = imageName.toLowerCase(Locale.getDefault()) + ":" + imageVersion;
        }
        return imageName;
    }

    /**
     * Creates a {@link DockerClient} from the given Docker host URL.
     *
     * @param env The URL of the Docker host. If this is null, a {@link DockerClient} pointed to the local Docker
     *            daemon will be created.
     * @return {@link DockerClient} object.
     */
    private DockerClient getDockerClient(String env) {
        DockerClient client;
        if (env == null) {
            env = LOCAL_DOCKER_DAEMON_SOCKET;
        }

        Config dockerClientConfig = new ConfigBuilder().withDockerUrl(env).build();

        client = new io.fabric8.docker.client.DefaultDockerClient(dockerClientConfig);
        return client;
    }

    /*
    Delete the temporary Dockerfile context used to build the Docker image.
     */
    private void cleanupTempDockerfileContext(Path tmpDir) throws IOException {
        FileUtils.deleteDirectory(tmpDir.toFile());
    }

    /*
    Create a temporary Dockerfile context, by copying the necessary Dockerfile and scripts. This is
    created at the OS temporary directory, or if specified, at java.io.tmpdir.
     */
    private Path prepTempDockerfileContext() throws IOException, BallerinaDockerClientException {
        // TODO: Until the tmp.dir modification is removed from ballerina-launcher
        String tmpDirLocation = System.getProperty(TMPDIR_SYSTEM_PROP);
        String ballerinaVersion = System.getProperty(BALLERINA_VERSION_SYSTEM_PROP);
        if (tmpDirLocation != null) {
            File tmpDirFile = new File(tmpDirLocation);
            if (!tmpDirFile.exists() && !tmpDirFile.mkdirs()) {
                throw new BallerinaDockerClientException("Couldn't create temporary directory: " + tmpDirLocation);
            }
            // Set Fabric8 docker-client temp directory since the default "/tmp" does not work on Windows
            System.setProperty(DOCKER_CLIENT_TMPDIR_SYSTEM_PROP, tmpDirLocation);
        }

        if (ballerinaVersion == null) {
            throw new BallerinaDockerClientException(
                    "[ERROR] System Property '" + BALLERINA_VERSION_SYSTEM_PROP + "' is not defined. ");
        }

        String tempDirName = PATH_TEMP_DOCKERFILE_CONTEXT_PREFIX + String.valueOf(Instant.now().getEpochSecond());
        Path tmpDir = Files.createTempDirectory(tempDirName);
        Files.createDirectory(Paths.get(tmpDir.toString() + File.separator + PATH_FILES));
        InputStream in = getClass().getResourceAsStream(PATH_DOCKER_IMAGE_ROOT + PATH_DOCKERFILE_NAME);

        Files.copy(in, Paths.get(tmpDir.toString() + File.separator + PATH_DOCKERFILE_NAME),
                StandardCopyOption.REPLACE_EXISTING);

        // Replace ${BALLERINA_VERSION} with ballerinaVersion
        Path path = Paths.get(tmpDir.toString() + File.separator + PATH_DOCKERFILE_NAME);
        Charset charset = StandardCharsets.UTF_8;

        String content = new String(Files.readAllBytes(path), charset);
        content = content.replaceAll("BALLERINA_VERSION", ballerinaVersion);
        Files.write(path, content.getBytes(charset));

        return tmpDir;
    }

    /*
    Execute a Docker image build using Fabric8 DSL.
     */
    private void buildImage(String dockerEnv, String imageName, Path tmpDir, String buildArgs)
            throws InterruptedException, IOException {

        DockerClient client = getDockerClient(dockerEnv);
        OutputHandle buildHandle = client.image().build().withRepositoryName(imageName).withNoCache()
                .alwaysRemovingIntermediate().withBuildArgs(buildArgs)
                .usingListener(new DockerBuilderEventListener()).fromFolder(tmpDir.toString());

        buildDone.await();
        buildHandle.close();
        client.close();
    }

    /**
     * An {@link EventListener} implementation to listen to Docker build events.
     */
    private class DockerBuilderEventListener implements EventListener {

        @Override
        public void onSuccess(String successEvent) {
            buildDone.countDown();
        }

        @Override
        public void onError(String errorEvent) {
            buildError = errorEvent;
            buildDone.countDown();
        }

        @Override
        public void onEvent(String ignore) {
            //..
        }
    }

    //TODO: Temporary fix to log build error. Intermittent build errors are thrown even though the
    //TODO: docker image build is successful
    public String getBuildError() {
        return buildError;
    }

    //    private static boolean isFunctionImage(DockerClient client, String serviceName) {
    //        for (String envVar : client.image()
    //                .withName(serviceName.toLowerCase(Locale.getDefault()) + ":latest")
    //                .inspect()
    //                .getConfig()
    //                .getEnv()) {
    //
    //            String[] envVarValue = envVar.split("=");
    //            if (envVarValue[0].equals("SVC_MODE") && envVarValue[1].equals("false")) {
    //                return true;
    //            }
    //        }
    //
    //        return false;
    //    }
    //
    //    @Override
    //    public String runMainContainer(String dockerEnv, String serviceName)
    //            throws InterruptedException, IOException, BallerinaDockerClientException {
    //        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    //        DockerClient client = getDockerClient(dockerEnv);
    //        if (!isFunctionImage(client, serviceName)) {
    //            throw new BallerinaDockerClientException("Invalid image to run: " +
    // serviceName.toLowerCase(Locale.getDefault()) +
    //                    ":latest");
    //        }
    //
    //        ContainerCreateResponse container = client.container().createNew()
    //                .withName(serviceName + "-latest")
    //                .withImage(serviceName.toLowerCase(Locale.getDefault()) + ":latest")
    //                .done();
    //
    //        // TODO: throws EOFException here.
    //        try (
    //                OutputHandle logHandle = client.container().
    //                        withName(container.getId())
    //                        .logs()
    //                        .writingOutput(outputStream)
    //                        .writingError(outputStream)
    //                        .display()
    //        ) {
    //
    //            if (client.container().withName(container.getId()).start()) {
    //////                ("Container started: " + container.getId());
    //                Thread.sleep(10000);
    ////                client.container().withName(container.getId()).stop();
    ////                return IOUtils.toString(logHandle.getOutput(), "UTF-8");
    //                client.container().withName(container.getId()).remove();
    //                return new String(outputStream.toByteArray(), Charset.defaultCharset());
    ////                return "";
    //            }
    //        }
    //
    //        client.container().withName(container.getId()).remove();
    //        return "";
    //
    //    }
    //
    //    @Override
    //    public String runServiceContainer(String packageName, String dockerEnv) throws BallerinaDockerClientException {
    //        DockerClient client = getDockerClient(dockerEnv);
    //        if (isFunctionImage(client, packageName)) {
    //            throw new BallerinaDockerClientException("Invalid image to run: " +
    // packageName.toLowerCase(Locale.getDefault()) +
    //                    ":latest");
    //        }
    //
    //        ContainerCreateResponse container = client.container().createNew()
    //                .withName(packageName + "-latest")
    //                .withImage(packageName.toLowerCase(Locale.getDefault()) + ":latest")
    //                .done();
    //
    //        client.container().withName(container.getId()).start();
    ////        if (client.container().withName(container.getId()).start()) {
    //////            ("Container started: " + container.getId());
    ////        }
    //
    //        String dockerUrl;
    //        if (dockerEnv == null) {
    //            dockerUrl = "http://localhost:" + "9090" + File.separator;
    //        } else {
    //            dockerUrl = dockerEnv.substring(0, dockerEnv.lastIndexOf(":")) + "9090" + File.separator;
    //        }
    //
    //        return dockerUrl;
    //    }
    //
    //    @Override
    //    public void stopContainer(String packageName, String dockerEnv) throws BallerinaDockerClientException {
    ////        DockerClient client = getDockerClient(dockerEnv);
    //        throw new BallerinaDockerClientException("Not implemented!");
    //    }
}