com.ibm.streamsx.topology.internal.context.remote.ZippedToolkitRemoteContext.java Source code

Java tutorial

Introduction

Here is the source code for com.ibm.streamsx.topology.internal.context.remote.ZippedToolkitRemoteContext.java

Source

/*
# Licensed Materials - Property of IBM
# Copyright IBM Corp. 2016, 2017  
 */
package com.ibm.streamsx.topology.internal.context.remote;

import static com.ibm.streamsx.topology.internal.context.remote.DeployKeys.keepArtifacts;
import static com.ibm.streamsx.topology.internal.graph.GraphKeys.CONFIG;
import static com.ibm.streamsx.topology.internal.graph.GraphKeys.graph;
import static com.ibm.streamsx.topology.internal.graph.GraphKeys.splAppName;
import static com.ibm.streamsx.topology.internal.graph.GraphKeys.splAppNamespace;
import static com.ibm.streamsx.topology.internal.gson.GsonUtilities.jstring;
import static com.ibm.streamsx.topology.internal.gson.GsonUtilities.object;
import static com.ibm.streamsx.topology.internal.gson.GsonUtilities.objectArray;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URISyntaxException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.Future;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;

import com.google.gson.JsonObject;
import com.ibm.streamsx.topology.context.remote.RemoteContext;
import com.ibm.streamsx.topology.internal.gson.GsonUtilities;
import com.ibm.streamsx.topology.internal.process.CompletedFuture;

public class ZippedToolkitRemoteContext extends ToolkitRemoteContext {

    private final boolean keepBuildArchive;

    public ZippedToolkitRemoteContext() {
        this.keepBuildArchive = false;
    }

    public ZippedToolkitRemoteContext(boolean keepBuildArchive) {
        this.keepBuildArchive = keepBuildArchive;
    }

    @Override
    public Type getType() {
        return Type.BUILD_ARCHIVE;
    }

    @Override
    public Future<File> _submit(JsonObject submission) throws Exception {
        final File toolkitRoot = super._submit(submission).get();
        return createCodeArchive(toolkitRoot, submission);
    }

    public Future<File> createCodeArchive(File toolkitRoot, JsonObject submission)
            throws IOException, URISyntaxException {

        String tkName = toolkitRoot.getName();

        Path zipOutPath = pack(toolkitRoot.toPath(), graph(submission), tkName);

        if (keepBuildArchive || keepArtifacts(submission)) {
            final JsonObject submissionResult = GsonUtilities.objectCreate(submission,
                    RemoteContext.SUBMISSION_RESULTS);
            submissionResult.addProperty(SubmissionResultsKeys.ARCHIVE_PATH, zipOutPath.toString());
        }

        JsonObject deployInfo = object(submission, SUBMISSION_DEPLOY);
        deleteToolkit(toolkitRoot, deployInfo);

        return new CompletedFuture<File>(zipOutPath.toFile());
    }

    private static Path pack(final Path folder, JsonObject graph, String tkName)
            throws IOException, URISyntaxException {
        String namespace = splAppNamespace(graph);
        String name = splAppName(graph);

        Path zipFilePath = Paths.get(folder.toAbsolutePath().toString() + ".zip");
        String workingDir = zipFilePath.getParent().toString();

        Path topologyToolkit = TkInfo.getTopologyToolkitRoot().getAbsoluteFile().toPath();

        // Paths to copy into the toolkit
        Map<Path, String> paths = new HashMap<>();

        // tkManifest is the list of toolkits contained in the archive
        try (PrintWriter tkManifest = new PrintWriter("manifest_tk.txt", "UTF-8")) {
            tkManifest.println(tkName);
            tkManifest.println("com.ibm.streamsx.topology");

            JsonObject configSpl = object(graph, CONFIG, "spl");
            if (configSpl != null) {
                objectArray(configSpl, "toolkits", tk -> {
                    File tkRoot = new File(jstring(tk, "root"));
                    String tkRootName = tkRoot.getName();
                    tkManifest.println(tkRootName);
                    paths.put(tkRoot.toPath(), tkRootName);
                });
            }
        }

        // mainComposite is a string of the namespace and the main composite.
        // This is used by the Makefile
        try (PrintWriter mainComposite = new PrintWriter("main_composite.txt", "UTF-8")) {
            mainComposite.print(namespace + "::" + name);
        }

        Path manifest = Paths.get(workingDir, "manifest_tk.txt");
        Path mainComp = Paths.get(workingDir, "main_composite.txt");
        Path makefile = topologyToolkit
                .resolve(Paths.get("opt", "python", "templates", "common", "Makefile.template"));

        paths.put(topologyToolkit, topologyToolkit.getFileName().toString());
        paths.put(manifest, "manifest_tk.txt");
        paths.put(mainComp, "main_composite.txt");
        paths.put(makefile, "Makefile");
        paths.put(folder, folder.getFileName().toString());

        addAllToZippedArchive(paths, zipFilePath);
        manifest.toFile().delete();
        mainComp.toFile().delete();

        return zipFilePath;
    }

    private static void addAllToZippedArchive(Map<Path, String> starts, Path zipFilePath) throws IOException {
        try (ZipArchiveOutputStream zos = new ZipArchiveOutputStream(zipFilePath.toFile())) {
            for (Path start : starts.keySet()) {
                final String rootEntryName = starts.get(start);
                Files.walkFileTree(start, new SimpleFileVisitor<Path>() {
                    public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                        // Skip pyc files.
                        if (file.getFileName().toString().endsWith(".pyc"))
                            return FileVisitResult.CONTINUE;

                        String entryName = rootEntryName;
                        String relativePath = start.relativize(file).toString();
                        // If empty, file is the start file.
                        if (!relativePath.isEmpty()) {
                            entryName = entryName + "/" + relativePath;
                        }
                        // Zip uses forward slashes
                        entryName = entryName.replace(File.separatorChar, '/');

                        ZipArchiveEntry entry = new ZipArchiveEntry(file.toFile(), entryName);
                        if (Files.isExecutable(file))
                            entry.setUnixMode(0100770);
                        else
                            entry.setUnixMode(0100660);

                        zos.putArchiveEntry(entry);
                        Files.copy(file, zos);
                        zos.closeArchiveEntry();
                        return FileVisitResult.CONTINUE;
                    }

                    public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
                            throws IOException {
                        final String dirName = dir.getFileName().toString();
                        // Don't include pyc files or .toolkit 
                        if (dirName.equals("__pycache__"))
                            return FileVisitResult.SKIP_SUBTREE;

                        ZipArchiveEntry dirEntry = new ZipArchiveEntry(dir.toFile(), rootEntryName + "/"
                                + start.relativize(dir).toString().replace(File.separatorChar, '/') + "/");
                        zos.putArchiveEntry(dirEntry);
                        zos.closeArchiveEntry();
                        return FileVisitResult.CONTINUE;
                    }
                });
            }
        }
    }
}