se.kth.karamel.backend.running.model.tasks.RunRecipeTask.java Source code

Java tutorial

Introduction

Here is the source code for se.kth.karamel.backend.running.model.tasks.RunRecipeTask.java

Source

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package se.kth.karamel.backend.running.model.tasks;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonIOException;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonSyntaxException;
import com.google.gson.stream.JsonReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.log4j.Logger;
import se.kth.karamel.backend.converter.ShellCommandBuilder;
import se.kth.karamel.backend.dag.DagParams;
import se.kth.karamel.backend.machines.MachineInterface;
import se.kth.karamel.backend.machines.TaskSubmitter;
import se.kth.karamel.backend.running.model.MachineRuntime;
import se.kth.karamel.common.stats.ClusterStats;
import se.kth.karamel.common.util.Settings;
import se.kth.karamel.common.exception.KaramelException;

/**
 *
 * @author kamal
 */
public class RunRecipeTask extends Task {

    private static final Logger logger = Logger.getLogger(RunRecipeTask.class);
    private final String recipeCanonicalName;
    private String json;
    private final String cookbookId;
    private final String cookbookName;

    public RunRecipeTask(MachineRuntime machine, ClusterStats clusterStats, String recipe, String json,
            TaskSubmitter submitter, String cookbookId, String cookbookName) {
        super(recipe, cookbookId + "/" + recipe, false, machine, clusterStats, submitter);
        this.recipeCanonicalName = recipe;
        this.json = json;
        this.cookbookId = cookbookId;
        this.cookbookName = cookbookName;
    }

    /**
     * Recursive method to merge two json objects. Scales O(N^2) - only viable for small json objects.
     *
     * @param origObj
     * @param paramObj
     */
    JsonObject merge(JsonObject origObj, JsonObject paramObj) {
        Set<Map.Entry<String, JsonElement>> original = origObj.entrySet();

        for (Map.Entry<String, JsonElement> entry : paramObj.entrySet()) {
            boolean exists = false;
            String pKey = entry.getKey();
            JsonElement pValue = entry.getValue();
            for (Map.Entry<String, JsonElement> o : original) {
                if (o.getKey().compareToIgnoreCase(pKey) == 0) {
                    if (o.getValue().isJsonObject() && pValue.isJsonObject()) {
                        merge(o.getValue().getAsJsonObject(), pValue.getAsJsonObject());
                        exists = true;
                    }
                }
            }
            if (exists == false) {
                origObj.add(pKey, pValue);
            }
        }
        return origObj;
    }

    @Override
    public List<ShellCommand> getCommands() throws IOException {

        Set<JsonElement> paramsToMerge = DagParams.getGlobalParams();
        if (paramsToMerge != null) {
            try {
                // Merge in Global return results into the json file.
                JsonElement obj = new JsonParser().parse(json);
                if (obj.isJsonObject()) {
                    JsonObject jsonObj = obj.getAsJsonObject();

                    for (JsonElement param : paramsToMerge) {
                        if (param.isJsonObject()) {
                            JsonObject paramObj = param.getAsJsonObject();
                            merge(jsonObj, paramObj);
                        }
                    }
                    GsonBuilder builder = new GsonBuilder();
                    builder.disableHtmlEscaping();
                    Gson gson = builder.setPrettyPrinting().create();
                    json = gson.toJson(jsonObj);
                } else {
                    logger.warn(String.format("Invalid json object for chef-solo: \n %s'", json));
                }
            } catch (JsonIOException | JsonSyntaxException ex) {
                logger.warn(String.format("Invalid return value as Json object: %s \n %s'", ex.toString(), json));
            }
        }

        if (commands == null) {
            String jsonFileName = recipeCanonicalName.replaceAll(Settings.COOKBOOK_DELIMITER, "__");
            commands = ShellCommandBuilder.makeSingleFileCommand(Settings.SCRIPT_PATH_RUN_RECIPE, "chef_json", json,
                    "json_file_name", jsonFileName, "log_file_name", jsonFileName, "sudo_command", getSudoCommand(),
                    "task_id", getId(), "succeedtasks_filepath", Settings.SUCCEED_TASKLIST_FILENAME, "pid_file",
                    Settings.PID_FILE_NAME);
        }
        return commands;
    }

    public String getRecipeCanonicalName() {
        return recipeCanonicalName;
    }

    public String getCookbookId() {
        return cookbookId;
    }

    public String getRecipeName() {
        return recipeCanonicalName.split(Settings.COOKBOOK_DELIMITER)[1];
    }

    public String getCookbookName() {
        return cookbookName;
    }

    public static String installRecipeIdFromCookbookName(String machineId, String cookbook) {
        String installName = cookbook + Settings.COOKBOOK_DELIMITER + Settings.INSTALL_RECIPE;
        return makeUniqueId(machineId, installName);
    }

    public static String installRecipeIdFromAnotherRecipeName(String machineId, String recipe) {
        String[] cmp = recipe.split(Settings.COOKBOOK_DELIMITER);
        String installName = cmp[0] + Settings.COOKBOOK_DELIMITER + Settings.INSTALL_RECIPE;
        return makeUniqueId(machineId, installName);
    }

    public static String makeUniqueId(String machineId, String recipe) {
        return recipe + " on " + machineId;
    }

    @Override
    public String uniqueId() {
        return makeUniqueId(super.getMachineId(), recipeCanonicalName);
    }

    @Override
    public Set<String> dagDependencies() {
        Set<String> deps = new HashSet<>();
        String installId = installRecipeIdFromAnotherRecipeName(getMachineId(), recipeCanonicalName);
        if (uniqueId().equals(installId)) {
            String id = VendorCookbookTask.makeUniqueId(getMachineId(), cookbookId);
            deps.add(id);
        } else {
            deps.add(installId);
        }
        return deps;
    }

    /**
     * It then parses the JSON object and updates a central location for Chef Attributes.
     *
     * @param sshMachine
     * @throws se.kth.karamel.common.exception.KaramelException
     */
    @Override
    public void collectResults(MachineInterface sshMachine) throws KaramelException {
        String remoteFile = Settings.RECIPE_RESULT_REMOTE_PATH(getRecipeCanonicalName());
        String localResultsFile = Settings.RECIPE_RESULT_LOCAL_PATH(getRecipeCanonicalName(),
                getMachine().getGroup().getCluster().getName(), getMachine().getPublicIp());
        try {
            sshMachine.downloadRemoteFile(remoteFile, localResultsFile, true);
        } catch (IOException ex) {
            logger.debug(String.format("No return values for %s on %s", getRecipeCanonicalName(),
                    getMachine().getPublicIp()));
            return;
        }
        JsonReader reader;
        try {
            reader = new JsonReader(new FileReader(localResultsFile));
        } catch (FileNotFoundException ex) {
            String msg = String.format("Cannot find the results file for %s on %s", getRecipeCanonicalName(),
                    getMachine().getPublicIp());
            throw new KaramelException(msg, ex);
        }
        JsonParser jsonParser = new JsonParser();
        try {
            JsonElement el = jsonParser.parse(reader);
            DagParams.setGlobalParams(el);
        } catch (JsonIOException | JsonSyntaxException ex) {
            throw new KaramelException(String.format("Invalid return value as Json object for %s on %s",
                    getRecipeCanonicalName(), getMachine().getPublicIp()), ex);
        }
    }

    @Override
    public void downloadExperimentResults(MachineInterface sshMachine) throws KaramelException {
        String remoteFile = Settings.EXPERIMENT_RESULT_REMOTE_PATH(getRecipeCanonicalName()) + ".out";
        String localResultsFile = Settings.EXPERIMENT_RESULT_LOCAL_PATH(getRecipeCanonicalName(),
                getMachine().getGroup().getCluster().getName(), getMachine().getPublicIp());
        try {
            sshMachine.downloadRemoteFile(remoteFile, localResultsFile, true);
        } catch (IOException ex) {
            logger.info(String.format("Cannot find experiment results for download for %s on %s",
                    getRecipeCanonicalName(), getMachine().getPublicIp()));
            return;
        }
    }
}