com.cloudera.kitten.lua.AsapLuaContainerLaunchParameters.java Source code

Java tutorial

Introduction

Here is the source code for com.cloudera.kitten.lua.AsapLuaContainerLaunchParameters.java

Source

/**
 * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
 *
 * Cloudera, Inc. licenses this file to you under the Apache License,
 * Version 2.0 (the "License"). You may not use this file except in
 * compliance with the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 * CONDITIONS OF ANY KIND, either express or implied. See the License for
 * the specific language governing permissions and limitations under the
 * License.
 */
package com.cloudera.kitten.lua;

import gr.ntua.cslab.asap.operators.Operator;
import gr.ntua.cslab.asap.rest.beans.OperatorDictionary;
import gr.ntua.cslab.asap.rest.beans.WorkflowDictionary;
import gr.ntua.cslab.asap.workflow.MaterializedWorkflow1;
import gr.ntua.cslab.asap.workflow.WorkflowNode;

import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.net.URI;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.UUID;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.luaj.vm2.LuaValue;

import com.cloudera.kitten.ContainerLaunchParameters;
import com.cloudera.kitten.util.Extras;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

public class AsapLuaContainerLaunchParameters implements ContainerLaunchParameters {

    private static final Log LOG = LogFactory.getLog(AsapLuaContainerLaunchParameters.class);

    private final LuaWrapper lv;
    public final Configuration conf;
    public final Map<String, URI> localFileUris;
    private final Extras extras;

    private String dir;
    private String name;
    private String execScript;
    private int globalContainerId;

    private MaterializedWorkflow1 workflow;

    private OperatorDictionary operatorDictionary;
    private WorkflowNode operator;

    private String opName;

    public AsapLuaContainerLaunchParameters(LuaValue lv, String name, Configuration conf,
            Map<String, URI> localFileUris, MaterializedWorkflow1 workflow, String opName) throws IOException {
        this(new LuaWrapper(lv.checktable()), name, conf, localFileUris, new Extras(), workflow, opName);
    }

    public AsapLuaContainerLaunchParameters(LuaWrapper lv, String name, Configuration conf,
            Map<String, URI> localFileUris, MaterializedWorkflow1 workflow, String opName) throws IOException {
        this(lv, name, conf, localFileUris, new Extras(), workflow, opName);
    }

    public AsapLuaContainerLaunchParameters(LuaWrapper lv, String name, Configuration conf,
            Map<String, URI> localFileUris, Extras extras, MaterializedWorkflow1 workflow, String opName)
            throws IOException {
        this.name = name;
        this.lv = lv;
        this.conf = conf;
        this.localFileUris = localFileUris;
        this.extras = extras;
        this.workflow = workflow;
        this.opName = opName;
        this.operator = workflow.nodes.get(opName);

        globalContainerId = 0;
    }

    public List<String> getStageOutFiles() {
        List<String> ret = new ArrayList<String>();
        if (!lv.isNil(LuaFields.STAGEOUT)) {
            LuaWrapper a = lv.getTable(LuaFields.STAGEOUT);
            Iterator<LuaPair> restIter = a.arrayIterator();
            while (restIter.hasNext()) {
                ret.add(restIter.next().value.tojstring());
            }
        }
        List<String> outputFiles = operator.getOutputFiles();
        LOG.info("Output files: " + outputFiles);
        ret.addAll(outputFiles);
        return ret;
    }

    /* private List<String> getStageInFiles() {
         List<String> ret = new ArrayList<String>();
         for(WorkflowNode in : operator.inputs){
     String path = in.dataset.getParameter("Execution.path");
         }
        ret.add(operator.operator.getParameter("Execution.Output0.fileName"));
        return ret;
            
     }*/

    public int getCores() {
        return lv.getInteger(LuaFields.CORES);
    }

    public int getMemory() {
        return lv.getInteger(LuaFields.MEMORY);
    }

    @Override
    public Resource getContainerResource(Resource clusterMax) {
        LOG.info("Max cores: " + clusterMax.getVirtualCores());
        LOG.info("Max memory: " + clusterMax.getMemory());
        Resource rsrc = Records.newRecord(Resource.class);
        rsrc.setMemory(Math.min(clusterMax.getMemory(), getMemory()));
        LOG.info("Setting cores: " + getCores());
        rsrc.setVirtualCores(Math.min(clusterMax.getVirtualCores(), getCores()));
        return rsrc;
    }

    @Override
    public int getPriority() {
        return lv.isNil(LuaFields.PRIORITY) ? 0 : lv.getInteger(LuaFields.PRIORITY);
    }

    @Override
    public int getNumInstances() {
        return lv.isNil(LuaFields.INSTANCES) ? 1 : lv.getInteger(LuaFields.INSTANCES);
    }

    @Override
    public Map<String, LocalResource> getLocalResources() throws IOException {
        //System.out.println("localFileUris: "+localFileUris);
        Map<String, LocalResource> localResources = Maps.newHashMap();
        if (!lv.isNil(LuaFields.RESOURCES)) {
            LuaWrapper lr = lv.getTable(LuaFields.RESOURCES);
            for (LuaPair lp : lr) {
                try {
                    NamedLocalResource nlr = constructResource(lp);
                    localResources.put(nlr.name, nlr.resource);
                } catch (IOException e) {
                    LOG.error("Error constructing local resource: " + lp.key, e);
                }
            }
        }
        for (Map.Entry<String, String> elr : extras.getResources().entrySet()) {
            LocalResource rsrc = constructExtraResource(elr.getValue());
            if (rsrc != null) {
                localResources.put(elr.getKey(), rsrc);
            }
        }

        // Get a local resource for the configuration object.
        LocalResource confRsrc = constructExtraResource(LuaFields.KITTEN_JOB_XML_FILE);
        if (confRsrc != null) {
            localResources.put(LuaFields.KITTEN_JOB_XML_FILE, confRsrc);
        }

        addScript(localResources);
        addOperatorInputs(localResources);
        //LOG.info("localFileUris: "+localFileUris);
        LOG.info("localResources: " + localResources.keySet());
        //System.out.println(localResources);

        return localResources;
    }

    private void addOperatorInputs(Map<String, LocalResource> localResources) throws IOException {
        LOG.info("Inputs: " + operator.getInputFiles());
        FileSystem fs = FileSystem.get(conf);
        for (Entry<String, String> e : operator.getInputFiles().entrySet()) {
            if ((!e.getValue().startsWith("hdfs://")) && (!e.getValue().startsWith("$HDFS"))) {
                LOG.info("adding local resource: " + e);
                String inDir = dir;
                LocalResource rsrc = Records.newRecord(LocalResource.class);
                rsrc.setType(LocalResourceType.FILE);
                rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
                LOG.info("Adding input: " + inDir + "/" + e.getValue());
                Path dst = new Path(inDir + "/" + e.getValue());
                dst = fs.makeQualified(dst);
                FileStatus stat = fs.getFileStatus(dst);
                rsrc.setSize(stat.getLen());
                rsrc.setTimestamp(stat.getModificationTime());
                rsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
                localResources.put(e.getKey(), rsrc);
            }
        }
        /*for(String in : operator.getArguments().split(" ")){
           LOG.info("Adding input: "+in);
           LocalResource nl = constructScriptResource();
           localResources.put(in, nl);
        }*/
    }

    private void addScript(Map<String, LocalResource> lres) throws IOException {
        LocalResource nl = constructScriptResource();
        lres.put(execScript, nl);
    }

    private LocalResource constructScriptResource() throws IOException {
        LocalResource rsrc = Records.newRecord(LocalResource.class);
        rsrc.setType(LocalResourceType.FILE);
        rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
        String src = "file:///tmp/" + execScript;
        Path path = new Path(src);

        configureLocalScriptResourceForPath(rsrc, path);
        return rsrc;
    }

    private void configureLocalScriptResourceForPath(LocalResource rsrc, Path path) throws IOException {
        //System.out.println("URI: "+path.toUri());
        FileSystem fs = FileSystem.get(conf);

        Path dst = new Path(dir + "/" + path.getName());
        fs.moveFromLocalFile(path, dst);
        dst = fs.makeQualified(dst);

        FileStatus stat = fs.getFileStatus(dst);
        rsrc.setSize(stat.getLen());
        rsrc.setTimestamp(stat.getModificationTime());
        rsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    }

    private LocalResource constructExtraResource(String key) {
        LocalResource rsrc = Records.newRecord(LocalResource.class);
        rsrc.setType(LocalResourceType.FILE);
        rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
        try {
            Path path = new Path(localFileUris.get(key));
            configureLocalResourceForPath(rsrc, path);
        } catch (IOException e) {
            LOG.error("Error constructing extra local resource: " + key, e);
            return null;
        }
        return rsrc;
    }

    private static class NamedLocalResource {
        public final String name;
        public final LocalResource resource;

        public NamedLocalResource(String name, LocalResource resource) {
            this.name = name;
            this.resource = resource;
        }
    }

    private NamedLocalResource constructResource(LuaPair lp) throws IOException {
        LocalResource rsrc = Records.newRecord(LocalResource.class);
        LuaWrapper value = new LuaWrapper(lp.value.checktable());
        String name = lp.key.isint() ? "" : lp.key.tojstring();
        if (value.isNil(LuaFields.LOCAL_RESOURCE_TYPE)) {
            rsrc.setType(LocalResourceType.FILE);
        } else {
            rsrc.setType(LocalResourceType.valueOf(value.getString(LuaFields.LOCAL_RESOURCE_TYPE).toUpperCase()));
        }
        if (value.isNil(LuaFields.LOCAL_RESOURCE_VISIBILITY)) {
            rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
        } else {
            rsrc.setVisibility(LocalResourceVisibility
                    .valueOf(value.getString(LuaFields.LOCAL_RESOURCE_VISIBILITY).toUpperCase()));
        }
        if (!value.isNil(LuaFields.LOCAL_RESOURCE_URL)) {
            URI uri = URI.create(value.getString(LuaFields.LOCAL_RESOURCE_URL));
            rsrc.setResource(ConverterUtils.getYarnUrlFromURI(uri));
            if (name.isEmpty()) {
                name = (new File(uri.getPath())).getName();
            }
        } else if (!value.isNil(LuaFields.LOCAL_RESOURCE_HDFS_FILE)) {
            Path path = new Path(value.getString(LuaFields.LOCAL_RESOURCE_HDFS_FILE));
            configureLocalResourceForPath(rsrc, path);
            if (name.isEmpty()) {
                name = path.getName();
            }
        } else if (!value.isNil(LuaFields.LOCAL_RESOURCE_LOCAL_FILE)) {
            String src = value.getString(LuaFields.LOCAL_RESOURCE_LOCAL_FILE);
            Path path = new Path(localFileUris.get(src));
            configureLocalResourceForPath(rsrc, path);
            if (name.isEmpty()) {
                name = new Path(src).getName();
            }
        } else {
            throw new IllegalArgumentException("Invalid resource: no 'url', 'hdfs', or 'file' fields specified.");
        }
        return new NamedLocalResource(name, rsrc);
    }

    private void configureLocalResourceForPath(LocalResource rsrc, Path path) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        FileStatus stat = fs.getFileStatus(path);
        rsrc.setSize(stat.getLen());
        rsrc.setTimestamp(stat.getModificationTime());
        rsrc.setResource(ConverterUtils.getYarnUrlFromPath(path));
    }

    @Override
    public Map<String, String> getEnvironment() {
        Map<String, String> env = Maps.newHashMap(extras.getEnv());
        if (!lv.isNil(LuaFields.ENV)) {
            env.putAll(lv.getTable(LuaFields.ENV).asMap());
        }
        return env;
    }

    @Override
    public List<String> getCommands() throws IOException {
        List<String> cmds = Lists.newArrayList();
        if (!lv.isNil(LuaFields.COMMANDS)) {
            Iterator<LuaPair> pairsIter = lv.getTable(LuaFields.COMMANDS).arrayIterator();
            while (pairsIter.hasNext()) {
                LuaValue c = pairsIter.next().value;
                if (c.isstring()) {
                    cmds.add(c.tojstring());
                } else if (c.istable()) {
                    cmds.add(toCommand(new LuaWrapper(c.checktable())));
                }
            }
        } else if (!lv.isNil(LuaFields.COMMAND)) {
            if (lv.isTable(LuaFields.COMMAND)) {
                cmds.add(toCommand(lv.getTable(LuaFields.COMMAND)));
            } else {
                cmds.add(lv.getString(LuaFields.COMMAND));
            }
        }
        if (cmds.isEmpty()) {
            LOG.fatal("No commands found in container!");
        }

        dir = localFileUris.get(LuaFields.KITTEN_JOB_XML_FILE).getPath();
        dir = dir.substring(0, dir.lastIndexOf("/"));
        //System.out.println("Dir: " +dir);
        //String args = opName+" "+operator.getArguments();
        String args = operator.getArguments();

        List<String> oldcmds = cmds;
        cmds = new ArrayList<String>();
        String outdir = dir + "/" + this.name;//+"_"+globalContainerId;

        LOG.info("Inputs: " + operator.getInputFiles());
        for (Entry<String, String> e : operator.getInputFiles().entrySet()) {
            String inPath = e.getValue().replace("$HDFS_DIR", dir);
            inPath = inPath.replace("$HDFS_OP_DIR", outdir);
            LOG.info("adding hdfs input: " + e);
            cmds.add("/opt/hadoop-2.6.0/bin/hadoop fs -copyToLocal " + inPath + " .");

        }
        cmds.add("/opt/hadoop-2.6.0/bin/hadoop fs -mkdir " + outdir);
        args = args.replace("$HDFS_DIR", dir);
        args = args.replace("$HDFS_OP_DIR", outdir);
        for (String c : oldcmds) {
            cmds.add(c + " " + args);
        }

        //List<String> stageInFiles = getStageInFiles();

        //System.out.println("stageOutFiles: "+stageOutFiles);
        cmds.add("ls -ltr");
        //cmds.add("ls -ltr asapData/");

        List<String> stageOutFiles = getStageOutFiles();
        for (String f : stageOutFiles) {
            cmds.add("/opt/hadoop-2.6.0/bin/hadoop fs -moveFromLocal " + f + " " + outdir);
        }
        //System.out.println("Container commands: "+cmds);
        execScript = writeExecutionScript(cmds);
        globalContainerId++;

        LOG.info("Commands: " + cmds);

        cmds = new ArrayList<String>();
        cmds.add("./" + execScript + " 1> <LOG_DIR>/stdout 2> <LOG_DIR>/stderr");

        return cmds;
    }

    private String writeExecutionScript(List<String> cmds) throws IOException {
        UUID id = UUID.randomUUID();
        String ret = "script_" + id + ".sh";
        File fout = new File("/tmp/" + ret);
        FileOutputStream fos = new FileOutputStream(fout);

        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fos));

        bw.write("#!/bin/bash");
        bw.newLine();
        for (String c : cmds) {
            bw.write(c);
            bw.newLine();
        }
        bw.close();
        fout.setExecutable(true);
        return ret;
    }

    public String toCommand(LuaWrapper table) {
        StringBuilder sb = new StringBuilder(table.getString(LuaFields.COMMAND_BASE));
        if (!table.isNil(LuaFields.ARGS)) {
            LuaWrapper a = table.getTable(LuaFields.ARGS);
            Iterator<LuaPair> namedArgsIter = a.hashIterator();
            while (namedArgsIter.hasNext()) {
                LuaPair lp = namedArgsIter.next();
                sb.append(" ");
                sb.append(lp.key.tojstring());
                sb.append("=");
                sb.append(lp.value.tojstring());
            }
            Iterator<LuaPair> restIter = a.arrayIterator();
            while (restIter.hasNext()) {
                sb.append(" ");
                sb.append(restIter.next().value.tojstring());
            }
        }
        return sb.toString();
    }

    @Override
    public String getName() {
        return name;
    }

    @Override
    public String getLabels() {
        String ret = lv.getString(LuaFields.LABELS);
        if (ret.equals("nil")) {
            return null;
        } else
            return ret;
    }

    @Override
    public String[] getNodes() {
        String nodes = lv.getString(LuaFields.NODES);
        LOG.info("nodes: " + nodes);
        if (nodes.equals("nil")) {
            return null;
        }
        return nodes.split(",");
    }
}