com.baifendian.swordfish.execserver.runner.streaming.StreamingRunner.java Source code

Introduction

Here is the source code for com.baifendian.swordfish.execserver.runner.streaming.StreamingRunner.java

Source

/*
 * Copyright (C) 2017 Baifendian Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *          http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.baifendian.swordfish.execserver.runner.streaming;

import com.baifendian.swordfish.common.config.BaseConfig;
import com.baifendian.swordfish.common.job.struct.node.BaseParam;
import com.baifendian.swordfish.common.job.struct.node.BaseParamFactory;
import com.baifendian.swordfish.common.mail.EmailManager;
import com.baifendian.swordfish.dao.enums.ExecType;
import com.baifendian.swordfish.dao.model.StreamingResult;
import com.baifendian.swordfish.execserver.job.Job;
import com.baifendian.swordfish.execserver.job.JobManager;
import com.baifendian.swordfish.execserver.job.JobProps;
import com.baifendian.swordfish.execserver.parameter.SystemParamManager;
import com.baifendian.swordfish.execserver.utils.EnvHelper;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;

public class StreamingRunner implements Callable<Boolean> {

    /**
     * ??
     */
    private StreamingResult streamingResult;

    /**
     * , ? job id
     */
    private Logger logger;

    /**
     * 
     */
    private Job job;

    public StreamingRunner(StreamingResult streamingResult, Logger jobLogger) {
        this.streamingResult = streamingResult;
        this.logger = jobLogger;
    }

    @Override
    public Boolean call() {
        // "id/streamingId/id"
        String jobScriptPath = BaseConfig.getStreamingExecDir(streamingResult.getProjectId(),
                streamingResult.getStreamingId(), streamingResult.getExecId());

        logger.info("streaming id:{}, exec id:{}, script path:{}", streamingResult.getStreamingId(),
                streamingResult.getExecId(), jobScriptPath);

        // ??
        // ?, ? schedule time ?
        Map<String, String> systemParamMap = SystemParamManager.buildSystemParam(ExecType.DIRECT,
                streamingResult.getScheduleTime(), streamingResult.getExecId(), streamingResult.getJobId());

        // ?
        Map<String, String> customParamMap = streamingResult.getUserDefinedParamMap();

        // ??
        Map<String, String> allParamMap = new HashMap<>();

        if (systemParamMap != null) {
            allParamMap.putAll(systemParamMap);
        }

        if (customParamMap != null) {
            allParamMap.putAll(customParamMap);
        }

        JobProps props = new JobProps();

        props.setJobParams(streamingResult.getParameter());
        props.setWorkDir(jobScriptPath);
        props.setProxyUser(streamingResult.getProxyUser());
        props.setDefinedParams(allParamMap);
        props.setProjectId(streamingResult.getProjectId());
        props.setExecJobId(streamingResult.getStreamingId());
        props.setExecId(streamingResult.getExecId());
        props.setEnvFile(BaseConfig.getSystemEnvPath());
        props.setQueue(streamingResult.getQueue());
        props.setExecJobStartTime(streamingResult.getScheduleTime());
        props.setJobAppId(streamingResult.getJobId()); //  job id 

        boolean success = false;

        try {
            // 
            EnvHelper.workDirAndUserCreate(jobScriptPath, streamingResult.getProxyUser(), logger);

            // ???? "?" ?
            List<String> projectRes = genProjectResFiles();

            //  hdfs ??
            EnvHelper.copyResToLocal(streamingResult.getProjectId(), jobScriptPath, projectRes, logger);

            // ??
            job = JobManager.newJob(streamingResult.getType(), props, logger);

            // job ?
            job.init();

            // job ??
            job.before();

            // job ?
            job.process();

            // job ??
            job.after();

            success = (job.getExitCode() == 0);
        } catch (Exception e) {
            success = false;

            logger.error(String.format("job process exception, streaming job id: %s, exec id: %s",
                    streamingResult.getStreamingId(), streamingResult.getExecId()), e);
        } finally {
            kill();

            // ?, ?, ??
            if (!BaseConfig.isDevelopMode()) {
                try {
                    FileUtils.deleteDirectory(new File(jobScriptPath));
                } catch (IOException e) {
                    logger.error(String.format("delete dir exception: %s", jobScriptPath), e);
                }
            }

            logger.info("job process done, streaming job id: {}, exec id: {}, success: {}",
                    streamingResult.getStreamingId(), streamingResult.getExecId(), success);
        }

        // ????
        EmailManager.sendMessageOfStreamingJob(streamingResult);

        return success;
    }

    /**
     * ?
     */
    private List<String> genProjectResFiles() throws IllegalArgumentException, InvocationTargetException,
            NoSuchMethodException, InstantiationException, IllegalAccessException {

        // ?
        Set<String> projectFiles = new HashSet<>();

        // ??
        BaseParam baseParam = BaseParamFactory.getBaseParam(streamingResult.getType(),
                streamingResult.getParameter());

        // ???
        if (baseParam != null) {
            List<String> projectResourceFiles = baseParam.getProjectResourceFiles();
            if (projectResourceFiles != null) {
                projectFiles.addAll(projectResourceFiles);
            }
        }

        return new ArrayList<>(projectFiles);
    }

    /**
     * 
     */
    public void kill() {
        if (job != null/* && job.isStarted()*/) {
            try {
                job.cancel(false);
            } catch (Exception e) {
                logger.error(e.getMessage(), e);
            }
        }
    }
}