Java tutorial
/* * Copyright (C) 2017 Baifendian Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.baifendian.swordfish.execserver.job; import com.baifendian.swordfish.common.hadoop.YarnRestClient; import com.baifendian.swordfish.dao.DaoFactory; import com.baifendian.swordfish.dao.FlowDao; import com.baifendian.swordfish.dao.StreamingDao; import com.baifendian.swordfish.dao.enums.FlowStatus; import com.baifendian.swordfish.dao.model.ExecutionNode; import com.baifendian.swordfish.dao.model.StreamingResult; import java.io.File; import java.io.IOException; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; public abstract class AbstractYarnJob extends Job { // ? id private static final Pattern APPLICATION_REGEX = Pattern.compile("application_\\d+_\\d+"); // ? job id private static final Pattern JOB_REGEX = Pattern.compile("job_\\d+_\\d+"); /** * ?? */ private FlowDao flowDao; /** * ??? */ private StreamingDao streamingDao; /** * links */ protected List<String> appLinks; /** * links */ protected List<String> jobLinks; public AbstractYarnJob(JobProps props, boolean isLongJob, Logger logger) { super(props, isLongJob, logger); flowDao = DaoFactory.getDaoInstance(FlowDao.class); streamingDao = DaoFactory.getDaoInstance(StreamingDao.class); appLinks = Collections.synchronizedList(new ArrayList<>()); jobLinks = Collections.synchronizedList(new ArrayList<>()); } /** * ? log , ? * * @param logs ? */ @Override public void logProcess(List<String> logs) { super.logProcess(logs); boolean captureAppLinks = false; boolean captureJobLinks = false; // ? for (String log : logs) { // app id ? String appId = findAppId(log); if (StringUtils.isNotEmpty(appId) && !appLinks.contains(appId)) { logger.info("find app id: {}", appId); appLinks.add(appId); captureAppLinks = true; } // job id ? String jobId = findJobId(log); if (StringUtils.isNotEmpty(jobId) && !jobLinks.contains(jobId)) { logger.info("find job id: {}", jobId); jobLinks.add(jobId); captureJobLinks = true; } } // ???? if (captureAppLinks || captureJobLinks) { // if (!isLongJob()) { ExecutionNode executionNode = flowDao.queryExecutionNode(props.getExecId(), props.getNodeName()); if (executionNode != null) { if (captureAppLinks) { // ?, ? add, ?, ? append executionNode.addAppLinkList(appLinks); } if (captureJobLinks) { // ?, ? add, ?, ? append executionNode.addJobLinkList(jobLinks); } logger.info("update execution node, execution id:{} and node name:{}", props.getExecId(), props.getNodeName()); flowDao.updateExecutionNode(executionNode); } } else { // StreamingResult streamingResult = streamingDao.queryStreamingExec(props.getExecId()); if (streamingResult != null) { if (captureAppLinks) { streamingResult.setAppLinkList(appLinks); } if (captureJobLinks) { streamingResult.setJobLinkList(appLinks); } logger.info("update streaming, execution id:{}", props.getExecId()); streamingDao.updateResult(streamingResult); } } } // ??, ?, ?? if (!isLongJob && isCancel() && !appLinks.isEmpty()) { try { cancelApplication(appLinks, props, logger); } catch (Exception e) { logger.error("catch an exception", e); } } } /** * ??, yarn ?, ???, ? */ @Override public boolean isCompleted() { if (CollectionUtils.isNotEmpty(appLinks)) { String appId = appLinks.get(appLinks.size() - 1); try { FlowStatus status = YarnRestClient.getInstance().getApplicationStatus(appId); if (status == null) { complete = false; return complete; } logger.info("current status is: {}", status); // ??, OK if (status.typeIsFinished() || status == FlowStatus.RUNNING) { complete = true; } } catch (Exception e) { logger.error(String.format("request status of application %s exception", appId), e); complete = true; } } return complete; } /** * ? yarn * * @param cancelApplication ?? yarn */ @Override public void cancel(boolean cancelApplication) throws Exception { cancel = true; if (cancelApplication) { cancelApplication(appLinks, props, logger); } } /** * * * @param appLinks * @param props ?, job application id, ? */ public static void cancelApplication(List<String> appLinks, JobProps props, Logger logger) throws IOException { logger.info("begin cancel yarn application..."); // ? kill application, ?, ?(???) if (CollectionUtils.isNotEmpty(appLinks)) { String appid = appLinks.get(appLinks.size() - 1); String commandFile = String.format("%s/%s_%s.kill", props.getWorkDir(), props.getJobAppId(), appid); String cmd = "yarn application -kill " + appid; StringBuilder sb = new StringBuilder(); sb.append("#!/bin/sh\n"); sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n"); sb.append("cd $BASEDIR\n"); if (props.getEnvFile() != null) { sb.append("source " + props.getEnvFile() + "\n"); } sb.append("\n\n"); sb.append(cmd); File f = new File(commandFile); if (!f.exists()) { FileUtils.writeStringToFile(new File(commandFile), sb.toString(), Charset.forName("UTF-8")); } // ??? String runCmd = "sh " + commandFile; if (StringUtils.isNotEmpty(props.getProxyUser())) { runCmd = "sudo -u " + props.getProxyUser() + " " + runCmd; } logger.info("kill cmd:{}", runCmd); try { // ?, ??, Runtime.getRuntime().exec(runCmd); } catch (Exception e) { logger.error(String.format("kill application %s exception", appid), e); } } logger.info("end cancel yarn application."); } /** * ? appid <p> * * @return appid */ private String findAppId(String line) { Matcher matcher = APPLICATION_REGEX.matcher(line); if (matcher.find()) { return matcher.group(); } return null; } /** * job id */ private String findJobId(String line) { Matcher matcher = JOB_REGEX.matcher(line); if (matcher.find()) { return matcher.group(); } return null; } public static void main(String[] args) { String msg = "INFO : Starting Job = job_1499151077551_0548, Tracking URL = http://bgsbtsp0006-dqf:8088/proxy/application_1499151077551_0548/\n"; // application id Matcher matcher = APPLICATION_REGEX.matcher(msg); while (matcher.find()) { System.out.println(matcher.group()); } // job id matcher = JOB_REGEX.matcher(msg); while (matcher.find()) { System.out.println(matcher.group()); } // ? msg msg = "sh.execserver.runner.node.NodeRunner:application[147] - hive execute log : INFO : Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 0"; // application id matcher = APPLICATION_REGEX.matcher(msg); while (matcher.find()) { System.out.println(matcher.group()); } // job id matcher = JOB_REGEX.matcher(msg); while (matcher.find()) { System.out.println(matcher.group()); } } }