Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapred; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.net.ConnectException; import java.net.InetSocketAddress; import java.net.SocketException; import java.net.ServerSocket; import java.net.Socket; import java.net.URLEncoder; import java.net.UnknownHostException; import java.util.Enumeration; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Pattern; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.PostMethod; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.corona.ClusterManagerService; import org.apache.hadoop.corona.CoronaConf; import org.apache.hadoop.corona.InetAddress; import org.apache.hadoop.corona.SessionHistoryManager; import org.apache.hadoop.corona.SessionInfo; import org.apache.hadoop.corona.TFactoryBasedThreadPoolServer; import org.apache.hadoop.corona.Utilities; import org.apache.hadoop.corona.CoronaProxyJobTrackerService; import org.apache.hadoop.corona.RunningSession; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.http.HttpServer; import org.apache.hadoop.ipc.ProtocolSignature; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.metrics.MetricsContext; import org.apache.hadoop.metrics.MetricsRecord; import org.apache.hadoop.metrics.MetricsUtil; import org.apache.hadoop.metrics.Updater; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.StringUtils; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.TException; import org.apache.thrift.server.TServer; import org.apache.thrift.transport.TFramedTransport; import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; /** * This is used to proxy HTTP requests to individual Corona Job Tracker web * UIs. * Also used for aggregating information about jobs such as job counters. */ public class ProxyJobTracker implements JobHistoryObserver, CoronaJobAggregator, Updater, CoronaProxyJobTrackerService.Iface { /** Logger. */ private static final Log LOG = LogFactory.getLog(ProxyJobTracker.class); public static final Pattern UNUSED_JOBFILE_PATTERN = Pattern.compile("^(.+)\\/job_(\\d+)\\.(\\d+)_(\\d+)$"); public static final Pattern UNUSED_JOBHISTORY_PATTERN = Pattern.compile("^(.+)\\/(\\d+)\\.(\\d+)$"); static { Utilities.makeProcessExitOnUncaughtException(LOG); } /** Local machine name. */ private static String LOCALMACHINE; /** Http server port. */ private static int LOCALPORT; /** Configuration. */ private static CoronaConf conf; /** Session History Manager. */ private static SessionHistoryManager sessionHistoryManager; /** Default clock. */ private static final Clock DEFAULTCLOCK = new Clock(); /** Clock. */ private Clock clock = null; /** Filesystem. */ private FileSystem fs = null; /** The HTTP server. */ private HttpServer infoServer; /** The RPC server. */ private Server rpcServer; /** Cache expiry. */ private ExpireUnusedFilesInCache expireUnusedFilesInCache; /** Job dir cleanup. */ private ExpireUnusedJobFiles expireUnusedJobFiles; private ExpireUnusedJobFiles expireUnusedJobHistory; /** Start time of the server. */ private long startTime; /** Aggregate job counters. */ private Counters aggregateCounters = new Counters(); /** Aggregate error counters. */ private Counters aggregateErrors = new Counters(); /** Aggregate job stats. */ private JobStats aggregateJobStats = new JobStats(); /** Job Counters aggregated by pool */ private Map<String, Counters> poolToJobCounters = new HashMap<String, Counters>(); /** Job Stats aggregated by pool */ private Map<String, JobStats> poolToJobStats = new HashMap<String, JobStats>(); /** Metrics context. */ private MetricsContext context; /** Metrics record. */ private MetricsRecord metricsRecord; /** Is the Cluster Manager in Safe Mode? */ private volatile boolean clusterManagerSafeMode; /** Metrics Record for pools */ private Map<String, MetricsRecord> poolToMetricsRecord = new HashMap<String, MetricsRecord>(); /* Thrift server */ private TServer thriftServer; /* This is the thrift server thread */ private TServerThread thriftServerThread; /* The thrift server thread class */ public class TServerThread extends Thread { private TServer server; public TServerThread(TServer server) { this.server = server; } public void run() { try { server.serve(); } catch (Exception e) { LOG.info("Got an exception: ", e); } } } @Override public void doUpdates(MetricsContext unused) { synchronized (aggregateJobStats) { // Update metrics with aggregate job stats and reset the aggregate. aggregateJobStats.incrementMetricsAndReset(metricsRecord); incrementMetricsAndReset(metricsRecord, aggregateCounters); incrementMetricsAndReset(metricsRecord, aggregateErrors); for (Map.Entry<String, MetricsRecord> entry : poolToMetricsRecord.entrySet()) { String pool = entry.getKey(); JobStats poolJobStats = poolToJobStats.get(pool); poolJobStats.incrementMetricsAndReset(entry.getValue()); Counters poolCounters = poolToJobCounters.get(pool); incrementMetricsAndReset(entry.getValue(), poolCounters); } } metricsRecord.update(); } private static void incrementMetricsAndReset(MetricsRecord record, Counters counters) { // Now update metrics with the counters and reset the aggregate. for (Counters.Group group : counters) { String groupName = group.getName(); for (Counter counter : group) { String name = groupName + "_" + counter.getName(); name = name.replaceAll("[^a-zA-Z_]", "_").toLowerCase(); record.incrMetric(name, counter.getValue()); } } for (Counters.Group group : counters) { for (Counter c : group) { c.setValue(0); } } } /** * Servlet to handle requests. */ public static class ProxyJobTrackerServlet extends HttpServlet { @Override public void init() throws ServletException { LOG.info("Initialized " + this.getClass().getName()); super.init(); } @Override protected void doPost(HttpServletRequest request, HttpServletResponse response) throws IOException { String destination = ""; String host = request.getParameter("host"); String port = request.getParameter("port"); String path = request.getParameter("path"); if (host == null || port == null || path == null) { response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Missing mandatory host and/or port parameters"); return; } try { destination = "http://" + host + ":" + port + "/" + path; PostMethod method = new PostMethod(destination); for (Enumeration e = request.getParameterNames(); e.hasMoreElements();) { String key = (String) e.nextElement(); if (key.equals("host") || key.equals("port") || key.equals("path")) { continue; } method.addParameter(key, request.getParameter(key)); } HttpClient httpclient = new HttpClient(); int statusCode = httpclient.executeMethod(method); response.setStatus(statusCode); response.setContentType("text/html"); InputStream is = method.getResponseBodyAsStream(); int len = 0; int bufferSize = 4096; byte[] buf = new byte[bufferSize]; while ((len = is.read(buf)) >= 0) { response.getOutputStream().write(buf, 0, len); } if (statusCode != HttpServletResponse.SC_OK) { LOG.warn("Status " + statusCode + " forwarding request to: " + destination); } } catch (ConnectException ce) { handleDeadJobTracker(response, host, port); } catch (SocketException se) { checkDeadJobTracker(se, response, host, port); } catch (IOException e) { LOG.warn("Exception forwarding request to: " + destination); throw e; } } private String getMethodString(HttpServletRequest request, boolean isRetry) throws IOException { String methodString = null; String jobIDParam = request.getParameter("jobid"); // Check if the full path is given. String jobHistoryFileLocParam = request.getParameter("jobhistoryfileloc"); Path jobHistoryFileLocation = jobHistoryFileLocParam == null ? null : new Path(jobHistoryFileLocParam); if (jobHistoryFileLocation == null) { // If the full path is not given, check the history directory. String historyDirParam = request.getParameter("historydir"); Path historyDir = null; if (historyDirParam != null) { historyDir = new Path(conf.getSessionsLogDir(), historyDirParam); } else if (jobIDParam != null) { // Infer the history location from the job id. JobID jobID = JobID.forName(jobIDParam); String sessionId = jobID.getJtIdentifier(); historyDir = new Path(sessionHistoryManager.getLogPath(sessionId)); } if (historyDir != null) { if (!isRetry) { Path doneDir = new Path(historyDir, "done"); jobHistoryFileLocation = new Path(doneDir, jobIDParam); } else { jobHistoryFileLocation = new Path(historyDir, jobIDParam); } } } //check if the job is in the jobHistory methodString = (jobHistoryFileLocation == null) ? null : urlInJobHistory(jobHistoryFileLocation, jobIDParam); if (methodString == null) { LOG.info("history file: " + jobHistoryFileLocation + " is not in jobhistory"); } return methodString; } private String getRunningMethodString(HttpServletRequest request) { String methodString = null; String host = request.getParameter("host"); String port = request.getParameter("port"); String path = request.getParameter("path"); StringBuffer sb = new StringBuffer("http://"); sb.append(host).append(":").append(port).append("/").append(path); Map<String, String[]> m = request.getParameterMap(); boolean firstArg = true; for (Map.Entry<String, String[]> e : m.entrySet()) { String key = e.getKey(); //also ingore the jobhistoryfileloc, only used when the job is done //and log is in the job history if (key.equals("host") || key.equals("path") || key.equals("port") || key.equals("jobhistoryfileloc")) { continue; } if (firstArg) { sb.append('?'); firstArg = false; } else { sb.append('&'); } sb.append(e.getKey() + "=" + e.getValue()[0]); } methodString = sb.toString(); return methodString; } private String getRunningJobUrl(HttpServletRequest request) { String jobIDParam = request.getParameter("jobid"); if (jobIDParam == null) { return null; } SessionInfo sessionInfo; try { String sessionHandle = JobID.forName(jobIDParam).getJtIdentifier(); sessionInfo = getRunningSessionInfo(sessionHandle); } catch (Exception e) { LOG.info("Failed to get running session info for jobid: " + jobIDParam + ", exception: " + e.getMessage()); return null; } return sessionInfo.getUrl(); } private boolean isJobTrackerAlive(String host, String port) { if (host == null || port == null) { return false; } try { Socket s = new Socket(host, Integer.parseInt(port)); s.close(); return true; } catch (Exception e) { LOG.info("The job tracker " + host + ":" + port + " is not alive", e); return false; } } private boolean getHistory(HttpServletRequest request, HttpServletResponse response, boolean isRetry) throws ServletException, IOException { String methodString = null; String host = request.getParameter("host"); String port = request.getParameter("port"); String path = request.getParameter("path"); HttpMethod method = null; try { if (!isJobTrackerAlive(host, port)) { methodString = getMethodString(request, isRetry); } //it's not in the job history //directly go to the job tracker to retrieve the job information //otherwise, directly load the jobhistory page if (methodString == null && !isRetry) { if (host == null || port == null) { String runningJobUrl = getRunningJobUrl(request); if (runningJobUrl != null) { response.sendRedirect(runningJobUrl); } else { response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Missing mandatory host and/or port parameters"); } return false; } methodString = getRunningMethodString(request); } if (methodString == null) { if (isRetry) { handleDeadJobTracker(response, host, port); } return false; } HttpClient httpclient = new HttpClient(); method = new GetMethod(methodString); int sc = httpclient.executeMethod(method); response.setStatus(sc); response.setContentType("text/html"); InputStream is = method.getResponseBodyAsStream(); int len = 0; int bufferSize = 4096; byte[] buf = new byte[bufferSize]; while ((len = is.read(buf)) >= 0) { response.getOutputStream().write(buf, 0, len); } if (sc != HttpServletResponse.SC_OK) { LOG.warn("Status " + sc + " forwarding request to: " + methodString); } } catch (ConnectException ce) { if (isRetry) { handleDeadJobTracker(response, host, port); } return false; } catch (SocketException se) { if (isRetry) { checkDeadJobTracker(se, response, host, port); } return false; } catch (IOException e) { if (isRetry) { LOG.warn("Exception forwarding request to: " + methodString); throw e; } return false; } finally { // release the HTTP connection if (method != null) { method.releaseConnection(); } } return true; } @Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { LOG.info(request.getRemoteHost() + " is reading the job history."); if (!getHistory(request, response, false)) { getHistory(request, response, true); } } /** * Check if the job tracker could be dead based on the exception * encountered and provide a helpful message in the response. * @param e the exception. * @param response the HTTP response * @param host The host of the job tracker. * @param port the port of the job tracker. * @throws IOException */ private void checkDeadJobTracker(IOException e, HttpServletResponse response, String host, String port) throws IOException { if (e.getMessage().contains("Broken pipe") || e.getMessage().contains("Connection reset")) { handleDeadJobTracker(response, host, port); } else { throw e; } } /** * Provide a helpful message in the response after the job tracker has * been determined to be dead. * @param response the HTTP response * @param host The host of the job tracker. * @param port the port of the job tracker. * @throws IOException */ private void handleDeadJobTracker(HttpServletResponse response, String host, String port) throws IOException { String msg = "Could not connect to Job Tracker at " + host + ":" + port + ". The job may have completed or been killed. Please go to the " + "Cluster Manager UI and click on your job again, or retry the " + "tracking URL if available."; byte[] msgBytes = msg.getBytes(); response.getOutputStream().write(msgBytes, 0, msgBytes.length); } } public int getPort() { return infoServer.getPort(); } public int getRpcPort() { return rpcServer.getListenerAddress().getPort(); } public String getProxyJobTrackerMachine() { return LOCALMACHINE; } Clock getClock() { return clock == null ? DEFAULTCLOCK : clock; } public void historyFileCopied(JobID jobid, String historyFile) { } public ProxyJobTracker(CoronaConf conf) throws IOException { this.conf = conf; fs = FileSystem.get(conf); String infoAddr = conf.get("mapred.job.tracker.corona.proxyaddr", "0.0.0.0:0"); InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr); String infoBindAddress = infoSocAddr.getHostName(); int port = infoSocAddr.getPort(); LOCALMACHINE = infoBindAddress; startTime = getClock().getTime(); CoronaConf coronaConf = new CoronaConf(conf); InetSocketAddress rpcSockAddr = NetUtils.createSocketAddr(coronaConf.getProxyJobTrackerAddress()); rpcServer = RPC.getServer(this, rpcSockAddr.getHostName(), rpcSockAddr.getPort(), conf.getInt("corona.proxy.job.tracker.handler.count", 10), false, conf); rpcServer.start(); LOG.info("ProxyJobTracker RPC Server up at " + rpcServer.getListenerAddress()); infoServer = new HttpServer("proxyjt", infoBindAddress, port, port == 0, conf); infoServer.setAttribute("proxy.job.tracker", this); infoServer.setAttribute("conf", conf); infoServer.addServlet("proxy", "/proxy", ProxyJobTrackerServlet.class); // initialize history parameters. JobConf jobConf = new JobConf(conf); boolean historyInitialized = JobHistory.init(this, jobConf, this.LOCALMACHINE, this.startTime); if (historyInitialized) { JobHistory.initDone(jobConf, fs); String historyLogDir = JobHistory.getCompletedJobHistoryLocation().toString(); FileSystem historyFS = new Path(historyLogDir).getFileSystem(conf); infoServer.setAttribute("historyLogDir", historyLogDir); infoServer.setAttribute("fileSys", historyFS); } infoServer.start(); LOCALPORT = infoServer.getPort(); context = MetricsUtil.getContext("mapred"); metricsRecord = MetricsUtil.createRecord(context, "proxyjobtracker"); context.registerUpdater(this); expireUnusedFilesInCache = new ExpireUnusedFilesInCache(conf, getClock(), new Path(getSystemDir())); expireUnusedFilesInCache.setName("Cache File cleanup thread"); expireUnusedFilesInCache.start(); // 10 days long clearJobFileThreshold = conf.getLong("mapred.job.file.expirethreshold", 864000000L); long clearJobFileInterval = conf.getLong("mapred.job.file.checkinterval", 86400000L); expireUnusedJobFiles = new ExpireUnusedJobFiles(getClock(), conf, new Path(getSystemDir()), UNUSED_JOBFILE_PATTERN, clearJobFileThreshold, clearJobFileInterval); expireUnusedJobFiles.setName("Job File Cleanup Thread"); expireUnusedJobFiles.start(); long clearJobHistoryThreshold = conf.getLong("mapred.job.history.expirethreshold", 864000000L); long clearJobHistoryInterval = conf.getLong("mapred.job.history.checkinterval", 86400000L); expireUnusedJobHistory = new ExpireUnusedJobFiles(getClock(), conf, new Path(conf.getSessionsLogDir()), UNUSED_JOBHISTORY_PATTERN, clearJobHistoryThreshold, clearJobHistoryInterval); expireUnusedJobHistory.setName("Job History Cleanup Thread"); expireUnusedJobHistory.start(); sessionHistoryManager = new SessionHistoryManager(); sessionHistoryManager.setConf(conf); String target = conf.getProxyJobTrackerThriftAddress(); InetSocketAddress addr = NetUtils.createSocketAddr(target); LOG.info("Trying to start the Thrift Server at: " + target); ServerSocket serverSocket = new ServerSocket(addr.getPort()); thriftServer = TFactoryBasedThreadPoolServer .createNewServer(new CoronaProxyJobTrackerService.Processor(this), serverSocket, 5000); thriftServerThread = new TServerThread(thriftServer); thriftServerThread.start(); LOG.info("Thrift server started on: " + target); } @Override public void reportJobStats(String jobId, String pool, JobStats stats, Counters counters) { synchronized (aggregateJobStats) { aggregateJobStats.accumulate(stats); JobStats poolJobStats = poolToJobStats.get(pool); if (poolJobStats == null) { poolJobStats = new JobStats(); poolToJobStats.put(pool, poolJobStats); } poolJobStats.accumulate(stats); accumulateCounters(aggregateCounters, counters); Counters poolCounters = poolToJobCounters.get(pool); if (poolCounters == null) { poolCounters = new Counters(); poolToJobCounters.put(pool, poolCounters); } accumulateCounters(poolCounters, counters); if (!poolToMetricsRecord.containsKey(pool)) { MetricsRecord poolRecord = context.createRecord("pool-" + pool); poolToMetricsRecord.put(pool, poolRecord); } } } @Override public void reportJobErrorCounters(Counters counters) { String taskErrorGroupName = TaskErrorCollector.COUNTER_GROUP_NAME; synchronized (aggregateJobStats) { for (Counters.Counter counter : counters.getGroup(taskErrorGroupName)) { aggregateErrors.incrCounter(taskErrorGroupName, counter.getName(), counter.getCounter()); } } } private static void accumulateCounters(Counters aggregate, Counters increment) { for (JobInProgress.Counter key : JobInProgress.Counter.values()) { Counter counter = increment.findCounter(key); if (counter != null) { aggregate.findCounter(key).increment(counter.getValue()); } } for (Task.Counter key : Task.Counter.values()) { Counter counter = increment.findCounter(key); if (counter != null) { aggregate.findCounter(key).increment(counter.getValue()); } } for (Counters.Counter counter : increment.getGroup(Task.FILESYSTEM_COUNTER_GROUP)) { aggregate.incrCounter(Task.FILESYSTEM_COUNTER_GROUP, counter.getName(), counter.getValue()); } } private static void accumulateCounters(Counters.Group aggregate, Counters.Group increment) { for (Counters.Counter counter : increment) { aggregate.getCounterForName(counter.getName()).increment(counter.getValue()); } } @Override public long getProtocolVersion(String protocol, long clientVersion) throws IOException { if (protocol.equals(CoronaJobAggregator.class.getName())) { return CoronaJobAggregator.versionID; } else if (protocol.equals(ClusterManagerSafeModeProtocol.class.getName())) { return ClusterManagerSafeModeProtocol.versionID; } else { throw new IOException("Unknown protocol " + protocol); } } @Override public ProtocolSignature getProtocolSignature(String protocol, long clientVersion, int clientMethodsHash) throws IOException { return ProtocolSignature.getProtocolSignature(this, protocol, clientVersion, clientMethodsHash); } // Used by the CM to tell the CPJT if it's in Safe Mode. @Override public void setClusterManagerSafeModeFlag(boolean safeMode) { clusterManagerSafeMode = safeMode; LOG.info("On ProxyJobTracker, clusterManagerSafeModeFlag: " + clusterManagerSafeMode); } // Has the CM gone into Safe Mode and told the CPJT about it? @Override public boolean getClusterManagerSafeModeFlag() { return clusterManagerSafeMode; } private static SessionInfo getRunningSessionInfo(String sessionHandle) throws Exception { // Connect to cluster manager thrift service String target = CoronaConf.getClusterManagerAddress(conf); LOG.info("Connecting to Cluster Manager at " + target); InetSocketAddress address = NetUtils.createSocketAddr(target); TTransport transport = new TFramedTransport(new TSocket(address.getHostName(), address.getPort())); TProtocol protocol = new TBinaryProtocol(transport); ClusterManagerService.Client client = new ClusterManagerService.Client(protocol); transport.open(); LOG.info("Requesting running session info for handle: " + sessionHandle); SessionInfo info = client.getSessionInfo(sessionHandle); transport.close(); return info; } public void shutdown() throws Exception { infoServer.stop(); rpcServer.stop(); thriftServer.stop(); // Do an dummy connect to the thrift server port. This will cause an thrift // exception and move the server beyond the blocking accept. // Thread.interrupt() does not help. String target = conf.getProxyJobTrackerThriftAddress(); InetSocketAddress addr = NetUtils.createSocketAddr(target); try { new Socket(addr.getAddress(), addr.getPort()).close(); } catch (IOException e) { } } public void join() throws InterruptedException { infoServer.join(); rpcServer.join(); thriftServerThread.join(); } public static ProxyJobTracker startProxyTracker(CoronaConf conf) throws IOException { ProxyJobTracker result = new ProxyJobTracker(conf); return result; } /** * Given the path to the jobHistoryFile, check if the file already exists. * 1. If FileNoFoundException is caught, means the job is not yet finished, * and there is not job hisotry log file in the done directory * 2. If not, it means we get a hit for the jobHistoryFile, directly recover * the url to the coronoajobdetailshistory page. * @param jobId * @return url if the job is done and the jobHistory is in the jobHistory * folder, null if the job cannot be found in the jobHistory folder. * @throws IOException */ public static String urlInJobHistory(Path jobHistoryFileLocation, String jobId) throws IOException { try { FileSystem fs = jobHistoryFileLocation.getFileSystem(conf); fs.getFileStatus(jobHistoryFileLocation); } catch (FileNotFoundException e) { return null; } return "http://" + LOCALMACHINE + ":" + LOCALPORT + "/coronajobdetailshistory.jsp?jobid=" + jobId + "&logFile=" + URLEncoder.encode(jobHistoryFileLocation.toString()); } @Override public String getSystemDir() { return CoronaJobTracker.getSystemDir(fs, conf); } public static void main(String[] argv) throws IOException { StringUtils.startupShutdownMessage(ProxyJobTracker.class, argv, LOG); ProxyJobTracker p = startProxyTracker(new CoronaConf(new Configuration())); boolean joined = false; while (!joined) { try { p.join(); joined = true; } catch (InterruptedException e) { LOG.warn("Ignoring InterruptedException"); } } } @Override public void cleanJobHistoryCache(String jobId) throws TException { JSPUtil.cleanJobInfo(jobId); } }