Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapred; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; import java.net.UnknownHostException; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.filecache.TrackerDistributedCacheManager; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.mapred.Counters.Counter; import org.apache.hadoop.mapred.Counters.Group; import org.apache.hadoop.mapred.QueueManager.QueueACL; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobSubmissionFiles; import org.apache.hadoop.mapreduce.security.TokenCache; import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.mapreduce.split.JobSplitWriter; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenRenewer; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.codehaus.jackson.JsonParseException; import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.ObjectMapper; /** * <code>JobClient</code> is the primary interface for the user-job to interact * with the {@link JobTracker}. * * <code>JobClient</code> provides facilities to submit jobs, track their * progress, access component-tasks' reports/logs, get the Map-Reduce cluster * status information etc. * * <p>The job submission process involves: * <ol> * <li> * Checking the input and output specifications of the job. * </li> * <li> * Computing the {@link InputSplit}s for the job. * </li> * <li> * Setup the requisite accounting information for the {@link DistributedCache} * of the job, if necessary. * </li> * <li> * Copying the job's jar and configuration to the map-reduce system directory * on the distributed file-system. * </li> * <li> * Submitting the job to the <code>JobTracker</code> and optionally monitoring * it's status. * </li> * </ol></p> * * Normally the user creates the application, describes various facets of the * job via {@link JobConf} and then uses the <code>JobClient</code> to submit * the job and monitor its progress. * * <p>Here is an example on how to use <code>JobClient</code>:</p> * <p><blockquote><pre> * // Create a new JobConf * JobConf job = new JobConf(new Configuration(), MyJob.class); * * // Specify various job-specific parameters * job.setJobName("myjob"); * * job.setInputPath(new Path("in")); * job.setOutputPath(new Path("out")); * * job.setMapperClass(MyJob.MyMapper.class); * job.setReducerClass(MyJob.MyReducer.class); * * // Submit the job, then poll for progress until the job is complete * JobClient.runJob(job); * </pre></blockquote></p> * * <h4 id="JobControl">Job Control</h4> * * <p>At times clients would chain map-reduce jobs to accomplish complex tasks * which cannot be done via a single map-reduce job. This is fairly easy since * the output of the job, typically, goes to distributed file-system and that * can be used as the input for the next job.</p> * * <p>However, this also means that the onus on ensuring jobs are complete * (success/failure) lies squarely on the clients. In such situations the * various job-control options are: * <ol> * <li> * {@link #runJob(JobConf)} : submits the job and returns only after * the job has completed. * </li> * <li> * {@link #submitJob(JobConf)} : only submits the job, then poll the * returned handle to the {@link RunningJob} to query status and make * scheduling decisions. * </li> * <li> * {@link JobConf#setJobEndNotificationURI(String)} : setup a notification * on job-completion, thus avoiding polling. * </li> * </ol></p> * * @see JobConf * @see ClusterStatus * @see Tool * @see DistributedCache */ public class JobClient extends Configured implements MRConstants, Tool { private static final Log LOG = LogFactory.getLog(JobClient.class); public static enum TaskStatusFilter { NONE, KILLED, FAILED, SUCCEEDED, ALL } private TaskStatusFilter taskOutputFilter = TaskStatusFilter.FAILED; private static final long MAX_JOBPROFILE_AGE = 1000 * 2; static { Configuration.addDefaultResource("mapred-default.xml"); Configuration.addDefaultResource("mapred-site.xml"); } /** * A NetworkedJob is an implementation of RunningJob. It holds * a JobProfile object to provide some info, and interacts with the * remote service to provide certain functionality. */ static class NetworkedJob implements RunningJob { private JobSubmissionProtocol jobSubmitClient; JobProfile profile; JobStatus status; long statustime; /** * We store a JobProfile and a timestamp for when we last * acquired the job profile. If the job is null, then we cannot * perform any of the tasks, so we throw an exception. * The job might be null if the JobTracker has completely forgotten * about the job. (eg, 24 hours after the job completes.) */ public NetworkedJob(JobStatus job, JobProfile prof, JobSubmissionProtocol jobSubmitClient) throws IOException { this.status = job; this.profile = prof; this.jobSubmitClient = jobSubmitClient; if (this.status == null) { throw new IOException("The Job status cannot be null"); } if (this.profile == null) { throw new IOException("The Job profile cannot be null"); } if (this.jobSubmitClient == null) { throw new IOException("The Job Submission Protocol cannot be null"); } this.statustime = System.currentTimeMillis(); } /** * Some methods rely on having a recent job profile object. Refresh * it, if necessary */ synchronized void ensureFreshStatus() throws IOException { if (System.currentTimeMillis() - statustime > MAX_JOBPROFILE_AGE) { updateStatus(); } } /** Some methods need to update status immediately. So, refresh * immediately * @throws IOException */ synchronized void updateStatus() throws IOException { this.status = jobSubmitClient.getJobStatus(profile.getJobID()); if (this.status == null) { throw new IOException("The job appears to have been removed."); } this.statustime = System.currentTimeMillis(); } /** * An identifier for the job */ public JobID getID() { return profile.getJobID(); } /** @deprecated This method is deprecated and will be removed. Applications should * rather use {@link #getID()}.*/ @Deprecated public String getJobID() { return profile.getJobID().toString(); } /** * The user-specified job name */ public String getJobName() { return profile.getJobName(); } /** * The name of the job file */ public String getJobFile() { return profile.getJobFile(); } /** * A URL where the job's status can be seen */ public String getTrackingURL() { return profile.getURL().toString(); } /** * A float between 0.0 and 1.0, indicating the % of map work * completed. */ public float mapProgress() throws IOException { ensureFreshStatus(); return status.mapProgress(); } /** * A float between 0.0 and 1.0, indicating the % of reduce work * completed. */ public float reduceProgress() throws IOException { ensureFreshStatus(); return status.reduceProgress(); } /** * A float between 0.0 and 1.0, indicating the % of cleanup work * completed. */ public float cleanupProgress() throws IOException { ensureFreshStatus(); return status.cleanupProgress(); } /** * A float between 0.0 and 1.0, indicating the % of setup work * completed. */ public float setupProgress() throws IOException { ensureFreshStatus(); return status.setupProgress(); } /** * Returns immediately whether the whole job is done yet or not. */ public synchronized boolean isComplete() throws IOException { updateStatus(); return (status.getRunState() == JobStatus.SUCCEEDED || status.getRunState() == JobStatus.FAILED || status.getRunState() == JobStatus.KILLED); } /** * True iff job completed successfully. */ public synchronized boolean isSuccessful() throws IOException { updateStatus(); return status.getRunState() == JobStatus.SUCCEEDED; } /** * Blocks until the job is finished */ public void waitForCompletion() throws IOException { while (!isComplete()) { try { Thread.sleep(5000); } catch (InterruptedException ie) { } } } /** * Tells the service to get the state of the current job. */ public synchronized int getJobState() throws IOException { updateStatus(); return status.getRunState(); } /** * Tells the service to terminate the current job. */ public synchronized void killJob() throws IOException { jobSubmitClient.killJob(getID()); } /** Set the priority of the job. * @param priority new priority of the job. */ public synchronized void setJobPriority(String priority) throws IOException { jobSubmitClient.setJobPriority(getID(), priority); } /** * Kill indicated task attempt. * @param taskId the id of the task to kill. * @param shouldFail if true the task is failed and added to failed tasks list, otherwise * it is just killed, w/o affecting job failure status. */ public synchronized void killTask(TaskAttemptID taskId, boolean shouldFail) throws IOException { jobSubmitClient.killTask(taskId, shouldFail); } /** @deprecated Applications should rather use {@link #killTask(TaskAttemptID, boolean)}*/ @Deprecated public synchronized void killTask(String taskId, boolean shouldFail) throws IOException { killTask(TaskAttemptID.forName(taskId), shouldFail); } /** * Fetch task completion events from jobtracker for this job. */ public synchronized TaskCompletionEvent[] getTaskCompletionEvents(int startFrom) throws IOException { return jobSubmitClient.getTaskCompletionEvents(getID(), startFrom, 10); } /** * Dump stats to screen */ @Override public String toString() { try { updateStatus(); } catch (IOException e) { } return "Job: " + profile.getJobID() + "\n" + "file: " + profile.getJobFile() + "\n" + "tracking URL: " + profile.getURL() + "\n" + "map() completion: " + status.mapProgress() + "\n" + "reduce() completion: " + status.reduceProgress() + "\n" + ((status.getRunState() == JobStatus.FAILED) ? ("Failure Info: " + status.getFailureInfo()) : ""); } /** * Returns the counters for this job */ public Counters getCounters() throws IOException { return jobSubmitClient.getJobCounters(getID()); } @Override public String[] getTaskDiagnostics(TaskAttemptID id) throws IOException { return jobSubmitClient.getTaskDiagnostics(id); } @Override public String getFailureInfo() throws IOException { //assuming that this is just being called after //we realized the job failed. SO we try avoiding //a rpc by not calling updateStatus ensureFreshStatus(); return status.getFailureInfo(); } } private JobSubmissionProtocol jobSubmitClient; private Path sysDir = null; private Path stagingAreaDir = null; private FileSystem fs = null; private UserGroupInformation ugi; private static final String TASKLOG_PULL_TIMEOUT_KEY = "mapreduce.client.tasklog.timeout"; private static final int DEFAULT_TASKLOG_TIMEOUT = 60000; static int tasklogtimeout; /** * Create a job client. */ public JobClient() { } /** * Build a job client with the given {@link JobConf}, and connect to the * default {@link JobTracker}. * * @param conf the job configuration. * @throws IOException */ public JobClient(JobConf conf) throws IOException { setConf(conf); init(conf); } /** * Connect to the default {@link JobTracker}. * @param conf the job configuration. * @throws IOException */ public void init(JobConf conf) throws IOException { String tracker = conf.get("mapred.job.tracker", "local"); tasklogtimeout = conf.getInt(TASKLOG_PULL_TIMEOUT_KEY, DEFAULT_TASKLOG_TIMEOUT); this.ugi = UserGroupInformation.getCurrentUser(); if ("local".equals(tracker)) { conf.setNumMapTasks(1); this.jobSubmitClient = new LocalJobRunner(conf); } else { this.jobSubmitClient = createRPCProxy(JobTracker.getAddress(conf), conf); } } private static JobSubmissionProtocol createRPCProxy(InetSocketAddress addr, Configuration conf) throws IOException { return (JobSubmissionProtocol) RPC.getProxy(JobSubmissionProtocol.class, JobSubmissionProtocol.versionID, addr, UserGroupInformation.getCurrentUser(), conf, NetUtils.getSocketFactory(conf, JobSubmissionProtocol.class)); } @InterfaceAudience.Private public static class Renewer extends TokenRenewer { @Override public boolean handleKind(Text kind) { return DelegationTokenIdentifier.MAPREDUCE_DELEGATION_KIND.equals(kind); } @SuppressWarnings("unchecked") @Override public long renew(Token<?> token, Configuration conf) throws IOException, InterruptedException { InetSocketAddress addr = SecurityUtil.getTokenServiceAddr(token); JobSubmissionProtocol jt = createRPCProxy(addr, conf); return jt.renewDelegationToken((Token<DelegationTokenIdentifier>) token); } @SuppressWarnings("unchecked") @Override public void cancel(Token<?> token, Configuration conf) throws IOException, InterruptedException { InetSocketAddress addr = SecurityUtil.getTokenServiceAddr(token); JobSubmissionProtocol jt = createRPCProxy(addr, conf); jt.cancelDelegationToken((Token<DelegationTokenIdentifier>) token); } @Override public boolean isManaged(Token<?> token) throws IOException { ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); DelegationTokenIdentifier id = new DelegationTokenIdentifier(); id.readFields(new DataInputStream(buf)); // AbstractDelegationToken converts given renewer to a short name, but // AbstractDelegationTokenSecretManager does not, so we have to do it String loginUser = UserGroupInformation.getLoginUser().getShortUserName(); return loginUser.equals(id.getRenewer().toString()); } } /** * Build a job client, connect to the indicated job tracker. * * @param jobTrackAddr the job tracker to connect to. * @param conf configuration. */ public JobClient(InetSocketAddress jobTrackAddr, Configuration conf) throws IOException { this.ugi = UserGroupInformation.getCurrentUser(); jobSubmitClient = createRPCProxy(jobTrackAddr, conf); } /** * Close the <code>JobClient</code>. */ public synchronized void close() throws IOException { if (!(jobSubmitClient instanceof LocalJobRunner)) { RPC.stopProxy(jobSubmitClient); } } /** * Get a filesystem handle. We need this to prepare jobs * for submission to the MapReduce system. * * @return the filesystem handle. * @throws IOException */ public synchronized FileSystem getFs() throws IOException { if (this.fs == null) { try { this.fs = ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { public FileSystem run() throws IOException { Path sysDir = getSystemDir(); return sysDir.getFileSystem(getConf()); } }); } catch (InterruptedException e) { throw new RuntimeException(e); } } return this.fs; } /* see if two file systems are the same or not * */ private boolean compareFs(FileSystem srcFs, FileSystem destFs) { URI srcUri = srcFs.getUri(); URI dstUri = destFs.getUri(); if (srcUri.getScheme() == null) { return false; } if (!srcUri.getScheme().equals(dstUri.getScheme())) { return false; } String srcHost = srcUri.getHost(); String dstHost = dstUri.getHost(); if ((srcHost != null) && (dstHost != null)) { try { srcHost = InetAddress.getByName(srcHost).getCanonicalHostName(); dstHost = InetAddress.getByName(dstHost).getCanonicalHostName(); } catch (UnknownHostException ue) { return false; } if (!srcHost.equals(dstHost)) { return false; } } else if (srcHost == null && dstHost != null) { return false; } else if (srcHost != null && dstHost == null) { return false; } //check for ports if (srcUri.getPort() != dstUri.getPort()) { return false; } return true; } // copies a file to the jobtracker filesystem and returns the path where it // was copied to private Path copyRemoteFiles(FileSystem jtFs, Path parentDir, final Path originalPath, final JobConf job, short replication) throws IOException, InterruptedException { //check if we do not need to copy the files // is jt using the same file system. // just checking for uri strings... doing no dns lookups // to see if the filesystems are the same. This is not optimal. // but avoids name resolution. FileSystem remoteFs = null; remoteFs = originalPath.getFileSystem(job); if (compareFs(remoteFs, jtFs)) { return originalPath; } // this might have name collisions. copy will throw an exception //parse the original path to create new path Path newPath = new Path(parentDir, originalPath.getName()); FileUtil.copy(remoteFs, originalPath, jtFs, newPath, false, job); jtFs.setReplication(newPath, replication); return newPath; } private URI getPathURI(Path destPath, String fragment) throws URISyntaxException { URI pathURI = destPath.toUri(); if (pathURI.getFragment() == null) { if (fragment == null) { pathURI = new URI(pathURI.toString() + "#" + destPath.getName()); } else { pathURI = new URI(pathURI.toString() + "#" + fragment); } } return pathURI; } /** * configure the jobconf of the user with the command line options of * -libjars, -files, -archives * @param job the JobConf * @param submitJobDir * @throws IOException */ private void copyAndConfigureFiles(JobConf job, Path jobSubmitDir) throws IOException, InterruptedException { short replication = (short) job.getInt("mapred.submit.replication", 10); copyAndConfigureFiles(job, jobSubmitDir, replication); // Set the working directory if (job.getWorkingDirectory() == null) { job.setWorkingDirectory(fs.getWorkingDirectory()); } } private void copyAndConfigureFiles(JobConf job, Path submitJobDir, short replication) throws IOException, InterruptedException { if (!(job.getBoolean("mapred.used.genericoptionsparser", false))) { LOG.warn("Use GenericOptionsParser for parsing the arguments. " + "Applications should implement Tool for the same."); } // Retrieve command line arguments placed into the JobConf // by GenericOptionsParser. String files = job.get("tmpfiles"); String libjars = job.get("tmpjars"); String archives = job.get("tmparchives"); // // Figure out what fs the JobTracker is using. Copy the // job to it, under a temporary name. This allows DFS to work, // and under the local fs also provides UNIX-like object loading // semantics. (that is, if the job file is deleted right after // submission, we can still run the submission to completion) // // Create a number of filenames in the JobTracker's fs namespace FileSystem fs = submitJobDir.getFileSystem(job); LOG.debug("default FileSystem: " + fs.getUri()); if (fs.exists(submitJobDir)) { throw new IOException("Not submitting job. Job directory " + submitJobDir + " already exists!! This is unexpected.Please check what's there in" + " that directory"); } submitJobDir = fs.makeQualified(submitJobDir); FsPermission mapredSysPerms = new FsPermission(JobSubmissionFiles.JOB_DIR_PERMISSION); FileSystem.mkdirs(fs, submitJobDir, mapredSysPerms); Path filesDir = JobSubmissionFiles.getJobDistCacheFiles(submitJobDir); Path archivesDir = JobSubmissionFiles.getJobDistCacheArchives(submitJobDir); Path libjarsDir = JobSubmissionFiles.getJobDistCacheLibjars(submitJobDir); // add all the command line files/ jars and archive // first copy them to jobtrackers filesystem if (files != null) { FileSystem.mkdirs(fs, filesDir, mapredSysPerms); String[] fileArr = files.split(","); for (String tmpFile : fileArr) { URI tmpURI; try { tmpURI = new URI(tmpFile); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } Path tmp = new Path(tmpURI); Path newPath = copyRemoteFiles(fs, filesDir, tmp, job, replication); try { URI pathURI = getPathURI(newPath, tmpURI.getFragment()); DistributedCache.addCacheFile(pathURI, job); } catch (URISyntaxException ue) { //should not throw a uri exception throw new IOException("Failed to create uri for " + tmpFile, ue); } DistributedCache.createSymlink(job); } } if (libjars != null) { FileSystem.mkdirs(fs, libjarsDir, mapredSysPerms); String[] libjarsArr = libjars.split(","); for (String tmpjars : libjarsArr) { Path tmp = new Path(tmpjars); Path newPath = copyRemoteFiles(fs, libjarsDir, tmp, job, replication); DistributedCache.addArchiveToClassPath(new Path(newPath.toUri().getPath()), job, fs); } } if (archives != null) { FileSystem.mkdirs(fs, archivesDir, mapredSysPerms); String[] archivesArr = archives.split(","); for (String tmpArchives : archivesArr) { URI tmpURI; try { tmpURI = new URI(tmpArchives); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } Path tmp = new Path(tmpURI); Path newPath = copyRemoteFiles(fs, archivesDir, tmp, job, replication); try { URI pathURI = getPathURI(newPath, tmpURI.getFragment()); DistributedCache.addCacheArchive(pathURI, job); } catch (URISyntaxException ue) { //should not throw an uri excpetion throw new IOException("Failed to create uri for " + tmpArchives, ue); } DistributedCache.createSymlink(job); } } // First we check whether the cached archives and files are legal. TrackerDistributedCacheManager.validate(job); // set the timestamps of the archives and files TrackerDistributedCacheManager.determineTimestamps(job); // set the public/private visibility of the archives and files TrackerDistributedCacheManager.determineCacheVisibilities(job); // get DelegationTokens for cache files TrackerDistributedCacheManager.getDelegationTokens(job, job.getCredentials()); String originalJarPath = job.getJar(); if (originalJarPath != null) { // copy jar to JobTracker's fs // use jar name if job is not named. if ("".equals(job.getJobName())) { job.setJobName(new Path(originalJarPath).getName()); } Path submitJarFile = JobSubmissionFiles.getJobJar(submitJobDir); job.setJar(submitJarFile.toString()); fs.copyFromLocalFile(new Path(originalJarPath), submitJarFile); fs.setReplication(submitJarFile, replication); fs.setPermission(submitJarFile, new FsPermission(JobSubmissionFiles.JOB_FILE_PERMISSION)); } else { LOG.warn("No job jar file set. User classes may not be found. " + "See JobConf(Class) or JobConf#setJar(String)."); } } /** * Submit a job to the MR system. * * This returns a handle to the {@link RunningJob} which can be used to track * the running-job. * * @param jobFile the job configuration. * @return a handle to the {@link RunningJob} which can be used to track the * running-job. * @throws FileNotFoundException * @throws InvalidJobConfException * @throws IOException */ public RunningJob submitJob(String jobFile) throws FileNotFoundException, InvalidJobConfException, IOException { // Load in the submitted job details JobConf job = new JobConf(jobFile); return submitJob(job); } /** * Submit a job to the MR system. * This returns a handle to the {@link RunningJob} which can be used to track * the running-job. * * @param job the job configuration. * @return a handle to the {@link RunningJob} which can be used to track the * running-job. * @throws FileNotFoundException * @throws IOException */ public RunningJob submitJob(JobConf job) throws FileNotFoundException, IOException { try { return submitJobInternal(job); } catch (InterruptedException ie) { throw new IOException("interrupted", ie); } catch (ClassNotFoundException cnfe) { throw new IOException("class not found", cnfe); } } /** * Internal method for submitting jobs to the system. * @param job the configuration to submit * @return a proxy object for the running job * @throws FileNotFoundException * @throws ClassNotFoundException * @throws InterruptedException * @throws IOException */ public RunningJob submitJobInternal(final JobConf job) throws FileNotFoundException, ClassNotFoundException, InterruptedException, IOException { /* * configure the command line options correctly on the submitting dfs */ return ugi.doAs(new PrivilegedExceptionAction<RunningJob>() { public RunningJob run() throws FileNotFoundException, ClassNotFoundException, InterruptedException, IOException { JobConf jobCopy = job; Path jobStagingArea = JobSubmissionFiles.getStagingDir(JobClient.this, jobCopy); JobID jobId = jobSubmitClient.getNewJobId(); Path submitJobDir = new Path(jobStagingArea, jobId.toString()); jobCopy.set("mapreduce.job.dir", submitJobDir.toString()); JobStatus status = null; try { populateTokenCache(jobCopy, jobCopy.getCredentials()); copyAndConfigureFiles(jobCopy, submitJobDir); // get delegation token for the dir TokenCache.obtainTokensForNamenodes(jobCopy.getCredentials(), new Path[] { submitJobDir }, jobCopy); Path submitJobFile = JobSubmissionFiles.getJobConfPath(submitJobDir); int reduces = jobCopy.getNumReduceTasks(); InetAddress ip = InetAddress.getLocalHost(); if (ip != null) { job.setJobSubmitHostAddress(ip.getHostAddress()); job.setJobSubmitHostName(ip.getHostName()); } JobContext context = new JobContext(jobCopy, jobId); // Check the output specification if (reduces == 0 ? jobCopy.getUseNewMapper() : jobCopy.getUseNewReducer()) { org.apache.hadoop.mapreduce.OutputFormat<?, ?> output = ReflectionUtils .newInstance(context.getOutputFormatClass(), jobCopy); output.checkOutputSpecs(context); } else { jobCopy.getOutputFormat().checkOutputSpecs(fs, jobCopy); } jobCopy = (JobConf) context.getConfiguration(); // Create the splits for the job FileSystem fs = submitJobDir.getFileSystem(jobCopy); LOG.debug("Creating splits at " + fs.makeQualified(submitJobDir)); int maps = writeSplits(context, submitJobDir); jobCopy.setNumMapTasks(maps); // write "queue admins of the queue to which job is being submitted" // to job file. String queue = jobCopy.getQueueName(); AccessControlList acl = jobSubmitClient.getQueueAdmins(queue); jobCopy.set(QueueManager.toFullPropertyName(queue, QueueACL.ADMINISTER_JOBS.getAclName()), acl.getACLString()); // Write job file to JobTracker's fs FSDataOutputStream out = FileSystem.create(fs, submitJobFile, new FsPermission(JobSubmissionFiles.JOB_FILE_PERMISSION)); try { jobCopy.writeXml(out); } finally { out.close(); } // // Now, actually submit the job (using the submit name) // printTokens(jobId, jobCopy.getCredentials()); status = jobSubmitClient.submitJob(jobId, submitJobDir.toString(), jobCopy.getCredentials()); JobProfile prof = jobSubmitClient.getJobProfile(jobId); if (status != null && prof != null) { return new NetworkedJob(status, prof, jobSubmitClient); } else { throw new IOException("Could not launch job"); } } finally { if (status == null) { LOG.info("Cleaning up the staging area " + submitJobDir); if (fs != null && submitJobDir != null) fs.delete(submitJobDir, true); } } } }); } @SuppressWarnings("unchecked") private void printTokens(JobID jobId, Credentials credentials) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Printing tokens for job: " + jobId); for (Token<?> token : credentials.getAllTokens()) { if (token.getKind().toString().equals("HDFS_DELEGATION_TOKEN")) { LOG.debug("Submitting with " + DFSClient.stringifyToken( (Token<org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier>) token)); } } } } @SuppressWarnings("unchecked") private <T extends InputSplit> int writeNewSplits(JobContext job, Path jobSubmitDir) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = job.getConfiguration(); InputFormat<?, ?> input = ReflectionUtils.newInstance(job.getInputFormatClass(), conf); List<InputSplit> splits = input.getSplits(job); T[] array = (T[]) splits.toArray(new InputSplit[splits.size()]); // sort the splits into order based on size, so that the biggest // go first Arrays.sort(array, new SplitComparator()); JobSplitWriter.createSplitFiles(jobSubmitDir, conf, jobSubmitDir.getFileSystem(conf), array); return array.length; } private int writeSplits(org.apache.hadoop.mapreduce.JobContext job, Path jobSubmitDir) throws IOException, InterruptedException, ClassNotFoundException { JobConf jConf = (JobConf) job.getConfiguration(); int maps; if (jConf.getUseNewMapper()) { maps = writeNewSplits(job, jobSubmitDir); } else { maps = writeOldSplits(jConf, jobSubmitDir); } return maps; } //method to write splits for old api mapper. private int writeOldSplits(JobConf job, Path jobSubmitDir) throws IOException { org.apache.hadoop.mapred.InputSplit[] splits = job.getInputFormat().getSplits(job, job.getNumMapTasks()); // sort the splits into order based on size, so that the biggest // go first Arrays.sort(splits, new Comparator<org.apache.hadoop.mapred.InputSplit>() { public int compare(org.apache.hadoop.mapred.InputSplit a, org.apache.hadoop.mapred.InputSplit b) { try { long left = a.getLength(); long right = b.getLength(); if (left == right) { return 0; } else if (left < right) { return 1; } else { return -1; } } catch (IOException ie) { throw new RuntimeException("Problem getting input split size", ie); } } }); JobSplitWriter.createSplitFiles(jobSubmitDir, job, jobSubmitDir.getFileSystem(job), splits); return splits.length; } private static class SplitComparator implements Comparator<InputSplit> { @Override public int compare(InputSplit o1, InputSplit o2) { try { long len1 = o1.getLength(); long len2 = o2.getLength(); if (len1 < len2) { return 1; } else if (len1 == len2) { return 0; } else { return -1; } } catch (IOException ie) { throw new RuntimeException("exception in compare", ie); } catch (InterruptedException ie) { throw new RuntimeException("exception in compare", ie); } } } /** * Checks if the job directory is clean and has all the required components * for (re) starting the job */ public static boolean isJobDirValid(Path jobDirPath, FileSystem fs) throws IOException { FileStatus[] contents = fs.listStatus(jobDirPath); int matchCount = 0; if (contents != null && contents.length >= 2) { for (FileStatus status : contents) { if ("job.xml".equals(status.getPath().getName())) { ++matchCount; } if ("job.split".equals(status.getPath().getName())) { ++matchCount; } } if (matchCount == 2) { return true; } } return false; } /** * Get an {@link RunningJob} object to track an ongoing job. Returns * null if the id does not correspond to any known job. * * @param jobid the jobid of the job. * @return the {@link RunningJob} handle to track the job, null if the * <code>jobid</code> doesn't correspond to any known job. * @throws IOException */ public RunningJob getJob(JobID jobid) throws IOException { JobStatus status = jobSubmitClient.getJobStatus(jobid); JobProfile profile = jobSubmitClient.getJobProfile(jobid); if (status != null && profile != null) { return new NetworkedJob(status, profile, jobSubmitClient); } else { return null; } } /**@deprecated Applications should rather use {@link #getJob(JobID)}. */ @Deprecated public RunningJob getJob(String jobid) throws IOException { return getJob(JobID.forName(jobid)); } /** * Get the information of the current state of the map tasks of a job. * * @param jobId the job to query. * @return the list of all of the map tips. * @throws IOException */ public TaskReport[] getMapTaskReports(JobID jobId) throws IOException { return jobSubmitClient.getMapTaskReports(jobId); } /**@deprecated Applications should rather use {@link #getMapTaskReports(JobID)}*/ @Deprecated public TaskReport[] getMapTaskReports(String jobId) throws IOException { return getMapTaskReports(JobID.forName(jobId)); } /** * Get the information of the current state of the reduce tasks of a job. * * @param jobId the job to query. * @return the list of all of the reduce tips. * @throws IOException */ public TaskReport[] getReduceTaskReports(JobID jobId) throws IOException { return jobSubmitClient.getReduceTaskReports(jobId); } /** * Get the information of the current state of the cleanup tasks of a job. * * @param jobId the job to query. * @return the list of all of the cleanup tips. * @throws IOException */ public TaskReport[] getCleanupTaskReports(JobID jobId) throws IOException { return jobSubmitClient.getCleanupTaskReports(jobId); } /** * Get the information of the current state of the setup tasks of a job. * * @param jobId the job to query. * @return the list of all of the setup tips. * @throws IOException */ public TaskReport[] getSetupTaskReports(JobID jobId) throws IOException { return jobSubmitClient.getSetupTaskReports(jobId); } /**@deprecated Applications should rather use {@link #getReduceTaskReports(JobID)}*/ @Deprecated public TaskReport[] getReduceTaskReports(String jobId) throws IOException { return getReduceTaskReports(JobID.forName(jobId)); } /** * Display the information about a job's tasks, of a particular type and * in a particular state * * @param jobId the ID of the job * @param type the type of the task (map/reduce/setup/cleanup) * @param state the state of the task * (pending/running/completed/failed/killed) */ public void displayTasks(JobID jobId, String type, String state) throws IOException { TaskReport[] reports = new TaskReport[0]; if (type.equals("map")) { reports = getMapTaskReports(jobId); } else if (type.equals("reduce")) { reports = getReduceTaskReports(jobId); } else if (type.equals("setup")) { reports = getSetupTaskReports(jobId); } else if (type.equals("cleanup")) { reports = getCleanupTaskReports(jobId); } for (TaskReport report : reports) { TIPStatus status = report.getCurrentStatus(); if ((state.equals("pending") && status == TIPStatus.PENDING) || (state.equals("running") && status == TIPStatus.RUNNING) || (state.equals("completed") && status == TIPStatus.COMPLETE) || (state.equals("failed") && status == TIPStatus.FAILED) || (state.equals("killed") && status == TIPStatus.KILLED)) { printTaskAttempts(report); } } } private void printTaskAttempts(TaskReport report) { if (report.getCurrentStatus() == TIPStatus.COMPLETE) { System.out.println(report.getSuccessfulTaskAttempt()); } else if (report.getCurrentStatus() == TIPStatus.RUNNING) { for (TaskAttemptID t : report.getRunningTaskAttempts()) { System.out.println(t); } } } /** * Get status information about the Map-Reduce cluster. * * @return the status information about the Map-Reduce cluster as an object * of {@link ClusterStatus}. * @throws IOException */ public ClusterStatus getClusterStatus() throws IOException { return getClusterStatus(false); } /** * Get status information about the Map-Reduce cluster. * * @param detailed if true then get a detailed status including the * tracker names and memory usage of the JobTracker * @return the status information about the Map-Reduce cluster as an object * of {@link ClusterStatus}. * @throws IOException */ public ClusterStatus getClusterStatus(boolean detailed) throws IOException { return jobSubmitClient.getClusterStatus(detailed); } /** * Grab the jobtracker's view of the staging directory path where * job-specific files will be placed. * * @return the staging directory where job-specific files are to be placed. */ public Path getStagingAreaDir() throws IOException { if (stagingAreaDir == null) { stagingAreaDir = new Path(jobSubmitClient.getStagingAreaDir()); } return stagingAreaDir; } /** * Get the jobs that are not completed and not failed. * * @return array of {@link JobStatus} for the running/to-be-run jobs. * @throws IOException */ public JobStatus[] jobsToComplete() throws IOException { return jobSubmitClient.jobsToComplete(); } private static void downloadProfile(TaskCompletionEvent e) throws IOException { URLConnection connection = new URL( getTaskLogURL(e.getTaskAttemptId(), e.getTaskTrackerHttp()) + "&filter=profile").openConnection(); InputStream in = connection.getInputStream(); OutputStream out = new FileOutputStream(e.getTaskAttemptId() + ".profile"); IOUtils.copyBytes(in, out, 64 * 1024, true); } /** * Get the jobs that are submitted. * * @return array of {@link JobStatus} for the submitted jobs. * @throws IOException */ public JobStatus[] getAllJobs() throws IOException { return jobSubmitClient.getAllJobs(); } /** * Utility that submits a job, then polls for progress until the job is * complete. * * @param job the job configuration. * @throws IOException if the job fails */ public static RunningJob runJob(JobConf job) throws IOException { JobClient jc = new JobClient(job); RunningJob rj = jc.submitJob(job); try { if (!jc.monitorAndPrintJob(job, rj)) { LOG.info("Job Failed: " + rj.getFailureInfo()); throw new IOException("Job failed!"); } } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } return rj; } /** * Monitor a job and print status in real-time as progress is made and tasks * fail. * @param conf the job's configuration * @param job the job to track * @return true if the job succeeded * @throws IOException if communication to the JobTracker fails */ public boolean monitorAndPrintJob(JobConf conf, RunningJob job) throws IOException, InterruptedException { String lastReport = null; TaskStatusFilter filter; filter = getTaskOutputFilter(conf); JobID jobId = job.getID(); LOG.info("Running job: " + jobId); int eventCounter = 0; boolean profiling = conf.getProfileEnabled(); Configuration.IntegerRanges mapRanges = conf.getProfileTaskRange(true); Configuration.IntegerRanges reduceRanges = conf.getProfileTaskRange(false); while (!job.isComplete()) { Thread.sleep(1000); String report = (" map " + StringUtils.formatPercent(job.mapProgress(), 0) + " reduce " + StringUtils.formatPercent(job.reduceProgress(), 0)); if (!report.equals(lastReport)) { LOG.info(report); lastReport = report; } TaskCompletionEvent[] events = job.getTaskCompletionEvents(eventCounter); eventCounter += events.length; for (TaskCompletionEvent event : events) { TaskCompletionEvent.Status status = event.getTaskStatus(); if (profiling && (status == TaskCompletionEvent.Status.SUCCEEDED || status == TaskCompletionEvent.Status.FAILED) && (event.isMap ? mapRanges : reduceRanges).isIncluded(event.idWithinJob())) { downloadProfile(event); } switch (filter) { case NONE: break; case SUCCEEDED: if (event.getTaskStatus() == TaskCompletionEvent.Status.SUCCEEDED) { LOG.info(event.toString()); displayTaskLogs(event.getTaskAttemptId(), event.getTaskTrackerHttp()); } break; case FAILED: if (event.getTaskStatus() == TaskCompletionEvent.Status.FAILED) { LOG.info(event.toString()); // Displaying the task diagnostic information TaskAttemptID taskId = event.getTaskAttemptId(); String[] taskDiagnostics = jobSubmitClient.getTaskDiagnostics(taskId); if (taskDiagnostics != null) { for (String diagnostics : taskDiagnostics) { System.err.println(diagnostics); } } // Displaying the task logs displayTaskLogs(event.getTaskAttemptId(), event.getTaskTrackerHttp()); } break; case KILLED: if (event.getTaskStatus() == TaskCompletionEvent.Status.KILLED) { LOG.info(event.toString()); } break; case ALL: LOG.info(event.toString()); displayTaskLogs(event.getTaskAttemptId(), event.getTaskTrackerHttp()); break; } } } LOG.info("Job complete: " + jobId); Counters counters = null; try { counters = job.getCounters(); } catch (IOException ie) { counters = null; LOG.info(ie.getMessage()); } if (counters != null) { counters.log(LOG); } return job.isSuccessful(); } static String getTaskLogURL(TaskAttemptID taskId, String baseUrl) { return (baseUrl + "/tasklog?plaintext=true&attemptid=" + taskId); } private static void displayTaskLogs(TaskAttemptID taskId, String baseUrl) throws IOException { // The tasktracker for a 'failed/killed' job might not be around... if (baseUrl != null) { // Construct the url for the tasklogs String taskLogUrl = getTaskLogURL(taskId, baseUrl); // Copy tasks's stdout of the JobClient getTaskLogs(taskId, new URL(taskLogUrl + "&filter=stdout"), System.out); // Copy task's stderr to stderr of the JobClient getTaskLogs(taskId, new URL(taskLogUrl + "&filter=stderr"), System.err); } } private static void getTaskLogs(TaskAttemptID taskId, URL taskLogUrl, OutputStream out) { try { URLConnection connection = taskLogUrl.openConnection(); connection.setReadTimeout(tasklogtimeout); connection.setConnectTimeout(tasklogtimeout); BufferedReader input = new BufferedReader(new InputStreamReader(connection.getInputStream())); BufferedWriter output = new BufferedWriter(new OutputStreamWriter(out)); try { String logData = null; while ((logData = input.readLine()) != null) { if (logData.length() > 0) { output.write(taskId + ": " + logData + "\n"); output.flush(); } } } finally { input.close(); } } catch (IOException ioe) { LOG.warn("Error reading task output" + ioe.getMessage()); } } static Configuration getConfiguration(String jobTrackerSpec) { Configuration conf = new Configuration(); if (jobTrackerSpec != null) { if (jobTrackerSpec.indexOf(":") >= 0) { conf.set("mapred.job.tracker", jobTrackerSpec); } else { String classpathFile = "hadoop-" + jobTrackerSpec + ".xml"; URL validate = conf.getResource(classpathFile); if (validate == null) { throw new RuntimeException(classpathFile + " not found on CLASSPATH"); } conf.addResource(classpathFile); } } return conf; } /** * Sets the output filter for tasks. only those tasks are printed whose * output matches the filter. * @param newValue task filter. */ @Deprecated public void setTaskOutputFilter(TaskStatusFilter newValue) { this.taskOutputFilter = newValue; } /** * Get the task output filter out of the JobConf. * * @param job the JobConf to examine. * @return the filter level. */ public static TaskStatusFilter getTaskOutputFilter(JobConf job) { return TaskStatusFilter.valueOf(job.get("jobclient.output.filter", "FAILED")); } /** * Modify the JobConf to set the task output filter. * * @param job the JobConf to modify. * @param newValue the value to set. */ public static void setTaskOutputFilter(JobConf job, TaskStatusFilter newValue) { job.set("jobclient.output.filter", newValue.toString()); } /** * Returns task output filter. * @return task filter. */ @Deprecated public TaskStatusFilter getTaskOutputFilter() { return this.taskOutputFilter; } private String getJobPriorityNames() { StringBuffer sb = new StringBuffer(); for (JobPriority p : JobPriority.values()) { sb.append(p.name()).append(" "); } return sb.substring(0, sb.length() - 1); } /** * Display usage of the command-line tool and terminate execution */ private void displayUsage(String cmd) { String prefix = "Usage: JobClient "; String jobPriorityValues = getJobPriorityNames(); String taskTypes = "map, reduce, setup, cleanup"; String taskStates = "running, completed"; if ("-submit".equals(cmd)) { System.err.println(prefix + "[" + cmd + " <job-file>]"); } else if ("-status".equals(cmd) || "-kill".equals(cmd)) { System.err.println(prefix + "[" + cmd + " <job-id>]"); } else if ("-counter".equals(cmd)) { System.err.println(prefix + "[" + cmd + " <job-id> <group-name> <counter-name>]"); } else if ("-events".equals(cmd)) { System.err.println(prefix + "[" + cmd + " <job-id> <from-event-#> <#-of-events>]"); } else if ("-history".equals(cmd)) { System.err.println(prefix + "[" + cmd + " <jobOutputDir>]"); } else if ("-list".equals(cmd)) { System.err.println(prefix + "[" + cmd + " [all]]"); } else if ("-kill-task".equals(cmd) || "-fail-task".equals(cmd)) { System.err.println(prefix + "[" + cmd + " <task-id>]"); } else if ("-set-priority".equals(cmd)) { System.err.println(prefix + "[" + cmd + " <job-id> <priority>]. " + "Valid values for priorities are: " + jobPriorityValues); } else if ("-list-active-trackers".equals(cmd)) { System.err.println(prefix + "[" + cmd + "]"); } else if ("-list-blacklisted-trackers".equals(cmd)) { System.err.println(prefix + "[" + cmd + "]"); } else if ("-list-attempt-ids".equals(cmd)) { System.err.println(prefix + "[" + cmd + " <job-id> <task-type> <task-state>]. " + "Valid values for <task-type> are " + taskTypes + ". " + "Valid values for <task-state> are " + taskStates); } else { System.err.printf(prefix + "<command> <args>\n"); System.err.printf("\t[-submit <job-file>]\n"); System.err.printf("\t[-status <job-id>]\n"); System.err.printf("\t[-counter <job-id> <group-name> <counter-name>]\n"); System.err.printf("\t[-kill <job-id>]\n"); System.err.printf("\t[-set-priority <job-id> <priority>]. " + "Valid values for priorities are: " + jobPriorityValues + "\n"); System.err.printf("\t[-events <job-id> <from-event-#> <#-of-events>]\n"); System.err.printf("\t[-history <jobOutputDir>]\n"); System.err.printf("\t[-list [all]]\n"); System.err.printf("\t[-list-active-trackers]\n"); System.err.printf("\t[-list-blacklisted-trackers]\n"); System.err.println("\t[-list-attempt-ids <job-id> <task-type> " + "<task-state>]\n"); System.err.printf("\t[-kill-task <task-id>]\n"); System.err.printf("\t[-fail-task <task-id>]\n\n"); ToolRunner.printGenericCommandUsage(System.out); } } public int run(String[] argv) throws Exception { int exitCode = -1; if (argv.length < 1) { displayUsage(""); return exitCode; } // process arguments String cmd = argv[0]; String submitJobFile = null; String jobid = null; String taskid = null; String outputDir = null; String counterGroupName = null; String counterName = null; String newPriority = null; String taskType = null; String taskState = null; int fromEvent = 0; int nEvents = 0; boolean getStatus = false; boolean getCounter = false; boolean killJob = false; boolean listEvents = false; boolean viewHistory = false; boolean viewAllHistory = false; boolean listJobs = false; boolean listAllJobs = false; boolean listActiveTrackers = false; boolean listBlacklistedTrackers = false; boolean displayTasks = false; boolean killTask = false; boolean failTask = false; boolean setJobPriority = false; if ("-submit".equals(cmd)) { if (argv.length != 2) { displayUsage(cmd); return exitCode; } submitJobFile = argv[1]; } else if ("-status".equals(cmd)) { if (argv.length != 2) { displayUsage(cmd); return exitCode; } jobid = argv[1]; getStatus = true; } else if ("-counter".equals(cmd)) { if (argv.length != 4) { displayUsage(cmd); return exitCode; } getCounter = true; jobid = argv[1]; counterGroupName = argv[2]; counterName = argv[3]; } else if ("-kill".equals(cmd)) { if (argv.length != 2) { displayUsage(cmd); return exitCode; } jobid = argv[1]; killJob = true; } else if ("-set-priority".equals(cmd)) { if (argv.length != 3) { displayUsage(cmd); return exitCode; } jobid = argv[1]; newPriority = argv[2]; try { JobPriority jp = JobPriority.valueOf(newPriority); } catch (IllegalArgumentException iae) { displayUsage(cmd); return exitCode; } setJobPriority = true; } else if ("-events".equals(cmd)) { if (argv.length != 4) { displayUsage(cmd); return exitCode; } jobid = argv[1]; fromEvent = Integer.parseInt(argv[2]); nEvents = Integer.parseInt(argv[3]); listEvents = true; } else if ("-history".equals(cmd)) { if (argv.length != 2 && !(argv.length == 3 && "all".equals(argv[1]))) { displayUsage(cmd); return exitCode; } viewHistory = true; if (argv.length == 3 && "all".equals(argv[1])) { viewAllHistory = true; outputDir = argv[2]; } else { outputDir = argv[1]; } } else if ("-list".equals(cmd)) { if (argv.length != 1 && !(argv.length == 2 && "all".equals(argv[1]))) { displayUsage(cmd); return exitCode; } if (argv.length == 2 && "all".equals(argv[1])) { listAllJobs = true; } else { listJobs = true; } } else if ("-kill-task".equals(cmd)) { if (argv.length != 2) { displayUsage(cmd); return exitCode; } killTask = true; taskid = argv[1]; } else if ("-fail-task".equals(cmd)) { if (argv.length != 2) { displayUsage(cmd); return exitCode; } failTask = true; taskid = argv[1]; } else if ("-list-active-trackers".equals(cmd)) { if (argv.length != 1) { displayUsage(cmd); return exitCode; } listActiveTrackers = true; } else if ("-list-blacklisted-trackers".equals(cmd)) { if (argv.length != 1) { displayUsage(cmd); return exitCode; } listBlacklistedTrackers = true; } else if ("-list-attempt-ids".equals(cmd)) { if (argv.length != 4) { displayUsage(cmd); return exitCode; } jobid = argv[1]; taskType = argv[2]; taskState = argv[3]; displayTasks = true; } else { displayUsage(cmd); return exitCode; } // initialize JobClient JobConf conf = null; if (submitJobFile != null) { conf = new JobConf(submitJobFile); } else { conf = new JobConf(getConf()); } init(conf); // Submit the request try { if (submitJobFile != null) { RunningJob job = submitJob(conf); System.out.println("Created job " + job.getID()); exitCode = 0; } else if (getStatus) { RunningJob job = getJob(JobID.forName(jobid)); if (job == null) { System.out.println("Could not find job " + jobid); } else { Throwable counterException = null; Counters counters = null; try { counters = job.getCounters(); } catch (IOException e) { counterException = e; } System.out.println(); System.out.println(job); if (counters != null) { System.out.println(counters); } else { if (counterException != null) { System.out.println("Error fetching counters: " + counterException.getMessage()); } else { System.out.println("Counters not available. Job is retired."); } } exitCode = 0; } } else if (getCounter) { RunningJob job = getJob(JobID.forName(jobid)); if (job == null) { System.out.println("Could not find job " + jobid); } else { Throwable counterException = null; Counters counters = null; try { counters = job.getCounters(); } catch (IOException e) { counterException = e; } if (counters == null) { if (counterException != null) { System.out.println("Error fetching counters: " + counterException.getMessage()); } else { System.out.println("Counters not available for retired job " + jobid); } exitCode = -1; } else { Group group = counters.getGroup(counterGroupName); Counter counter = group.getCounterForName(counterName); System.out.println(counter.getCounter()); exitCode = 0; } } } else if (killJob) { RunningJob job = getJob(JobID.forName(jobid)); if (job == null) { System.out.println("Could not find job " + jobid); } else { job.killJob(); System.out.println("Killed job " + jobid); exitCode = 0; } } else if (setJobPriority) { RunningJob job = getJob(JobID.forName(jobid)); if (job == null) { System.out.println("Could not find job " + jobid); } else { job.setJobPriority(newPriority); System.out.println("Changed job priority."); exitCode = 0; } } else if (viewHistory) { viewHistory(outputDir, viewAllHistory); exitCode = 0; } else if (listEvents) { listEvents(JobID.forName(jobid), fromEvent, nEvents); exitCode = 0; } else if (listJobs) { listJobs(); exitCode = 0; } else if (listAllJobs) { listAllJobs(); exitCode = 0; } else if (listActiveTrackers) { listActiveTrackers(); exitCode = 0; } else if (listBlacklistedTrackers) { listBlacklistedTrackers(); exitCode = 0; } else if (displayTasks) { displayTasks(JobID.forName(jobid), taskType, taskState); } else if (killTask) { if (jobSubmitClient.killTask(TaskAttemptID.forName(taskid), false)) { System.out.println("Killed task " + taskid); exitCode = 0; } else { System.out.println("Could not kill task " + taskid); exitCode = -1; } } else if (failTask) { if (jobSubmitClient.killTask(TaskAttemptID.forName(taskid), true)) { System.out.println("Killed task " + taskid + " by failing it"); exitCode = 0; } else { System.out.println("Could not fail task " + taskid); exitCode = -1; } } } catch (RemoteException re) { IOException unwrappedException = re.unwrapRemoteException(); if (unwrappedException instanceof AccessControlException) { System.out.println(unwrappedException.getMessage()); } else { throw re; } } finally { close(); } return exitCode; } private void viewHistory(String outputDir, boolean all) throws IOException { HistoryViewer historyViewer = new HistoryViewer(outputDir, getConf(), all); historyViewer.print(); } /** * List the events for the given job * @param jobId the job id for the job's events to list * @throws IOException */ private void listEvents(JobID jobId, int fromEventId, int numEvents) throws IOException { TaskCompletionEvent[] events = jobSubmitClient.getTaskCompletionEvents(jobId, fromEventId, numEvents); System.out.println("Task completion events for " + jobId); System.out.println("Number of events (from " + fromEventId + ") are: " + events.length); for (TaskCompletionEvent event : events) { System.out.println(event.getTaskStatus() + " " + event.getTaskAttemptId() + " " + getTaskLogURL(event.getTaskAttemptId(), event.getTaskTrackerHttp())); } } /** * Dump a list of currently running jobs * @throws IOException */ private void listJobs() throws IOException { JobStatus[] jobs = jobsToComplete(); if (jobs == null) jobs = new JobStatus[0]; System.out.printf("%d jobs currently running\n", jobs.length); displayJobList(jobs); } /** * Dump a list of all jobs submitted. * @throws IOException */ private void listAllJobs() throws IOException { JobStatus[] jobs = getAllJobs(); if (jobs == null) jobs = new JobStatus[0]; System.out.printf("%d jobs submitted\n", jobs.length); System.out.printf("States are:\n\tRunning : 1\tSucceded : 2" + "\tFailed : 3\tPrep : 4\n"); displayJobList(jobs); } /** * Display the list of active trackers */ private void listActiveTrackers() throws IOException { ClusterStatus c = jobSubmitClient.getClusterStatus(true); Collection<String> trackers = c.getActiveTrackerNames(); for (String trackerName : trackers) { System.out.println(trackerName); } } /** * Display the list of blacklisted trackers */ private void listBlacklistedTrackers() throws IOException { ClusterStatus c = jobSubmitClient.getClusterStatus(true); Collection<String> trackers = c.getBlacklistedTrackerNames(); for (String trackerName : trackers) { System.out.println(trackerName); } } void displayJobList(JobStatus[] jobs) { System.out.printf("JobId\tState\tStartTime\tUserName\tPriority\tSchedulingInfo\n"); for (JobStatus job : jobs) { System.out.printf("%s\t%d\t%d\t%s\t%s\t%s\n", job.getJobID(), job.getRunState(), job.getStartTime(), job.getUsername(), job.getJobPriority().name(), job.getSchedulingInfo()); } } /** * Get status information about the max available Maps in the cluster. * * @return the max available Maps in the cluster * @throws IOException */ public int getDefaultMaps() throws IOException { return getClusterStatus().getMaxMapTasks(); } /** * Get status information about the max available Reduces in the cluster. * * @return the max available Reduces in the cluster * @throws IOException */ public int getDefaultReduces() throws IOException { return getClusterStatus().getMaxReduceTasks(); } /** * Grab the jobtracker system directory path where job-specific files are to be placed. * * @return the system directory where job-specific files are to be placed. */ public Path getSystemDir() { if (sysDir == null) { sysDir = new Path(jobSubmitClient.getSystemDir()); } return sysDir; } /** * Return an array of queue information objects about all the Job Queues * configured. * * @return Array of JobQueueInfo objects * @throws IOException */ public JobQueueInfo[] getQueues() throws IOException { return jobSubmitClient.getQueues(); } /** * Gets all the jobs which were added to particular Job Queue * * @param queueName name of the Job Queue * @return Array of jobs present in the job queue * @throws IOException */ public JobStatus[] getJobsFromQueue(String queueName) throws IOException { return jobSubmitClient.getJobsFromQueue(queueName); } /** * Gets the queue information associated to a particular Job Queue * * @param queueName name of the job queue. * @return Queue information associated to particular queue. * @throws IOException */ public JobQueueInfo getQueueInfo(String queueName) throws IOException { return jobSubmitClient.getQueueInfo(queueName); } /** * Gets the Queue ACLs for current user * @return array of QueueAclsInfo object for current user. * @throws IOException */ public QueueAclsInfo[] getQueueAclsForCurrentUser() throws IOException { return jobSubmitClient.getQueueAclsForCurrentUser(); } /* Get a delegation token for the user from the JobTracker. * @param renewer the user who can renew the token * @return the new token * @throws IOException */ public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer) throws IOException, InterruptedException { Token<DelegationTokenIdentifier> result = jobSubmitClient.getDelegationToken(renewer); SecurityUtil.setTokenService(result, JobTracker.getAddress(getConf())); return result; } /** * Renew a delegation token * @param token the token to renew * @return the new expiration time * @throws InvalidToken * @throws IOException */ public long renewDelegationToken(Token<DelegationTokenIdentifier> token) throws InvalidToken, IOException, InterruptedException { try { return jobSubmitClient.renewDelegationToken(token); } catch (RemoteException re) { throw re.unwrapRemoteException(InvalidToken.class, AccessControlException.class); } } /** * Cancel a delegation token from the JobTracker * @param token the token to cancel * @throws IOException */ public void cancelDelegationToken(Token<DelegationTokenIdentifier> token) throws IOException, InterruptedException { try { jobSubmitClient.cancelDelegationToken(token); } catch (RemoteException re) { throw re.unwrapRemoteException(InvalidToken.class, AccessControlException.class); } } /** */ public static void main(String argv[]) throws Exception { int res = ToolRunner.run(new JobClient(), argv); System.exit(res); } @SuppressWarnings("unchecked") private void readTokensFromFiles(Configuration conf, Credentials credentials) throws IOException { // add tokens and secrets coming from a token storage file String binaryTokenFilename = conf.get("mapreduce.job.credentials.binary"); if (binaryTokenFilename != null) { Credentials binary = Credentials.readTokenStorageFile(new Path("file:///" + binaryTokenFilename), conf); credentials.addAll(binary); } // add secret keys coming from a json file String tokensFileName = conf.get("mapreduce.job.credentials.json"); if (tokensFileName != null) { LOG.info("loading user's secret keys from " + tokensFileName); String localFileName = new Path(tokensFileName).toUri().getPath(); boolean json_error = false; try { // read JSON ObjectMapper mapper = new ObjectMapper(); Map<String, String> nm = mapper.readValue(new File(localFileName), Map.class); for (Map.Entry<String, String> ent : nm.entrySet()) { credentials.addSecretKey(new Text(ent.getKey()), ent.getValue().getBytes()); } } catch (JsonMappingException e) { json_error = true; } catch (JsonParseException e) { json_error = true; } if (json_error) LOG.warn("couldn't parse Token Cache JSON file with user secret keys"); } } //get secret keys and tokens and store them into TokenCache @SuppressWarnings("unchecked") private void populateTokenCache(Configuration conf, Credentials credentials) throws IOException { readTokensFromFiles(conf, credentials); // add the delegation tokens from configuration String[] nameNodes = conf.getStrings(JobContext.JOB_NAMENODES); LOG.debug("adding the following namenodes' delegation tokens:" + Arrays.toString(nameNodes)); if (nameNodes != null) { Path[] ps = new Path[nameNodes.length]; for (int i = 0; i < nameNodes.length; i++) { ps[i] = new Path(nameNodes[i]); } TokenCache.obtainTokensForNamenodes(credentials, ps, conf); } } }