Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapred; import java.io.FileNotFoundException; import java.io.IOException; import java.net.InetSocketAddress; import java.net.URL; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collection; import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.ClusterStatus.BlackListInfo; import org.apache.hadoop.mapreduce.Cluster; import org.apache.hadoop.mapreduce.ClusterMetrics; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.QueueInfo; import org.apache.hadoop.mapreduce.TaskTrackerInfo; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.mapreduce.tools.CLI; import org.apache.hadoop.mapreduce.util.ConfigUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenRenewer; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; /** * <code>JobClient</code> is the primary interface for the user-job to interact * with the cluster. * * <code>JobClient</code> provides facilities to submit jobs, track their * progress, access component-tasks' reports/logs, get the Map-Reduce cluster * status information etc. * * <p>The job submission process involves: * <ol> * <li> * Checking the input and output specifications of the job. * </li> * <li> * Computing the {@link InputSplit}s for the job. * </li> * <li> * Setup the requisite accounting information for the {@link DistributedCache} * of the job, if necessary. * </li> * <li> * Copying the job's jar and configuration to the map-reduce system directory * on the distributed file-system. * </li> * <li> * Submitting the job to the cluster and optionally monitoring * it's status. * </li> * </ol> * * Normally the user creates the application, describes various facets of the * job via {@link JobConf} and then uses the <code>JobClient</code> to submit * the job and monitor its progress. * * <p>Here is an example on how to use <code>JobClient</code>:</p> * <p><blockquote><pre> * // Create a new JobConf * JobConf job = new JobConf(new Configuration(), MyJob.class); * * // Specify various job-specific parameters * job.setJobName("myjob"); * * job.setInputPath(new Path("in")); * job.setOutputPath(new Path("out")); * * job.setMapperClass(MyJob.MyMapper.class); * job.setReducerClass(MyJob.MyReducer.class); * * // Submit the job, then poll for progress until the job is complete * JobClient.runJob(job); * </pre></blockquote> * * <b id="JobControl">Job Control</b> * * <p>At times clients would chain map-reduce jobs to accomplish complex tasks * which cannot be done via a single map-reduce job. This is fairly easy since * the output of the job, typically, goes to distributed file-system and that * can be used as the input for the next job.</p> * * <p>However, this also means that the onus on ensuring jobs are complete * (success/failure) lies squarely on the clients. In such situations the * various job-control options are: * <ol> * <li> * {@link #runJob(JobConf)} : submits the job and returns only after * the job has completed. * </li> * <li> * {@link #submitJob(JobConf)} : only submits the job, then poll the * returned handle to the {@link RunningJob} to query status and make * scheduling decisions. * </li> * <li> * {@link JobConf#setJobEndNotificationURI(String)} : setup a notification * on job-completion, thus avoiding polling. * </li> * </ol> * * @see JobConf * @see ClusterStatus * @see Tool * @see DistributedCache */ @InterfaceAudience.Public @InterfaceStability.Stable public class JobClient extends CLI implements AutoCloseable { @InterfaceAudience.Private public static final String MAPREDUCE_CLIENT_RETRY_POLICY_ENABLED_KEY = "mapreduce.jobclient.retry.policy.enabled"; @InterfaceAudience.Private public static final boolean MAPREDUCE_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = false; @InterfaceAudience.Private public static final String MAPREDUCE_CLIENT_RETRY_POLICY_SPEC_KEY = "mapreduce.jobclient.retry.policy.spec"; @InterfaceAudience.Private public static final String MAPREDUCE_CLIENT_RETRY_POLICY_SPEC_DEFAULT = "10000,6,60000,10"; // t1,n1,t2,n2,... public enum TaskStatusFilter { NONE, KILLED, FAILED, SUCCEEDED, ALL } private TaskStatusFilter taskOutputFilter = TaskStatusFilter.FAILED; private int maxRetry = MRJobConfig.DEFAULT_MR_CLIENT_JOB_MAX_RETRIES; private long retryInterval = MRJobConfig.DEFAULT_MR_CLIENT_JOB_RETRY_INTERVAL; static { ConfigUtil.loadResources(); } /** * A NetworkedJob is an implementation of RunningJob. It holds * a JobProfile object to provide some info, and interacts with the * remote service to provide certain functionality. */ static class NetworkedJob implements RunningJob { Job job; /** * We store a JobProfile and a timestamp for when we last * acquired the job profile. If the job is null, then we cannot * perform any of the tasks. The job might be null if the cluster * has completely forgotten about the job. (eg, 24 hours after the * job completes.) */ public NetworkedJob(JobStatus status, Cluster cluster) throws IOException { this(status, cluster, new JobConf(status.getJobFile())); } private NetworkedJob(JobStatus status, Cluster cluster, JobConf conf) throws IOException { this(Job.getInstance(cluster, status, conf)); } public NetworkedJob(Job job) throws IOException { this.job = job; } public Configuration getConfiguration() { return job.getConfiguration(); } /** * An identifier for the job */ public JobID getID() { return JobID.downgrade(job.getJobID()); } /** @deprecated This method is deprecated and will be removed. Applications should * rather use {@link #getID()}.*/ @Deprecated public String getJobID() { return getID().toString(); } /** * The user-specified job name */ public String getJobName() { return job.getJobName(); } /** * The name of the job file */ public String getJobFile() { return job.getJobFile(); } /** * A URL where the job's status can be seen */ public String getTrackingURL() { return job.getTrackingURL(); } /** * A float between 0.0 and 1.0, indicating the % of map work * completed. */ public float mapProgress() throws IOException { return job.mapProgress(); } /** * A float between 0.0 and 1.0, indicating the % of reduce work * completed. */ public float reduceProgress() throws IOException { return job.reduceProgress(); } /** * A float between 0.0 and 1.0, indicating the % of cleanup work * completed. */ public float cleanupProgress() throws IOException { try { return job.cleanupProgress(); } catch (InterruptedException ie) { throw new IOException(ie); } } /** * A float between 0.0 and 1.0, indicating the % of setup work * completed. */ public float setupProgress() throws IOException { return job.setupProgress(); } /** * Returns immediately whether the whole job is done yet or not. */ public synchronized boolean isComplete() throws IOException { return job.isComplete(); } /** * True iff job completed successfully. */ public synchronized boolean isSuccessful() throws IOException { return job.isSuccessful(); } /** * Blocks until the job is finished */ public void waitForCompletion() throws IOException { try { job.waitForCompletion(false); } catch (InterruptedException ie) { throw new IOException(ie); } catch (ClassNotFoundException ce) { throw new IOException(ce); } } /** * Tells the service to get the state of the current job. */ public synchronized int getJobState() throws IOException { try { return job.getJobState().getValue(); } catch (InterruptedException ie) { throw new IOException(ie); } } /** * Tells the service to terminate the current job. */ public synchronized void killJob() throws IOException { job.killJob(); } /** Set the priority of the job. * @param priority new priority of the job. */ public synchronized void setJobPriority(String priority) throws IOException { try { job.setPriority(org.apache.hadoop.mapreduce.JobPriority.valueOf(priority)); } catch (InterruptedException ie) { throw new IOException(ie); } } /** * Kill indicated task attempt. * @param taskId the id of the task to kill. * @param shouldFail if true the task is failed and added to failed tasks list, otherwise * it is just killed, w/o affecting job failure status. */ public synchronized void killTask(TaskAttemptID taskId, boolean shouldFail) throws IOException { if (shouldFail) { job.failTask(taskId); } else { job.killTask(taskId); } } /** @deprecated Applications should rather use {@link #killTask(TaskAttemptID, boolean)}*/ @Deprecated public synchronized void killTask(String taskId, boolean shouldFail) throws IOException { killTask(TaskAttemptID.forName(taskId), shouldFail); } /** * Fetch task completion events from cluster for this job. */ public synchronized TaskCompletionEvent[] getTaskCompletionEvents(int startFrom) throws IOException { try { org.apache.hadoop.mapreduce.TaskCompletionEvent[] acls = job.getTaskCompletionEvents(startFrom, 10); TaskCompletionEvent[] ret = new TaskCompletionEvent[acls.length]; for (int i = 0; i < acls.length; i++) { ret[i] = TaskCompletionEvent.downgrade(acls[i]); } return ret; } catch (InterruptedException ie) { throw new IOException(ie); } } /** * Dump stats to screen */ @Override public String toString() { return job.toString(); } /** * Returns the counters for this job */ public Counters getCounters() throws IOException { Counters result = null; org.apache.hadoop.mapreduce.Counters temp = job.getCounters(); if (temp != null) { result = Counters.downgrade(temp); } return result; } @Override public String[] getTaskDiagnostics(TaskAttemptID id) throws IOException { try { return job.getTaskDiagnostics(id); } catch (InterruptedException ie) { throw new IOException(ie); } } public String getHistoryUrl() throws IOException { try { return job.getHistoryUrl(); } catch (InterruptedException ie) { throw new IOException(ie); } } public boolean isRetired() throws IOException { try { return job.isRetired(); } catch (InterruptedException ie) { throw new IOException(ie); } } boolean monitorAndPrintJob() throws IOException, InterruptedException { return job.monitorAndPrintJob(); } @Override public String getFailureInfo() throws IOException { try { return job.getStatus().getFailureInfo(); } catch (InterruptedException ie) { throw new IOException(ie); } } @Override public JobStatus getJobStatus() throws IOException { try { return JobStatus.downgrade(job.getStatus()); } catch (InterruptedException ie) { throw new IOException(ie); } } } /** * Ugi of the client. We store this ugi when the client is created and * then make sure that the same ugi is used to run the various protocols. */ UserGroupInformation clientUgi; /** * Create a job client. */ public JobClient() { } /** * Build a job client with the given {@link JobConf}, and connect to the * default cluster * * @param conf the job configuration. * @throws IOException */ public JobClient(JobConf conf) throws IOException { init(conf); } /** * Build a job client with the given {@link Configuration}, * and connect to the default cluster * * @param conf the configuration. * @throws IOException */ public JobClient(Configuration conf) throws IOException { init(new JobConf(conf)); } /** * Connect to the default cluster * @param conf the job configuration. * @throws IOException */ public void init(JobConf conf) throws IOException { setConf(conf); cluster = new Cluster(conf); clientUgi = UserGroupInformation.getCurrentUser(); maxRetry = conf.getInt(MRJobConfig.MR_CLIENT_JOB_MAX_RETRIES, MRJobConfig.DEFAULT_MR_CLIENT_JOB_MAX_RETRIES); retryInterval = conf.getLong(MRJobConfig.MR_CLIENT_JOB_RETRY_INTERVAL, MRJobConfig.DEFAULT_MR_CLIENT_JOB_RETRY_INTERVAL); } /** * Build a job client, connect to the indicated job tracker. * * @param jobTrackAddr the job tracker to connect to. * @param conf configuration. */ public JobClient(InetSocketAddress jobTrackAddr, Configuration conf) throws IOException { cluster = new Cluster(jobTrackAddr, conf); clientUgi = UserGroupInformation.getCurrentUser(); } /** * Close the <code>JobClient</code>. */ @Override public synchronized void close() throws IOException { cluster.close(); } /** * Get a filesystem handle. We need this to prepare jobs * for submission to the MapReduce system. * * @return the filesystem handle. */ public synchronized FileSystem getFs() throws IOException { try { return cluster.getFileSystem(); } catch (InterruptedException ie) { throw new IOException(ie); } } /** * Get a handle to the Cluster */ public Cluster getClusterHandle() { return cluster; } /** * Submit a job to the MR system. * * This returns a handle to the {@link RunningJob} which can be used to track * the running-job. * * @param jobFile the job configuration. * @return a handle to the {@link RunningJob} which can be used to track the * running-job. * @throws FileNotFoundException * @throws InvalidJobConfException * @throws IOException */ public RunningJob submitJob(String jobFile) throws FileNotFoundException, InvalidJobConfException, IOException { // Load in the submitted job details JobConf job = new JobConf(jobFile); return submitJob(job); } /** * Submit a job to the MR system. * This returns a handle to the {@link RunningJob} which can be used to track * the running-job. * * @param conf the job configuration. * @return a handle to the {@link RunningJob} which can be used to track the * running-job. * @throws FileNotFoundException * @throws IOException */ public RunningJob submitJob(final JobConf conf) throws FileNotFoundException, IOException { return submitJobInternal(conf); } @InterfaceAudience.Private public RunningJob submitJobInternal(final JobConf conf) throws FileNotFoundException, IOException { try { conf.setBooleanIfUnset("mapred.mapper.new-api", false); conf.setBooleanIfUnset("mapred.reducer.new-api", false); Job job = clientUgi.doAs(new PrivilegedExceptionAction<Job>() { @Override public Job run() throws IOException, ClassNotFoundException, InterruptedException { Job job = Job.getInstance(conf); job.submit(); return job; } }); Cluster prev = cluster; // update our Cluster instance with the one created by Job for submission // (we can't pass our Cluster instance to Job, since Job wraps the config // instance, and the two configs would then diverge) cluster = job.getCluster(); // It is important to close the previous cluster instance // to cleanup resources. if (prev != null) { prev.close(); } return new NetworkedJob(job); } catch (InterruptedException ie) { throw new IOException("interrupted", ie); } } private Job getJobUsingCluster(final JobID jobid) throws IOException, InterruptedException { return clientUgi.doAs(new PrivilegedExceptionAction<Job>() { public Job run() throws IOException, InterruptedException { return cluster.getJob(jobid); } }); } protected RunningJob getJobInner(final JobID jobid) throws IOException { try { Job job = getJobUsingCluster(jobid); if (job != null) { JobStatus status = JobStatus.downgrade(job.getStatus()); if (status != null) { return new NetworkedJob(status, cluster, new JobConf(job.getConfiguration())); } } } catch (InterruptedException ie) { throw new IOException(ie); } return null; } /** * Get an {@link RunningJob} object to track an ongoing job. Returns * null if the id does not correspond to any known job. * * @param jobid the jobid of the job. * @return the {@link RunningJob} handle to track the job, null if the * <code>jobid</code> doesn't correspond to any known job. * @throws IOException */ public RunningJob getJob(final JobID jobid) throws IOException { for (int i = 0; i <= maxRetry; i++) { if (i > 0) { try { Thread.sleep(retryInterval); } catch (Exception e) { } } RunningJob job = getJobInner(jobid); if (job != null) { return job; } } return null; } /**@deprecated Applications should rather use {@link #getJob(JobID)}. */ @Deprecated public RunningJob getJob(String jobid) throws IOException { return getJob(JobID.forName(jobid)); } private static final TaskReport[] EMPTY_TASK_REPORTS = new TaskReport[0]; /** * Get the information of the current state of the map tasks of a job. * * @param jobId the job to query. * @return the list of all of the map tips. * @throws IOException */ public TaskReport[] getMapTaskReports(JobID jobId) throws IOException { return getTaskReports(jobId, TaskType.MAP); } private TaskReport[] getTaskReports(final JobID jobId, TaskType type) throws IOException { try { Job j = getJobUsingCluster(jobId); if (j == null) { return EMPTY_TASK_REPORTS; } return TaskReport.downgradeArray(j.getTaskReports(type)); } catch (InterruptedException ie) { throw new IOException(ie); } } /**@deprecated Applications should rather use {@link #getMapTaskReports(JobID)}*/ @Deprecated public TaskReport[] getMapTaskReports(String jobId) throws IOException { return getMapTaskReports(JobID.forName(jobId)); } /** * Get the information of the current state of the reduce tasks of a job. * * @param jobId the job to query. * @return the list of all of the reduce tips. * @throws IOException */ public TaskReport[] getReduceTaskReports(JobID jobId) throws IOException { return getTaskReports(jobId, TaskType.REDUCE); } /** * Get the information of the current state of the cleanup tasks of a job. * * @param jobId the job to query. * @return the list of all of the cleanup tips. * @throws IOException */ public TaskReport[] getCleanupTaskReports(JobID jobId) throws IOException { return getTaskReports(jobId, TaskType.JOB_CLEANUP); } /** * Get the information of the current state of the setup tasks of a job. * * @param jobId the job to query. * @return the list of all of the setup tips. * @throws IOException */ public TaskReport[] getSetupTaskReports(JobID jobId) throws IOException { return getTaskReports(jobId, TaskType.JOB_SETUP); } /**@deprecated Applications should rather use {@link #getReduceTaskReports(JobID)}*/ @Deprecated public TaskReport[] getReduceTaskReports(String jobId) throws IOException { return getReduceTaskReports(JobID.forName(jobId)); } /** * Display the information about a job's tasks, of a particular type and * in a particular state * * @param jobId the ID of the job * @param type the type of the task (map/reduce/setup/cleanup) * @param state the state of the task * (pending/running/completed/failed/killed) * @throws IOException when there is an error communicating with the master * @throws IllegalArgumentException if an invalid type/state is passed */ public void displayTasks(final JobID jobId, String type, String state) throws IOException { try { Job job = getJobUsingCluster(jobId); super.displayTasks(job, type, state); } catch (InterruptedException ie) { throw new IOException(ie); } } /** * Get status information about the Map-Reduce cluster. * * @return the status information about the Map-Reduce cluster as an object * of {@link ClusterStatus}. * @throws IOException */ public ClusterStatus getClusterStatus() throws IOException { try { return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() { public ClusterStatus run() throws IOException, InterruptedException { ClusterMetrics metrics = cluster.getClusterStatus(); return new ClusterStatus(metrics.getTaskTrackerCount(), metrics.getBlackListedTaskTrackerCount(), cluster.getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(), metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(), metrics.getReduceSlotCapacity(), cluster.getJobTrackerStatus(), metrics.getDecommissionedTaskTrackerCount(), metrics.getGrayListedTaskTrackerCount()); } }); } catch (InterruptedException ie) { throw new IOException(ie); } } private Collection<String> arrayToStringList(TaskTrackerInfo[] objs) { Collection<String> list = new ArrayList<String>(); for (TaskTrackerInfo info : objs) { list.add(info.getTaskTrackerName()); } return list; } private Collection<BlackListInfo> arrayToBlackListInfo(TaskTrackerInfo[] objs) { Collection<BlackListInfo> list = new ArrayList<BlackListInfo>(); for (TaskTrackerInfo info : objs) { BlackListInfo binfo = new BlackListInfo(); binfo.setTrackerName(info.getTaskTrackerName()); binfo.setReasonForBlackListing(info.getReasonForBlacklist()); binfo.setBlackListReport(info.getBlacklistReport()); list.add(binfo); } return list; } /** * Get status information about the Map-Reduce cluster. * * @param detailed if true then get a detailed status including the * tracker names * @return the status information about the Map-Reduce cluster as an object * of {@link ClusterStatus}. * @throws IOException */ public ClusterStatus getClusterStatus(boolean detailed) throws IOException { try { return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() { public ClusterStatus run() throws IOException, InterruptedException { ClusterMetrics metrics = cluster.getClusterStatus(); return new ClusterStatus(arrayToStringList(cluster.getActiveTaskTrackers()), arrayToBlackListInfo(cluster.getBlackListedTaskTrackers()), cluster.getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(), metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(), metrics.getReduceSlotCapacity(), cluster.getJobTrackerStatus()); } }); } catch (InterruptedException ie) { throw new IOException(ie); } } /** * Get the jobs that are not completed and not failed. * * @return array of {@link JobStatus} for the running/to-be-run jobs. * @throws IOException */ public JobStatus[] jobsToComplete() throws IOException { List<JobStatus> stats = new ArrayList<JobStatus>(); for (JobStatus stat : getAllJobs()) { if (!stat.isJobComplete()) { stats.add(stat); } } return stats.toArray(new JobStatus[0]); } /** * Get the jobs that are submitted. * * @return array of {@link JobStatus} for the submitted jobs. * @throws IOException */ public JobStatus[] getAllJobs() throws IOException { try { org.apache.hadoop.mapreduce.JobStatus[] jobs = clientUgi .doAs(new PrivilegedExceptionAction<org.apache.hadoop.mapreduce.JobStatus[]>() { public org.apache.hadoop.mapreduce.JobStatus[] run() throws IOException, InterruptedException { return cluster.getAllJobStatuses(); } }); JobStatus[] stats = new JobStatus[jobs.length]; for (int i = 0; i < jobs.length; i++) { stats[i] = JobStatus.downgrade(jobs[i]); } return stats; } catch (InterruptedException ie) { throw new IOException(ie); } } /** * Utility that submits a job, then polls for progress until the job is * complete. * * @param job the job configuration. * @throws IOException if the job fails */ public static RunningJob runJob(JobConf job) throws IOException { JobClient jc = new JobClient(job); RunningJob rj = jc.submitJob(job); try { if (!jc.monitorAndPrintJob(job, rj)) { throw new IOException("Job failed!"); } } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } return rj; } /** * Monitor a job and print status in real-time as progress is made and tasks * fail. * @param conf the job's configuration * @param job the job to track * @return true if the job succeeded * @throws IOException if communication to the JobTracker fails */ public boolean monitorAndPrintJob(JobConf conf, RunningJob job) throws IOException, InterruptedException { return ((NetworkedJob) job).monitorAndPrintJob(); } static String getTaskLogURL(TaskAttemptID taskId, String baseUrl) { return (baseUrl + "/tasklog?plaintext=true&attemptid=" + taskId); } static Configuration getConfiguration(String jobTrackerSpec) { Configuration conf = new Configuration(); if (jobTrackerSpec != null) { if (jobTrackerSpec.indexOf(":") >= 0) { conf.set("mapred.job.tracker", jobTrackerSpec); } else { String classpathFile = "hadoop-" + jobTrackerSpec + ".xml"; URL validate = conf.getResource(classpathFile); if (validate == null) { throw new RuntimeException(classpathFile + " not found on CLASSPATH"); } conf.addResource(classpathFile); } } return conf; } /** * Sets the output filter for tasks. only those tasks are printed whose * output matches the filter. * @param newValue task filter. */ @Deprecated public void setTaskOutputFilter(TaskStatusFilter newValue) { this.taskOutputFilter = newValue; } /** * Get the task output filter out of the JobConf. * * @param job the JobConf to examine. * @return the filter level. */ public static TaskStatusFilter getTaskOutputFilter(JobConf job) { return TaskStatusFilter.valueOf(job.get("jobclient.output.filter", "FAILED")); } /** * Modify the JobConf to set the task output filter. * * @param job the JobConf to modify. * @param newValue the value to set. */ public static void setTaskOutputFilter(JobConf job, TaskStatusFilter newValue) { job.set("jobclient.output.filter", newValue.toString()); } /** * Returns task output filter. * @return task filter. */ @Deprecated public TaskStatusFilter getTaskOutputFilter() { return this.taskOutputFilter; } protected long getCounter(org.apache.hadoop.mapreduce.Counters cntrs, String counterGroupName, String counterName) throws IOException { Counters counters = Counters.downgrade(cntrs); return counters.findCounter(counterGroupName, counterName).getValue(); } /** * Get status information about the max available Maps in the cluster. * * @return the max available Maps in the cluster * @throws IOException */ public int getDefaultMaps() throws IOException { try { return clientUgi.doAs(new PrivilegedExceptionAction<Integer>() { @Override public Integer run() throws IOException, InterruptedException { return cluster.getClusterStatus().getMapSlotCapacity(); } }); } catch (InterruptedException ie) { throw new IOException(ie); } } /** * Get status information about the max available Reduces in the cluster. * * @return the max available Reduces in the cluster * @throws IOException */ public int getDefaultReduces() throws IOException { try { return clientUgi.doAs(new PrivilegedExceptionAction<Integer>() { @Override public Integer run() throws IOException, InterruptedException { return cluster.getClusterStatus().getReduceSlotCapacity(); } }); } catch (InterruptedException ie) { throw new IOException(ie); } } /** * Grab the jobtracker system directory path where job-specific files are to be placed. * * @return the system directory where job-specific files are to be placed. */ public Path getSystemDir() { try { return clientUgi.doAs(new PrivilegedExceptionAction<Path>() { @Override public Path run() throws IOException, InterruptedException { return cluster.getSystemDir(); } }); } catch (IOException ioe) { return null; } catch (InterruptedException ie) { return null; } } /** * Checks if the job directory is clean and has all the required components * for (re) starting the job */ public static boolean isJobDirValid(Path jobDirPath, FileSystem fs) throws IOException { FileStatus[] contents = fs.listStatus(jobDirPath); int matchCount = 0; if (contents != null && contents.length >= 2) { for (FileStatus status : contents) { if ("job.xml".equals(status.getPath().getName())) { ++matchCount; } if ("job.split".equals(status.getPath().getName())) { ++matchCount; } } if (matchCount == 2) { return true; } } return false; } /** * Fetch the staging area directory for the application * * @return path to staging area directory * @throws IOException */ public Path getStagingAreaDir() throws IOException { try { return clientUgi.doAs(new PrivilegedExceptionAction<Path>() { @Override public Path run() throws IOException, InterruptedException { return cluster.getStagingAreaDir(); } }); } catch (InterruptedException ie) { // throw RuntimeException instead for compatibility reasons throw new RuntimeException(ie); } } private JobQueueInfo getJobQueueInfo(QueueInfo queue) { JobQueueInfo ret = new JobQueueInfo(queue); // make sure to convert any children if (queue.getQueueChildren().size() > 0) { List<JobQueueInfo> childQueues = new ArrayList<JobQueueInfo>(queue.getQueueChildren().size()); for (QueueInfo child : queue.getQueueChildren()) { childQueues.add(getJobQueueInfo(child)); } ret.setChildren(childQueues); } return ret; } private JobQueueInfo[] getJobQueueInfoArray(QueueInfo[] queues) throws IOException { JobQueueInfo[] ret = new JobQueueInfo[queues.length]; for (int i = 0; i < queues.length; i++) { ret[i] = getJobQueueInfo(queues[i]); } return ret; } /** * Returns an array of queue information objects about root level queues * configured * * @return the array of root level JobQueueInfo objects * @throws IOException */ public JobQueueInfo[] getRootQueues() throws IOException { try { return clientUgi.doAs(new PrivilegedExceptionAction<JobQueueInfo[]>() { public JobQueueInfo[] run() throws IOException, InterruptedException { return getJobQueueInfoArray(cluster.getRootQueues()); } }); } catch (InterruptedException ie) { throw new IOException(ie); } } /** * Returns an array of queue information objects about immediate children * of queue queueName. * * @param queueName * @return the array of immediate children JobQueueInfo objects * @throws IOException */ public JobQueueInfo[] getChildQueues(final String queueName) throws IOException { try { return clientUgi.doAs(new PrivilegedExceptionAction<JobQueueInfo[]>() { public JobQueueInfo[] run() throws IOException, InterruptedException { return getJobQueueInfoArray(cluster.getChildQueues(queueName)); } }); } catch (InterruptedException ie) { throw new IOException(ie); } } /** * Return an array of queue information objects about all the Job Queues * configured. * * @return Array of JobQueueInfo objects * @throws IOException */ public JobQueueInfo[] getQueues() throws IOException { try { return clientUgi.doAs(new PrivilegedExceptionAction<JobQueueInfo[]>() { public JobQueueInfo[] run() throws IOException, InterruptedException { return getJobQueueInfoArray(cluster.getQueues()); } }); } catch (InterruptedException ie) { throw new IOException(ie); } } /** * Gets all the jobs which were added to particular Job Queue * * @param queueName name of the Job Queue * @return Array of jobs present in the job queue * @throws IOException */ public JobStatus[] getJobsFromQueue(final String queueName) throws IOException { try { QueueInfo queue = clientUgi.doAs(new PrivilegedExceptionAction<QueueInfo>() { @Override public QueueInfo run() throws IOException, InterruptedException { return cluster.getQueue(queueName); } }); if (queue == null) { return null; } org.apache.hadoop.mapreduce.JobStatus[] stats = queue.getJobStatuses(); JobStatus[] ret = new JobStatus[stats.length]; for (int i = 0; i < stats.length; i++) { ret[i] = JobStatus.downgrade(stats[i]); } return ret; } catch (InterruptedException ie) { throw new IOException(ie); } } /** * Gets the queue information associated to a particular Job Queue * * @param queueName name of the job queue. * @return Queue information associated to particular queue. * @throws IOException */ public JobQueueInfo getQueueInfo(final String queueName) throws IOException { try { QueueInfo queueInfo = clientUgi.doAs(new PrivilegedExceptionAction<QueueInfo>() { public QueueInfo run() throws IOException, InterruptedException { return cluster.getQueue(queueName); } }); if (queueInfo != null) { return new JobQueueInfo(queueInfo); } return null; } catch (InterruptedException ie) { throw new IOException(ie); } } /** * Gets the Queue ACLs for current user * @return array of QueueAclsInfo object for current user. * @throws IOException */ public QueueAclsInfo[] getQueueAclsForCurrentUser() throws IOException { try { org.apache.hadoop.mapreduce.QueueAclsInfo[] acls = clientUgi .doAs(new PrivilegedExceptionAction<org.apache.hadoop.mapreduce.QueueAclsInfo[]>() { public org.apache.hadoop.mapreduce.QueueAclsInfo[] run() throws IOException, InterruptedException { return cluster.getQueueAclsForCurrentUser(); } }); QueueAclsInfo[] ret = new QueueAclsInfo[acls.length]; for (int i = 0; i < acls.length; i++) { ret[i] = QueueAclsInfo.downgrade(acls[i]); } return ret; } catch (InterruptedException ie) { throw new IOException(ie); } } /** * Get a delegation token for the user from the JobTracker. * @param renewer the user who can renew the token * @return the new token * @throws IOException */ public Token<DelegationTokenIdentifier> getDelegationToken(final Text renewer) throws IOException, InterruptedException { return clientUgi.doAs(new PrivilegedExceptionAction<Token<DelegationTokenIdentifier>>() { public Token<DelegationTokenIdentifier> run() throws IOException, InterruptedException { return cluster.getDelegationToken(renewer); } }); } /** * Renew a delegation token * @param token the token to renew * @return true if the renewal went well * @throws InvalidToken * @throws IOException * @deprecated Use {@link Token#renew} instead */ public long renewDelegationToken(Token<DelegationTokenIdentifier> token) throws InvalidToken, IOException, InterruptedException { return token.renew(getConf()); } /** * Cancel a delegation token from the JobTracker * @param token the token to cancel * @throws IOException * @deprecated Use {@link Token#cancel} instead */ public void cancelDelegationToken(Token<DelegationTokenIdentifier> token) throws InvalidToken, IOException, InterruptedException { token.cancel(getConf()); } /** */ public static void main(String argv[]) throws Exception { int res = ToolRunner.run(new JobClient(), argv); System.exit(res); } }