Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapred; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.AfterClass; import org.junit.Test; import java.io.DataOutputStream; import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.test.system.MRCluster; import org.apache.hadoop.mapreduce.test.system.JTProtocol; import org.apache.hadoop.mapreduce.test.system.JobInfo; import org.apache.hadoop.mapreduce.test.system.TaskInfo; import org.apache.hadoop.mapreduce.test.system.TTClient; import org.apache.hadoop.mapreduce.test.system.JTClient; import org.apache.hadoop.mapreduce.test.system.FinishTaskControlAction; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.examples.SleepJob; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.mapred.UtilsForTests; /** * A System test for verifying the status after killing the * tasks at different conditions. */ public class TestTaskKilling { private static final Log LOG = LogFactory.getLog(TestTaskKilling.class); private static MRCluster cluster; private static JobClient jobClient = null; private static JTClient jtClient = null; private static JTProtocol remoteJTClient = null; private static Configuration conf = new Configuration(); @BeforeClass public static void before() throws Exception { cluster = MRCluster.createCluster(conf); cluster.setUp(); jtClient = cluster.getJTClient(); jobClient = jtClient.getClient(); remoteJTClient = jtClient.getProxy(); } @AfterClass public static void after() throws Exception { cluster.tearDown(); } /** * Verifying the running job status whether it succeeds or not * after failing some of its tasks. */ @Test public void testFailedTaskJobStatus() throws IOException, InterruptedException { conf = remoteJTClient.getDaemonConf(); TaskInfo taskInfo = null; SleepJob job = new SleepJob(); job.setConf(conf); JobConf jobConf = job.setupJobConf(1, 1, 10000, 4000, 100, 100); RunningJob runJob = jobClient.submitJob(jobConf); JobID jobId = runJob.getID(); JobInfo jInfo = remoteJTClient.getJobInfo(jobId); Assert.assertTrue("Job has not been started for 1 min.", jtClient.isJobStarted(jobId)); TaskInfo[] taskInfos = remoteJTClient.getTaskInfo(jobId); for (TaskInfo taskinfo : taskInfos) { if (!taskinfo.isSetupOrCleanup() && taskinfo.getTaskID().isMap()) { taskInfo = taskinfo; break; } } Assert.assertTrue("Task has not been started for 1 min.", jtClient.isTaskStarted(taskInfo)); // Fail the running task. RunningJob networkJob = jobClient.getJob(jobId); TaskID tID = TaskID.downgrade(taskInfo.getTaskID()); TaskAttemptID taskAttID = new TaskAttemptID(tID, 0); networkJob.killTask(taskAttID, true); LOG.info("Waiting till the job is completed..."); while (!jInfo.getStatus().isJobComplete()) { UtilsForTests.waitFor(100); jInfo = remoteJTClient.getJobInfo(jobId); } Assert.assertEquals("JobStatus", JobStatus.SUCCEEDED, jInfo.getStatus().getRunState()); } /** * Verifying whether task temporary output directory is cleaned up or not * after killing the task. */ @Test public void testDirCleanupAfterTaskKilled() throws IOException, InterruptedException { TaskInfo taskInfo = null; boolean isTempFolderExists = false; String localTaskDir = null; TTClient ttClient = null; FileStatus filesStatus[] = null; Path inputDir = new Path("input"); Path outputDir = new Path("output"); Configuration conf = new Configuration(cluster.getConf()); JobConf jconf = new JobConf(conf); jconf.setJobName("Word Count"); jconf.setJarByClass(WordCount.class); jconf.setMapperClass(WordCount.MapClass.class); jconf.setCombinerClass(WordCount.Reduce.class); jconf.setReducerClass(WordCount.Reduce.class); jconf.setNumMapTasks(1); jconf.setNumReduceTasks(1); jconf.setOutputKeyClass(Text.class); jconf.setOutputValueClass(IntWritable.class); cleanup(inputDir, conf); cleanup(outputDir, conf); createInput(inputDir, conf); FileInputFormat.setInputPaths(jconf, inputDir); FileOutputFormat.setOutputPath(jconf, outputDir); RunningJob runJob = jobClient.submitJob(jconf); JobID id = runJob.getID(); JobInfo jInfo = remoteJTClient.getJobInfo(id); Assert.assertTrue("Job has not been started for 1 min.", jtClient.isJobStarted(id)); JobStatus[] jobStatus = jobClient.getAllJobs(); String userName = jobStatus[0].getUsername(); TaskInfo[] taskInfos = remoteJTClient.getTaskInfo(id); for (TaskInfo taskinfo : taskInfos) { if (!taskinfo.isSetupOrCleanup() && taskinfo.getTaskID().isMap()) { taskInfo = taskinfo; break; } } Assert.assertTrue("Task has not been started for 1 min.", jtClient.isTaskStarted(taskInfo)); TaskID tID = TaskID.downgrade(taskInfo.getTaskID()); FinishTaskControlAction action = new FinishTaskControlAction(tID); String[] taskTrackers = taskInfo.getTaskTrackers(); int counter = 0; TaskInfo prvTaskInfo = taskInfo; while (counter++ < 30) { if (taskTrackers.length > 0) { break; } else { UtilsForTests.waitFor(100); taskInfo = remoteJTClient.getTaskInfo(taskInfo.getTaskID()); if (taskInfo == null) { taskInfo = prvTaskInfo; } else { prvTaskInfo = taskInfo; } taskTrackers = taskInfo.getTaskTrackers(); } } Assert.assertTrue("TaskTracker is not found.", taskTrackers.length > 0); String hostName = taskTrackers[0].split("_")[1]; hostName = hostName.split(":")[0]; ttClient = cluster.getTTClient(hostName); String localDirs[] = ttClient.getMapredLocalDirs(); TaskAttemptID taskAttID = new TaskAttemptID(tID, 0); for (String localDir : localDirs) { localTaskDir = localDir + "/" + TaskTracker.getLocalTaskDir(userName, id.toString(), taskAttID.toString()); filesStatus = ttClient.listStatus(localTaskDir, true); if (filesStatus.length > 0) { isTempFolderExists = true; break; } } Assert.assertTrue("Task Attempt directory " + taskAttID + " has not been found while task was running.", isTempFolderExists); RunningJob networkJob = jobClient.getJob(id); networkJob.killTask(taskAttID, false); ttClient.getProxy().sendAction(action); taskInfo = remoteJTClient.getTaskInfo(tID); while (taskInfo.getTaskStatus()[0].getRunState() == TaskStatus.State.RUNNING) { UtilsForTests.waitFor(1000); taskInfo = remoteJTClient.getTaskInfo(tID); } UtilsForTests.waitFor(1000); taskInfo = remoteJTClient.getTaskInfo(tID); Assert.assertTrue("Task status has not been changed to KILLED.", (TaskStatus.State.KILLED == taskInfo.getTaskStatus()[0].getRunState() || TaskStatus.State.KILLED_UNCLEAN == taskInfo.getTaskStatus()[0].getRunState())); taskInfo = remoteJTClient.getTaskInfo(tID); counter = 0; while (counter++ < 60) { filesStatus = ttClient.listStatus(localTaskDir, true); if (filesStatus.length == 0) { break; } else { UtilsForTests.waitFor(100); } } Assert.assertTrue("Task attempt temporary folder has not been cleaned.", isTempFolderExists && filesStatus.length == 0); UtilsForTests.waitFor(1000); jInfo = remoteJTClient.getJobInfo(id); LOG.info("Waiting till the job is completed..."); while (!jInfo.getStatus().isJobComplete()) { UtilsForTests.waitFor(100); jInfo = remoteJTClient.getJobInfo(id); } } private void cleanup(Path dir, Configuration conf) throws IOException { FileSystem fs = dir.getFileSystem(conf); fs.delete(dir, true); } private void createInput(Path inDir, Configuration conf) throws IOException { String input = "Hadoop is framework for data intensive distributed " + "applications.\n" + "Hadoop enables applications to work with thousands of nodes."; FileSystem fs = inDir.getFileSystem(conf); if (!fs.mkdirs(inDir)) { throw new IOException("Failed to create the input directory:" + inDir.toString()); } fs.setPermission(inDir, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); DataOutputStream file = fs.create(new Path(inDir, "data.txt")); int i = 0; while (i < 1000 * 3000) { file.writeBytes(input); i++; } file.close(); } /** * Verifying whether task temporary output directory is cleaned up or not * after failing the task. */ @Test public void testDirCleanupAfterTaskFailed() throws IOException, InterruptedException { TTClient ttClient = null; FileStatus filesStatus[] = null; String localTaskDir = null; TaskInfo taskInfo = null; TaskID tID = null; boolean isTempFolderExists = false; conf = remoteJTClient.getDaemonConf(); SleepJob job = new SleepJob(); job.setConf(conf); JobConf jobConf = job.setupJobConf(1, 0, 10000, 100, 10, 10); RunningJob runJob = jobClient.submitJob(jobConf); JobID id = runJob.getID(); JobInfo jInfo = remoteJTClient.getJobInfo(id); Assert.assertTrue("Job has not been started for 1 min.", jtClient.isJobStarted(id)); JobStatus[] jobStatus = jobClient.getAllJobs(); String userName = jobStatus[0].getUsername(); TaskInfo[] taskInfos = remoteJTClient.getTaskInfo(id); for (TaskInfo taskinfo : taskInfos) { if (!taskinfo.isSetupOrCleanup() && taskinfo.getTaskID().isMap()) { taskInfo = taskinfo; break; } } Assert.assertTrue("Task has not been started for 1 min.", jtClient.isTaskStarted(taskInfo)); tID = TaskID.downgrade(taskInfo.getTaskID()); FinishTaskControlAction action = new FinishTaskControlAction(tID); String[] taskTrackers = taskInfo.getTaskTrackers(); int counter = 0; TaskInfo prvTaskInfo = taskInfo; while (counter++ < 30) { if (taskTrackers.length > 0) { break; } else { UtilsForTests.waitFor(1000); taskInfo = remoteJTClient.getTaskInfo(taskInfo.getTaskID()); if (taskInfo == null) { taskInfo = prvTaskInfo; } else { prvTaskInfo = taskInfo; } taskTrackers = taskInfo.getTaskTrackers(); } } Assert.assertTrue("Task tracker not found.", taskTrackers.length > 0); String hostName = taskTrackers[0].split("_")[1]; hostName = hostName.split(":")[0]; ttClient = cluster.getTTClient(hostName); String localDirs[] = ttClient.getMapredLocalDirs(); TaskAttemptID taskAttID = new TaskAttemptID(tID, 0); for (String localDir : localDirs) { localTaskDir = localDir + "/" + TaskTracker.getLocalTaskDir(userName, id.toString(), taskAttID.toString()); filesStatus = ttClient.listStatus(localTaskDir, true); if (filesStatus.length > 0) { isTempFolderExists = true; break; } } Assert.assertTrue("Task Attempt directory " + taskAttID + " has not been found while task was running.", isTempFolderExists); boolean isFailTask = false; JobInfo jobInfo = remoteJTClient.getJobInfo(id); int MAX_MAP_TASK_ATTEMPTS = Integer.parseInt(jobConf.get("mapred.map.max.attempts")); if (!isFailTask) { TaskID taskId = TaskID.downgrade(taskInfo.getTaskID()); TaskAttemptID tAttID = new TaskAttemptID(taskId, taskInfo.numFailedAttempts()); while (taskInfo.numFailedAttempts() < MAX_MAP_TASK_ATTEMPTS) { RunningJob networkJob = jobClient.getJob(id); networkJob.killTask(taskAttID, true); taskInfo = remoteJTClient.getTaskInfo(taskInfo.getTaskID()); taskAttID = new TaskAttemptID(taskId, taskInfo.numFailedAttempts()); } isFailTask = true; } ttClient.getProxy().sendAction(action); taskInfo = remoteJTClient.getTaskInfo(tID); Assert.assertTrue("Task status has not been changed to FAILED.", TaskStatus.State.FAILED == taskInfo.getTaskStatus()[0].getRunState() || TaskStatus.State.FAILED_UNCLEAN == taskInfo.getTaskStatus()[0].getRunState()); UtilsForTests.waitFor(1000); filesStatus = ttClient.listStatus(localTaskDir, true); Assert.assertTrue("Temporary folder has not been cleanup.", filesStatus.length == 0); UtilsForTests.waitFor(1000); jInfo = remoteJTClient.getJobInfo(id); LOG.info("Waiting till the job is completed..."); while (!jInfo.getStatus().isJobComplete()) { UtilsForTests.waitFor(100); jInfo = remoteJTClient.getJobInfo(id); } } @Test /** * This tests verification of job killing by killing of all task * attempts of a particular task * @param none * @return void */ public void testAllTaskAttemptKill() throws Exception { Configuration conf = new Configuration(cluster.getConf()); JobStatus[] jobStatus = null; SleepJob job = new SleepJob(); job.setConf(conf); conf = job.setupJobConf(2, 1, 40000, 1000, 100, 100); JobConf jconf = new JobConf(conf); //Submitting the job RunningJob rJob = cluster.getJTClient().getClient().submitJob(jconf); int MAX_MAP_TASK_ATTEMPTS = Integer.parseInt(jconf.get("mapred.map.max.attempts")); LOG.info("MAX_MAP_TASK_ATTEMPTS is : " + MAX_MAP_TASK_ATTEMPTS); Assert.assertTrue(MAX_MAP_TASK_ATTEMPTS > 0); TTClient tClient = null; TTClient[] ttClients = null; JobInfo jInfo = remoteJTClient.getJobInfo(rJob.getID()); //Assert if jobInfo is null Assert.assertNotNull(jInfo.getStatus().getRunState()); //Wait for the job to start running. while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) { try { Thread.sleep(10000); } catch (InterruptedException e) { } ; jInfo = remoteJTClient.getJobInfo(rJob.getID()); } //Temporarily store the jobid to use it later for comparision. JobID jobidStore = rJob.getID(); jobidStore = JobID.downgrade(jobidStore); LOG.info("job id is :" + jobidStore.toString()); TaskInfo[] taskInfos = null; //After making sure that the job is running, //the test execution has to make sure that //at least one task has started running before continuing. boolean runningCount = false; int count = 0; do { taskInfos = cluster.getJTClient().getProxy().getTaskInfo(rJob.getID()); runningCount = false; for (TaskInfo taskInfo : taskInfos) { TaskStatus[] taskStatuses = taskInfo.getTaskStatus(); if (taskStatuses.length > 0) { LOG.info("taskStatuses[0].getRunState() is :" + taskStatuses[0].getRunState()); if (taskStatuses[0].getRunState() == TaskStatus.State.RUNNING) { runningCount = true; break; } else { LOG.info("Sleeping 5 seconds"); Thread.sleep(5000); } } } count++; //If the count goes beyond a point, then break; This is to avoid //infinite loop under unforeseen circumstances. Testcase will anyway //fail later. if (count > 10) { Assert.fail("Since the sleep count has reached beyond a point" + "failing at this point"); } } while (!runningCount); //This whole module is about getting the task Attempt id //of one task and killing it MAX_MAP_TASK_ATTEMPTS times, //whenever it re-attempts to run. String taskIdKilled = null; for (int i = 0; i < MAX_MAP_TASK_ATTEMPTS; i++) { taskInfos = cluster.getJTClient().getProxy().getTaskInfo(rJob.getID()); for (TaskInfo taskInfo : taskInfos) { TaskAttemptID taskAttemptID; if (!taskInfo.isSetupOrCleanup()) { //This is the task which is going to be killed continously in //all its task attempts.The first task is getting picked up. TaskID taskid = TaskID.downgrade(taskInfo.getTaskID()); LOG.info("taskid is :" + taskid); if (i == 0) { taskIdKilled = taskid.toString(); taskAttemptID = new TaskAttemptID(taskid, i); LOG.info("taskAttemptid going to be killed is : " + taskAttemptID); rJob.killTask(taskAttemptID, true); checkTaskCompletionEvent(taskAttemptID, jInfo); break; } else { if (taskIdKilled.equals(taskid.toString())) { taskAttemptID = new TaskAttemptID(taskid, i); //Make sure that task is midway and then kill UtilsForTests.waitFor(20000); LOG.info("taskAttemptid going to be killed is : " + taskAttemptID); rJob.killTask(taskAttemptID, true); checkTaskCompletionEvent(taskAttemptID, jInfo); break; } } } } } //Making sure that the job is complete. while (jInfo != null && !jInfo.getStatus().isJobComplete()) { Thread.sleep(10000); jInfo = remoteJTClient.getJobInfo(rJob.getID()); } //Making sure that the correct jobstatus is got from all the jobs jobStatus = jobClient.getAllJobs(); JobStatus jobStatusFound = null; for (JobStatus jobStatusTmp : jobStatus) { if (JobID.downgrade(jobStatusTmp.getJobID()).equals(jobidStore)) { jobStatusFound = jobStatusTmp; LOG.info("jobStatus found is :" + jobStatusFound.getJobId().toString()); } } //Making sure that the job has FAILED Assert.assertEquals("The job should have failed at this stage", JobStatus.FAILED, jobStatusFound.getRunState()); } //This method checks if task Attemptid occurs in the list //of tasks that are completed (killed) for a job.This is //required because after issuing a kill comamnd, the task //has to be killed and appear in the taskCompletion event. //After this a new task attempt will start running in a //matter of few seconds. public void checkTaskCompletionEvent(TaskAttemptID taskAttemptID, JobInfo jInfo) throws Exception { boolean match = false; int count = 0; while (!match) { org.apache.hadoop.mapreduce.JobID temp = jInfo.getID(); RunningJob rJob = jobClient.getJob(new JobID(temp.getJtIdentifier(), temp.getId())); TaskCompletionEvent[] taskCompletionEvents = rJob.getTaskCompletionEvents(0); for (TaskCompletionEvent taskCompletionEvent : taskCompletionEvents) { LOG.info("taskCompletionEvent.getTaskAttemptId().toString() is : " + taskCompletionEvent.getTaskAttemptId().toString()); LOG.info("compared to taskAttemptID.toString() :" + taskAttemptID.toString()); if ((taskCompletionEvent.getTaskAttemptId().toString()).equals(taskAttemptID.toString())) { match = true; //Sleeping for 10 seconds giving time for the next task //attempt to run Thread.sleep(10000); break; } } if (!match) { LOG.info("Thread is sleeping for 10 seconds"); Thread.sleep(10000); count++; } //If the count goes beyond a point, then break; This is to avoid //infinite loop under unforeseen circumstances.Testcase will anyway //fail later. if (count > 10) { Assert.fail("Since the task attemptid is not appearing in the" + "TaskCompletionEvent, it seems this task attempt was not killed"); } } } }