Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapred; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.AfterClass; import org.junit.Test; import java.io.DataOutputStream; import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.ClusterMetrics; import org.apache.hadoop.mapreduce.test.system.MRCluster; import org.apache.hadoop.mapreduce.test.system.JTProtocol; import org.apache.hadoop.mapreduce.test.system.JobInfo; import org.apache.hadoop.mapreduce.test.system.JTClient; import org.apache.hadoop.mapred.JobQueueInfo; import org.apache.hadoop.mapred.JobHistory.Keys; import org.apache.hadoop.mapred.lib.IdentityReducer; import org.apache.hadoop.examples.SleepJob; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import testjar.GenerateTaskChildProcess; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.security.UserGroupInformation; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; /** * Verifying the job-summary information at the end of a job. */ public class TestJobSummary { private static final Log LOG = LogFactory.getLog(TestJobSummary.class); private static MRCluster cluster; private static JobClient jobClient = null; private static JTProtocol remoteJTClient = null; private static JTClient jtClient; private static Configuration conf = new Configuration(); private static Path inputDir = new Path("input"); private static Path outputDir = new Path("output"); private static String queueName = null; @BeforeClass public static void before() throws Exception { String[] expExcludeList = { "java.net.ConnectException", "java.io.IOException" }; cluster = MRCluster.createCluster(conf); cluster.setExcludeExpList(expExcludeList); cluster.setUp(); createInput(inputDir, conf); jtClient = cluster.getJTClient(); jobClient = jtClient.getClient(); remoteJTClient = cluster.getJTClient().getProxy(); conf = remoteJTClient.getDaemonConf(); } @AfterClass public static void after() throws Exception { cleanup(inputDir, conf); cleanup(outputDir, conf); cluster.tearDown(); } /** * Verifying the job summary information for killed job. */ @Test public void testJobSummaryInfoOfKilledJob() throws IOException, InterruptedException { SleepJob job = new SleepJob(); conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false); job.setConf(conf); conf = job.setupJobConf(2, 1, 4000, 4000, 100, 100); JobConf jobConf = new JobConf(conf); RunningJob runJob = jobClient.submitJob(jobConf); JobID jobId = runJob.getID(); Assert.assertTrue("Job has not been started for 1 min.", jtClient.isJobStarted(jobId)); jobClient.killJob(jobId); LOG.info("Waiting till the job is completed..."); Assert.assertTrue("Job has not been completed for 1 min.", jtClient.isJobStopped(jobId)); JobInfo jInfo = remoteJTClient.getJobInfo(jobId); Assert.assertEquals("Job has not been succeeded", jInfo.getStatus().getRunState(), JobStatus.KILLED); verifyJobSummaryInfo(jInfo, jobId); } /** * Verifying the job summary information for failed job. */ @Test public void testJobSummaryInfoOfFailedJob() throws IOException, InterruptedException { conf = remoteJTClient.getDaemonConf(); conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false); JobConf jobConf = new JobConf(conf); jobConf.setJobName("Fail Job"); jobConf.setJarByClass(GenerateTaskChildProcess.class); jobConf.setMapperClass(GenerateTaskChildProcess.FailMapper.class); jobConf.setReducerClass(IdentityReducer.class); jobConf.setNumMapTasks(1); jobConf.setNumReduceTasks(1); cleanup(outputDir, conf); FileInputFormat.setInputPaths(jobConf, inputDir); FileOutputFormat.setOutputPath(jobConf, outputDir); RunningJob runJob = jobClient.submitJob(jobConf); JobID jobId = runJob.getID(); Assert.assertTrue("Job has not been started for 1 min.", jtClient.isJobStarted(jobId)); LOG.info("Waiting till the job is completed..."); Assert.assertTrue("Job has not been completed for 1 min.", jtClient.isJobStopped(jobId)); JobInfo jInfo = remoteJTClient.getJobInfo(jobId); Assert.assertEquals("Job has not been failed", jInfo.getStatus().getRunState(), JobStatus.FAILED); verifyJobSummaryInfo(jInfo, jobId); } /** * Submit the job in different queue and verifying * the job queue information in job summary * after job is completed. */ @Test public void testJobQueueInfoInJobSummary() throws IOException, InterruptedException { SleepJob job = new SleepJob(); job.setConf(conf); conf = job.setupJobConf(2, 1, 4000, 4000, 100, 100); conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false); JobConf jobConf = new JobConf(conf); JobQueueInfo[] queues = jobClient.getQueues(); for (JobQueueInfo queueInfo : queues) { if (!queueInfo.getQueueName().equals("default")) { queueName = queueInfo.getQueueName(); break; } } Assert.assertNotNull("No multiple queues in the cluster.", queueName); LOG.info("queueName:" + queueName); jobConf.setQueueName(queueName); RunningJob runJob = jobClient.submitJob(jobConf); JobID jobId = runJob.getID(); Assert.assertTrue("Job has not been started for 1 min.", jtClient.isJobStarted(jobId)); LOG.info("Waiting till the job is completed..."); Assert.assertTrue("Job has not been completed for 1 min.", jtClient.isJobStopped(jobId)); JobInfo jInfo = remoteJTClient.getJobInfo(jobId); Assert.assertEquals("Job has not been succeeded", jInfo.getStatus().getRunState(), JobStatus.SUCCEEDED); verifyJobSummaryInfo(jInfo, jobId); } /** * Verify the job summary information for high RAM jobs. */ @Test public void testJobSummaryInfoOfHighMemoryJob() throws IOException, Exception { final HighRamJobHelper helper = new HighRamJobHelper(); JobID jobId = helper.runHighRamJob(conf, jobClient, remoteJTClient, "Job did not succeed"); JobInfo jInfo = remoteJTClient.getJobInfo(jobId); verifyJobSummaryInfo(jInfo, jobId); } @Test public void testJobSummaryInfoForDifferentUser() throws Exception { UserGroupInformation proxyUGI; UserGroupInformation ugi = UserGroupInformation.getLoginUser(); ArrayList<String> users = cluster.getHadoopMultiUsersList(); Assert.assertTrue("proxy users are not found.", users.size() > 0); if (conf.get("hadoop.security.authentication").equals("simple")) { proxyUGI = UserGroupInformation.createRemoteUser(users.get(0)); } else { proxyUGI = UserGroupInformation.createProxyUser(users.get(0), ugi); } SleepJob job = new SleepJob(); job.setConf(conf); final JobConf jobConf = job.setupJobConf(2, 1, 2000, 2000, 100, 100); final JobClient jClient = proxyUGI.doAs(new PrivilegedExceptionAction<JobClient>() { public JobClient run() throws IOException { return new JobClient(jobConf); } }); RunningJob runJob = proxyUGI.doAs(new PrivilegedExceptionAction<RunningJob>() { public RunningJob run() throws IOException { return jClient.submitJob(jobConf); } }); JobID jobId = runJob.getID(); Assert.assertTrue("Job has not been started for 1 min.", jtClient.isJobStarted(jobId)); LOG.info("Waiting till the job is completed..."); Assert.assertTrue("Job has not been completed for 1 min.", jtClient.isJobStopped(jobId)); JobInfo jInfo = remoteJTClient.getJobInfo(jobId); Assert.assertEquals("Job has not been succeeded", jInfo.getStatus().getRunState(), JobStatus.SUCCEEDED); verifyJobSummaryInfo(jInfo, jobId); } private void verifyJobSummaryInfo(JobInfo jInfo, JobID id) throws IOException { java.util.HashMap<String, String> map = jtClient.getJobSummary(id); Assert.assertEquals("Job id has not been matched", id.toString(), map.get("jobId")); Assert.assertEquals("User name has not been matched in JobSummary", jInfo.getStatus().getUsername(), map.get("user")); Assert.assertEquals("StartTime has not been matched in JobSummary", String.valueOf(jInfo.getStatus().getStartTime()), map.get("startTime")); Assert.assertEquals("LaunchTime has not been matched in JobSummary", String.valueOf(jInfo.getLaunchTime()), map.get("launchTime")); Assert.assertEquals("FinshedTime has not been matched in JobSummary", String.valueOf(jInfo.getFinishTime()), map.get("finishTime")); Assert.assertEquals("Maps are not matched in Job summary", String.valueOf(jInfo.numMaps()), map.get("numMaps")); Assert.assertEquals("Reducers are not matched in Job summary", String.valueOf(jInfo.numReduces()), map.get("numReduces")); Assert.assertEquals("Number of slots per map is not matched in Job summary", String.valueOf(jInfo.getNumSlotsPerMap()), map.get("numSlotsPerMap")); Assert.assertEquals("Number of slots per reduce is not matched in Job summary", String.valueOf(jInfo.getNumSlotsPerReduce()), map.get("numSlotsPerReduce")); } private static void cleanup(Path dir, Configuration conf) throws IOException { FileSystem fs = dir.getFileSystem(conf); fs.delete(dir, true); } private static void createInput(Path inDir, Configuration conf) throws IOException { String input = "Hadoop is framework for data intensive distributed " + "applications.\n" + "Hadoop enables applications to work with thousands of nodes."; FileSystem fs = inDir.getFileSystem(conf); if (!fs.mkdirs(inDir)) { throw new IOException("Failed to create the input directory:" + inDir.toString()); } fs.setPermission(inDir, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); DataOutputStream file = fs.create(new Path(inDir, "data.txt")); int i = 0; while (i < 2) { file.writeBytes(input); i++; } file.close(); } }