Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapred; import java.io.*; import java.util.ArrayList; import java.util.List; import junit.extensions.TestSetup; import junit.framework.Test; import junit.framework.TestSuite; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.util.StringUtils; /** * Validates removal of user-created-files(and set non-writable permissions) in * tasks under taskWorkDir by TT with LinuxTaskController. */ public class TestChildTaskDirs extends ClusterWithLinuxTaskController { private static final Log LOG = LogFactory.getLog(TestChildTaskDirs.class); private static final File TEST_DIR = new File(System.getProperty("test.build.data", "/tmp"), "child-dirs"); private static final String MY_DIR = "my-test-dir"; private static final String MY_FILE = "my-test-file"; private static final LocalDirAllocator LOCAL_DIR_ALLOC = new LocalDirAllocator("mapred.local.dir"); public static Test suite() { TestSetup setup = new TestSetup(new TestSuite(TestChildTaskDirs.class)) { protected void setUp() throws Exception { TEST_DIR.mkdirs(); } protected void tearDown() throws Exception { FileUtil.fullyDelete(TEST_DIR); } }; return setup; } class InlineCleanupQueue extends CleanupQueue { List<String> stalePaths = new ArrayList<String>(); public InlineCleanupQueue() { // do nothing } @Override public void addToQueue(PathDeletionContext... contexts) { // delete paths in-line for (PathDeletionContext context : contexts) { try { if (!deletePath(context)) { LOG.warn("Stale path " + context.fullPath); stalePaths.add(context.fullPath); } } catch (IOException e) { LOG.warn("Caught exception while deleting path " + context.fullPath); LOG.info(StringUtils.stringifyException(e)); stalePaths.add(context.fullPath); } } } } // Mapper that creates dirs // job-id/ // -attempt-id/ // -work/ // -my-test-dir(555) // -my-test-file(555) static class CreateDir extends MapReduceBase implements Mapper<WritableComparable, Writable, WritableComparable, Writable> { File taskWorkDir = null; public void map(WritableComparable key, Writable value, OutputCollector<WritableComparable, Writable> out, Reporter reporter) throws IOException { File subDir = new File(taskWorkDir, MY_DIR); LOG.info("Child folder : " + subDir); subDir.mkdirs(); File newFile = new File(subDir, MY_FILE); LOG.info("Child file : " + newFile); newFile.createNewFile(); // Set the permissions of my-test-dir and my-test-dir/my-test-file to 555 try { FileUtil.chmod(subDir.getAbsolutePath(), "a=rx", true); } catch (Exception e) { throw new IOException(e); } } @Override public void configure(JobConf conf) { String jobId = conf.get("mapred.job.id"); String taskId = conf.get("mapred.task.id"); String taskDir = TaskTracker.getLocalTaskDir(jobId, taskId); try { Path taskDirPath = LOCAL_DIR_ALLOC.getLocalPathForWrite(taskDir, conf); taskWorkDir = new File(taskDirPath.toString(), "work"); LOG.info("Task work-dir : " + taskWorkDir.toString()); } catch (IOException ioe) { throw new RuntimeException(ioe); } } } public void testChildDirCleanup() throws Exception { LOG.info("Testing if the dirs created by the child process is cleaned up properly"); if (!shouldRun()) { return; } // start the cluster startCluster(); // make sure that only one tracker is configured if (mrCluster.getNumTaskTrackers() != 1) { throw new Exception("Cluster started with " + mrCluster.getNumTaskTrackers() + " instead of 1"); } // configure a job JobConf jConf = getClusterConf(); jConf.setJobName("Mkdir job"); jConf.setMapperClass(CreateDir.class); jConf.setNumMapTasks(1); jConf.setNumReduceTasks(0); FileSystem fs = FileSystem.get(jConf); Path inDir = new Path("in"); Path outDir = new Path("out"); if (fs.exists(outDir)) { fs.delete(outDir, true); } if (!fs.exists(inDir)) { fs.mkdirs(inDir); } String input = "The quick brown fox"; DataOutputStream file = fs.create(new Path(inDir, "part-0")); file.writeBytes(input); file.close(); jConf.setInputFormat(TextInputFormat.class); jConf.setOutputKeyClass(LongWritable.class); jConf.setOutputValueClass(Text.class); FileInputFormat.setInputPaths(jConf, inDir); FileOutputFormat.setOutputPath(jConf, outDir); // set inline cleanup queue in TT mrCluster.getTaskTrackerRunner(0).getTaskTracker().directoryCleanupThread = new InlineCleanupQueue(); JobClient jobClient = new JobClient(jConf); RunningJob job = jobClient.submitJob(jConf); JobID id = job.getID(); // wait for the job to finish job.waitForCompletion(); JobInProgress jip = mrCluster.getJobTrackerRunner().getJobTracker().getJob(id); String attemptId = jip.getTasks(TaskType.MAP)[0].getTaskStatuses()[0].getTaskID().toString(); String taskTrackerLocalDir = mrCluster.getTaskTrackerRunner(0).getLocalDir(); String taskDir = TaskTracker.getLocalTaskDir(id.toString(), attemptId); Path taskDirPath = new Path(taskTrackerLocalDir, taskDir); LOG.info("Checking task dir " + taskDirPath); FileSystem localFS = FileSystem.getLocal(jConf); assertFalse("task dir still exists", localFS.exists(taskDirPath)); } }