org.apache.hadoop.mapreduce.lib.output.TestFileOutputCommitter.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.mapreduce.lib.output.TestFileOutputCommitter.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.mapreduce.lib.output;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

import junit.framework.TestCase;
import org.junit.Assert;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.MapFile;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.task.JobContextImpl;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;

@SuppressWarnings("unchecked")
public class TestFileOutputCommitter extends TestCase {
    private static final Path outDir = new Path(
            System.getProperty("test.build.data", System.getProperty("java.io.tmpdir")),
            TestFileOutputCommitter.class.getName());

    private final static String SUB_DIR = "SUB_DIR";
    private final static Path OUT_SUB_DIR = new Path(outDir, SUB_DIR);

    private static final Log LOG = LogFactory.getLog(TestFileOutputCommitter.class);

    // A random task attempt id for testing.
    private static final String attempt = "attempt_200707121733_0001_m_000000_0";
    private static final String partFile = "part-m-00000";
    private static final TaskAttemptID taskID = TaskAttemptID.forName(attempt);

    private static final String attempt1 = "attempt_200707121733_0001_m_000001_0";
    private static final TaskAttemptID taskID1 = TaskAttemptID.forName(attempt1);

    private Text key1 = new Text("key1");
    private Text key2 = new Text("key2");
    private Text val1 = new Text("val1");
    private Text val2 = new Text("val2");

    private static void cleanup() throws IOException {
        Configuration conf = new Configuration();
        FileSystem fs = outDir.getFileSystem(conf);
        fs.delete(outDir, true);
    }

    @Override
    public void setUp() throws IOException {
        cleanup();
    }

    @Override
    public void tearDown() throws IOException {
        cleanup();
    }

    private void writeOutput(RecordWriter theRecordWriter, TaskAttemptContext context)
            throws IOException, InterruptedException {
        NullWritable nullWritable = NullWritable.get();

        try {
            theRecordWriter.write(key1, val1);
            theRecordWriter.write(null, nullWritable);
            theRecordWriter.write(null, val1);
            theRecordWriter.write(nullWritable, val2);
            theRecordWriter.write(key2, nullWritable);
            theRecordWriter.write(key1, null);
            theRecordWriter.write(null, null);
            theRecordWriter.write(key2, val2);
        } finally {
            theRecordWriter.close(context);
        }
    }

    private void writeMapFileOutput(RecordWriter theRecordWriter, TaskAttemptContext context)
            throws IOException, InterruptedException {
        try {
            int key = 0;
            for (int i = 0; i < 10; ++i) {
                key = i;
                Text val = (i % 2 == 1) ? val1 : val2;
                theRecordWriter.write(new LongWritable(key), val);
            }
        } finally {
            theRecordWriter.close(context);
        }
    }

    private void testRecoveryInternal(int commitVersion, int recoveryVersion) throws Exception {
        Job job = Job.getInstance();
        FileOutputFormat.setOutputPath(job, outDir);
        Configuration conf = job.getConfiguration();
        conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
        conf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 1);
        conf.setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, commitVersion);
        JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
        TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
        FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

        // setup
        committer.setupJob(jContext);
        committer.setupTask(tContext);

        // write output
        TextOutputFormat theOutputFormat = new TextOutputFormat();
        RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(tContext);
        writeOutput(theRecordWriter, tContext);

        // do commit
        committer.commitTask(tContext);

        Path jobTempDir1 = committer.getCommittedTaskPath(tContext);
        File jtd = new File(jobTempDir1.toUri().getPath());
        if (commitVersion == 1) {
            assertTrue("Version 1 commits to temporary dir " + jtd, jtd.exists());
            validateContent(jtd);
        } else {
            assertFalse("Version 2 commits to output dir " + jtd, jtd.exists());
        }

        //now while running the second app attempt, 
        //recover the task output from first attempt
        Configuration conf2 = job.getConfiguration();
        conf2.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
        conf2.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 2);
        conf2.setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, recoveryVersion);
        JobContext jContext2 = new JobContextImpl(conf2, taskID.getJobID());
        TaskAttemptContext tContext2 = new TaskAttemptContextImpl(conf2, taskID);
        FileOutputCommitter committer2 = new FileOutputCommitter(outDir, tContext2);
        committer2.setupJob(tContext2);
        Path jobTempDir2 = committer2.getCommittedTaskPath(tContext2);
        File jtd2 = new File(jobTempDir2.toUri().getPath());

        committer2.recoverTask(tContext2);
        if (recoveryVersion == 1) {
            assertTrue("Version 1 recovers to " + jtd2, jtd2.exists());
            validateContent(jtd2);
        } else {
            assertFalse("Version 2 commits to output dir " + jtd2, jtd2.exists());
            if (commitVersion == 1) {
                assertTrue("Version 2  recovery moves to output dir from " + jtd, jtd.list().length == 0);
            }
        }

        committer2.commitJob(jContext2);
        validateContent(outDir);
        FileUtil.fullyDelete(new File(outDir.toString()));
    }

    public void testRecoveryV1() throws Exception {
        testRecoveryInternal(1, 1);
    }

    public void testRecoveryV2() throws Exception {
        testRecoveryInternal(2, 2);
    }

    public void testRecoveryUpgradeV1V2() throws Exception {
        testRecoveryInternal(1, 2);
    }

    private void validateContent(Path dir) throws IOException {
        validateContent(new File(dir.toUri().getPath()));
    }

    private void validateContent(File dir) throws IOException {
        File expectedFile = new File(dir, partFile);
        assertTrue("Could not find " + expectedFile, expectedFile.exists());
        StringBuffer expectedOutput = new StringBuffer();
        expectedOutput.append(key1).append('\t').append(val1).append("\n");
        expectedOutput.append(val1).append("\n");
        expectedOutput.append(val2).append("\n");
        expectedOutput.append(key2).append("\n");
        expectedOutput.append(key1).append("\n");
        expectedOutput.append(key2).append('\t').append(val2).append("\n");
        String output = slurp(expectedFile);
        assertEquals(output, expectedOutput.toString());
    }

    private void validateMapFileOutputContent(FileSystem fs, Path dir) throws IOException {
        // map output is a directory with index and data files
        Path expectedMapDir = new Path(dir, partFile);
        assert (fs.getFileStatus(expectedMapDir).isDirectory());
        FileStatus[] files = fs.listStatus(expectedMapDir);
        int fileCount = 0;
        boolean dataFileFound = false;
        boolean indexFileFound = false;
        for (FileStatus f : files) {
            if (f.isFile()) {
                ++fileCount;
                if (f.getPath().getName().equals(MapFile.INDEX_FILE_NAME)) {
                    indexFileFound = true;
                } else if (f.getPath().getName().equals(MapFile.DATA_FILE_NAME)) {
                    dataFileFound = true;
                }
            }
        }
        assert (fileCount > 0);
        assert (dataFileFound && indexFileFound);
    }

    private void testCommitterInternal(int version) throws Exception {
        Job job = Job.getInstance();
        FileOutputFormat.setOutputPath(job, outDir);
        Configuration conf = job.getConfiguration();
        conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
        conf.setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, version);
        JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
        TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
        FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

        // setup
        committer.setupJob(jContext);
        committer.setupTask(tContext);

        // write output
        TextOutputFormat theOutputFormat = new TextOutputFormat();
        RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(tContext);
        writeOutput(theRecordWriter, tContext);

        // do commit
        committer.commitTask(tContext);
        committer.commitJob(jContext);

        // validate output
        validateContent(outDir);
        FileUtil.fullyDelete(new File(outDir.toString()));
    }

    public void testCommitterV1() throws Exception {
        testCommitterInternal(1);
    }

    public void testCommitterV2() throws Exception {
        testCommitterInternal(2);
    }

    public void testCommitterWithDuplicatedCommitV1() throws Exception {
        testCommitterWithDuplicatedCommitInternal(1);
    }

    public void testCommitterWithDuplicatedCommitV2() throws Exception {
        testCommitterWithDuplicatedCommitInternal(2);
    }

    private void testCommitterWithDuplicatedCommitInternal(int version) throws Exception {
        Job job = Job.getInstance();
        FileOutputFormat.setOutputPath(job, outDir);
        Configuration conf = job.getConfiguration();
        conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
        conf.setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, version);
        JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
        TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
        FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

        // setup
        committer.setupJob(jContext);
        committer.setupTask(tContext);

        // write output
        TextOutputFormat theOutputFormat = new TextOutputFormat();
        RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(tContext);
        writeOutput(theRecordWriter, tContext);

        // do commit
        committer.commitTask(tContext);
        committer.commitJob(jContext);

        // validate output
        validateContent(outDir);

        // commit job again on a successful commit job.
        try {
            committer.commitJob(jContext);
            if (version == 1) {
                Assert.fail("Duplicate commit success: wrong behavior for version 1.");
            }
        } catch (IOException e) {
            if (version == 2) {
                Assert.fail("Duplicate commit failed: wrong behavior for version 2.");
            }
        }
        FileUtil.fullyDelete(new File(outDir.toString()));
    }

    public void testCommitterWithFailureV1() throws Exception {
        testCommitterWithFailureInternal(1, 1);
        testCommitterWithFailureInternal(1, 2);
    }

    public void testCommitterWithFailureV2() throws Exception {
        testCommitterWithFailureInternal(2, 1);
        testCommitterWithFailureInternal(2, 2);
    }

    private void testCommitterWithFailureInternal(int version, int maxAttempts) throws Exception {
        Job job = Job.getInstance();
        FileOutputFormat.setOutputPath(job, outDir);
        Configuration conf = job.getConfiguration();
        conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
        conf.setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, version);
        conf.setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_FAILURE_ATTEMPTS, maxAttempts);

        JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
        TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
        FileOutputCommitter committer = new CommitterWithFailedThenSucceed(outDir, tContext);

        // setup
        committer.setupJob(jContext);
        committer.setupTask(tContext);

        // write output
        TextOutputFormat theOutputFormat = new TextOutputFormat();
        RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(tContext);
        writeOutput(theRecordWriter, tContext);

        // do commit
        committer.commitTask(tContext);

        try {
            committer.commitJob(jContext);
            // (1,1), (1,2), (2,1) shouldn't reach to here.
            if (version == 1 || maxAttempts <= 1) {
                Assert.fail("Commit successful: wrong behavior for version 1.");
            }
        } catch (IOException e) {
            // (2,2) shouldn't reach to here.
            if (version == 2 && maxAttempts > 2) {
                Assert.fail("Commit failed: wrong behavior for version 2.");
            }
        }

        FileUtil.fullyDelete(new File(outDir.toString()));
    }

    public void testCommitterRepeatableV1() throws Exception {
        testCommitterRetryInternal(1);
    }

    public void testCommitterRepeatableV2() throws Exception {
        testCommitterRetryInternal(2);
    }

    // retry committer for 2 times.
    private void testCommitterRetryInternal(int version) throws Exception {
        Job job = Job.getInstance();
        FileOutputFormat.setOutputPath(job, outDir);
        Configuration conf = job.getConfiguration();
        conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
        conf.setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, version);
        // only attempt for 1 time.
        conf.setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_FAILURE_ATTEMPTS, 1);

        JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
        TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
        FileOutputCommitter committer = new CommitterWithFailedThenSucceed(outDir, tContext);

        // setup
        committer.setupJob(jContext);
        committer.setupTask(tContext);

        // write output
        TextOutputFormat theOutputFormat = new TextOutputFormat();
        RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(tContext);
        writeOutput(theRecordWriter, tContext);

        // do commit
        committer.commitTask(tContext);

        try {
            committer.commitJob(jContext);
            Assert.fail("Commit successful: wrong behavior for the first time " + "commit.");
        } catch (IOException e) {
            // commit again.
            try {
                committer.commitJob(jContext);
                // version 1 shouldn't reach to here.
                if (version == 1) {
                    Assert.fail("Commit successful after retry: wrong behavior for " + "version 1.");
                }
            } catch (FileNotFoundException ex) {
                if (version == 2) {
                    Assert.fail("Commit failed after retry: wrong behavior for" + " version 2.");
                }
                assertTrue(ex.getMessage()
                        .contains(committer.getJobAttemptPath(jContext).toString() + " does not exist"));
            }
        }

        FileUtil.fullyDelete(new File(outDir.toString()));
    }

    private void testMapFileOutputCommitterInternal(int version) throws Exception {
        Job job = Job.getInstance();
        FileOutputFormat.setOutputPath(job, outDir);
        Configuration conf = job.getConfiguration();
        conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
        conf.setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, version);
        JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
        TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
        FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

        // setup
        committer.setupJob(jContext);
        committer.setupTask(tContext);

        // write output
        MapFileOutputFormat theOutputFormat = new MapFileOutputFormat();
        RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(tContext);
        writeMapFileOutput(theRecordWriter, tContext);

        // do commit
        committer.commitTask(tContext);
        committer.commitJob(jContext);

        // Ensure getReaders call works and also ignores
        // hidden filenames (_ or . prefixes)
        try {
            MapFileOutputFormat.getReaders(outDir, conf);
        } catch (Exception e) {
            fail("Fail to read from MapFileOutputFormat: " + e);
            e.printStackTrace();
        }

        // validate output
        validateMapFileOutputContent(FileSystem.get(job.getConfiguration()), outDir);
        FileUtil.fullyDelete(new File(outDir.toString()));
    }

    public void testMapFileOutputCommitterV1() throws Exception {
        testMapFileOutputCommitterInternal(1);
    }

    public void testMapFileOutputCommitterV2() throws Exception {
        testMapFileOutputCommitterInternal(2);
    }

    public void testInvalidVersionNumber() throws IOException {
        Job job = Job.getInstance();
        FileOutputFormat.setOutputPath(job, outDir);
        Configuration conf = job.getConfiguration();
        conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
        conf.setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, 3);
        TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
        try {
            new FileOutputCommitter(outDir, tContext);
            fail("should've thrown an exception!");
        } catch (IOException e) {
            //test passed
        }
    }

    private void testAbortInternal(int version) throws IOException, InterruptedException {
        Job job = Job.getInstance();
        FileOutputFormat.setOutputPath(job, outDir);
        Configuration conf = job.getConfiguration();
        conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
        conf.setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, version);
        JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
        TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
        FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

        // do setup
        committer.setupJob(jContext);
        committer.setupTask(tContext);

        // write output
        TextOutputFormat theOutputFormat = new TextOutputFormat();
        RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(tContext);
        writeOutput(theRecordWriter, tContext);

        // do abort
        committer.abortTask(tContext);
        File expectedFile = new File(new Path(committer.getWorkPath(), partFile).toString());
        assertFalse("task temp dir still exists", expectedFile.exists());

        committer.abortJob(jContext, JobStatus.State.FAILED);
        expectedFile = new File(new Path(outDir, FileOutputCommitter.PENDING_DIR_NAME).toString());
        assertFalse("job temp dir still exists", expectedFile.exists());
        assertEquals("Output directory not empty", 0, new File(outDir.toString()).listFiles().length);
        FileUtil.fullyDelete(new File(outDir.toString()));
    }

    public void testAbortV1() throws IOException, InterruptedException {
        testAbortInternal(1);
    }

    public void testAbortV2() throws IOException, InterruptedException {
        testAbortInternal(2);
    }

    public static class FakeFileSystem extends RawLocalFileSystem {
        public FakeFileSystem() {
            super();
        }

        public URI getUri() {
            return URI.create("faildel:///");
        }

        @Override
        public boolean delete(Path p, boolean recursive) throws IOException {
            throw new IOException("fake delete failed");
        }
    }

    private void testFailAbortInternal(int version) throws IOException, InterruptedException {
        Job job = Job.getInstance();
        Configuration conf = job.getConfiguration();
        conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "faildel:///");
        conf.setClass("fs.faildel.impl", FakeFileSystem.class, FileSystem.class);
        conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
        conf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 1);
        conf.setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, version);
        FileOutputFormat.setOutputPath(job, outDir);
        JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
        TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
        FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

        // do setup
        committer.setupJob(jContext);
        committer.setupTask(tContext);

        // write output
        TextOutputFormat<?, ?> theOutputFormat = new TextOutputFormat();
        RecordWriter<?, ?> theRecordWriter = theOutputFormat.getRecordWriter(tContext);
        writeOutput(theRecordWriter, tContext);

        // do abort
        Throwable th = null;
        try {
            committer.abortTask(tContext);
        } catch (IOException ie) {
            th = ie;
        }
        assertNotNull(th);
        assertTrue(th instanceof IOException);
        assertTrue(th.getMessage().contains("fake delete failed"));
        Path jtd = committer.getJobAttemptPath(jContext);
        File jobTmpDir = new File(jtd.toUri().getPath());
        Path ttd = committer.getTaskAttemptPath(tContext);
        File taskTmpDir = new File(ttd.toUri().getPath());
        File expectedFile = new File(taskTmpDir, partFile);
        assertTrue(expectedFile + " does not exists", expectedFile.exists());

        th = null;
        try {
            committer.abortJob(jContext, JobStatus.State.FAILED);
        } catch (IOException ie) {
            th = ie;
        }
        assertNotNull(th);
        assertTrue(th instanceof IOException);
        assertTrue(th.getMessage().contains("fake delete failed"));
        assertTrue("job temp dir does not exists", jobTmpDir.exists());
        FileUtil.fullyDelete(new File(outDir.toString()));
    }

    public void testFailAbortV1() throws Exception {
        testFailAbortInternal(1);
    }

    public void testFailAbortV2() throws Exception {
        testFailAbortInternal(2);
    }

    static class RLFS extends RawLocalFileSystem {
        private final ThreadLocal<Boolean> needNull = new ThreadLocal<Boolean>() {
            @Override
            protected Boolean initialValue() {
                return true;
            }
        };

        public RLFS() {
        }

        @Override
        public FileStatus getFileStatus(Path f) throws IOException {
            if (needNull.get() && OUT_SUB_DIR.toUri().getPath().equals(f.toUri().getPath())) {
                needNull.set(false); // lie once per thread
                return null;
            }
            return super.getFileStatus(f);
        }
    }

    private void testConcurrentCommitTaskWithSubDir(int version) throws Exception {
        final Job job = Job.getInstance();
        FileOutputFormat.setOutputPath(job, outDir);
        final Configuration conf = job.getConfiguration();
        conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
        conf.setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, version);

        conf.setClass("fs.file.impl", RLFS.class, FileSystem.class);
        FileSystem.closeAll();

        final JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
        final FileOutputCommitter amCommitter = new FileOutputCommitter(outDir, jContext);
        amCommitter.setupJob(jContext);

        final TaskAttemptContext[] taCtx = new TaskAttemptContextImpl[2];
        taCtx[0] = new TaskAttemptContextImpl(conf, taskID);
        taCtx[1] = new TaskAttemptContextImpl(conf, taskID1);

        final TextOutputFormat[] tof = new TextOutputFormat[2];
        for (int i = 0; i < tof.length; i++) {
            tof[i] = new TextOutputFormat() {
                @Override
                public Path getDefaultWorkFile(TaskAttemptContext context, String extension) throws IOException {
                    final FileOutputCommitter foc = (FileOutputCommitter) getOutputCommitter(context);
                    return new Path(new Path(foc.getWorkPath(), SUB_DIR),
                            getUniqueFile(context, getOutputName(context), extension));
                }
            };
        }

        final ExecutorService executor = Executors.newFixedThreadPool(2);
        try {
            for (int i = 0; i < taCtx.length; i++) {
                final int taskIdx = i;
                executor.submit(new Callable<Void>() {
                    @Override
                    public Void call() throws IOException, InterruptedException {
                        final OutputCommitter outputCommitter = tof[taskIdx].getOutputCommitter(taCtx[taskIdx]);
                        outputCommitter.setupTask(taCtx[taskIdx]);
                        final RecordWriter rw = tof[taskIdx].getRecordWriter(taCtx[taskIdx]);
                        writeOutput(rw, taCtx[taskIdx]);
                        outputCommitter.commitTask(taCtx[taskIdx]);
                        return null;
                    }
                });
            }
        } finally {
            executor.shutdown();
            while (!executor.awaitTermination(1, TimeUnit.SECONDS)) {
                LOG.info("Awaiting thread termination!");
            }
        }

        amCommitter.commitJob(jContext);
        final RawLocalFileSystem lfs = new RawLocalFileSystem();
        lfs.setConf(conf);
        assertFalse("Must not end up with sub_dir/sub_dir", lfs.exists(new Path(OUT_SUB_DIR, SUB_DIR)));

        // validate output
        validateContent(OUT_SUB_DIR);
        FileUtil.fullyDelete(new File(outDir.toString()));
    }

    public void testConcurrentCommitTaskWithSubDirV1() throws Exception {
        testConcurrentCommitTaskWithSubDir(1);
    }

    public void testConcurrentCommitTaskWithSubDirV2() throws Exception {
        testConcurrentCommitTaskWithSubDir(2);
    }

    public static String slurp(File f) throws IOException {
        int len = (int) f.length();
        byte[] buf = new byte[len];
        FileInputStream in = new FileInputStream(f);
        String contents = null;
        try {
            in.read(buf, 0, len);
            contents = new String(buf, "UTF-8");
        } finally {
            in.close();
        }
        return contents;
    }

    /**
     * The class provides a overrided implementation of commitJobInternal which
     * causes the commit failed for the first time then succeed.
     */
    public static class CommitterWithFailedThenSucceed extends FileOutputCommitter {
        boolean firstTimeFail = true;

        public CommitterWithFailedThenSucceed(Path outputPath, JobContext context) throws IOException {
            super(outputPath, context);
        }

        @Override
        protected void commitJobInternal(JobContext context) throws IOException {
            super.commitJobInternal(context);
            if (firstTimeFail) {
                firstTimeFail = false;
                throw new IOException();
            } else {
                // succeed then, nothing to do
            }
        }
    }

}