org.apache.hadoop.tools.distcp2.mapred.TestCopyMapper.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.tools.distcp2.mapred.TestCopyMapper.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.tools.distcp2.mapred;

import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.tools.distcp2.DistCpConstants;
import org.apache.hadoop.tools.distcp2.DistCpOptionSwitch;
import org.apache.hadoop.tools.distcp2.DistCpOptions;
import org.apache.hadoop.tools.distcp2.StubContext;
import org.apache.hadoop.tools.distcp2.util.DistCpUtils;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;

public class TestCopyMapper {
    private static final Log LOG = LogFactory.getLog(TestCopyMapper.class);
    private static List<Path> pathList = new ArrayList<Path>();
    private static int nFiles = 0;
    private static final int FILE_SIZE = 1024;

    private static MiniDFSCluster cluster;

    private static final String SOURCE_PATH = "/tmp/source";
    private static final String TARGET_PATH = "/tmp/target";

    private static Configuration configuration;

    @BeforeClass
    public static void setup() throws Exception {
        configuration = getConfigurationForCluster();
        cluster = new MiniDFSCluster(configuration, 1, true, null);
    }

    private static Configuration getConfigurationForCluster() throws IOException {
        Configuration configuration = new Configuration();
        System.setProperty("test.build.data", "target/tmp/build/TEST_COPY_MAPPER/data");
        configuration.set("hadoop.log.dir", "target/tmp");
        LOG.debug("fs.default.name  == " + configuration.get("fs.default.name"));
        LOG.debug("dfs.http.address == " + configuration.get("dfs.http.address"));
        return configuration;
    }

    private static Configuration getConfiguration() throws IOException {
        Configuration configuration = getConfigurationForCluster();
        final FileSystem fs = cluster.getFileSystem();
        Path workPath = new Path(TARGET_PATH).makeQualified(fs.getUri(), fs.getWorkingDirectory());
        configuration.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath.toString());
        configuration.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, workPath.toString());
        configuration.setBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(), false);
        configuration.setBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(), true);
        configuration.setBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), true);
        configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(), "br");
        return configuration;
    }

    private static void createSourceData() throws Exception {
        mkdirs(SOURCE_PATH + "/1");
        mkdirs(SOURCE_PATH + "/2");
        mkdirs(SOURCE_PATH + "/2/3/4");
        mkdirs(SOURCE_PATH + "/2/3");
        mkdirs(SOURCE_PATH + "/5");
        touchFile(SOURCE_PATH + "/5/6");
        mkdirs(SOURCE_PATH + "/7");
        mkdirs(SOURCE_PATH + "/7/8");
        touchFile(SOURCE_PATH + "/7/8/9");
    }

    private static void mkdirs(String path) throws Exception {
        FileSystem fileSystem = cluster.getFileSystem();
        final Path qualifiedPath = new Path(path).makeQualified(fileSystem.getUri(),
                fileSystem.getWorkingDirectory());
        pathList.add(qualifiedPath);
        fileSystem.mkdirs(qualifiedPath);
    }

    private static void touchFile(String path) throws Exception {
        FileSystem fs;
        DataOutputStream outputStream = null;
        try {
            fs = cluster.getFileSystem();
            final Path qualifiedPath = new Path(path).makeQualified(fs.getUri(), fs.getWorkingDirectory());
            final long blockSize = fs.getDefaultBlockSize() * 2;
            outputStream = fs.create(qualifiedPath, true, 0, (short) (fs.getDefaultReplication() * 2), blockSize);
            outputStream.write(new byte[FILE_SIZE]);
            pathList.add(qualifiedPath);
            ++nFiles;

            FileStatus fileStatus = fs.getFileStatus(qualifiedPath);
            System.out.println(fileStatus.getBlockSize());
            System.out.println(fileStatus.getReplication());
        } finally {
            IOUtils.cleanup(null, outputStream);
        }
    }

    @Test
    public void testRun() {
        try {
            deleteState();
            createSourceData();

            FileSystem fs = cluster.getFileSystem();
            CopyMapper copyMapper = new CopyMapper();
            StubContext stubContext = new StubContext(getConfiguration(), null, 0);
            Mapper<Text, FileStatus, Text, Text>.Context context = stubContext.getContext();
            copyMapper.setup(context);

            for (Path path : pathList) {
                copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)),
                        fs.getFileStatus(path), context);
            }

            // Check that the maps worked.
            for (Path path : pathList) {
                final Path targetPath = new Path(path.toString().replaceAll(SOURCE_PATH, TARGET_PATH));
                Assert.assertTrue(fs.exists(targetPath));
                Assert.assertTrue(fs.isFile(targetPath) == fs.isFile(path));
                Assert.assertEquals(fs.getFileStatus(path).getReplication(),
                        fs.getFileStatus(targetPath).getReplication());
                Assert.assertEquals(fs.getFileStatus(path).getBlockSize(),
                        fs.getFileStatus(targetPath).getBlockSize());
                Assert.assertTrue(
                        !fs.isFile(targetPath) || fs.getFileChecksum(targetPath).equals(fs.getFileChecksum(path)));
            }

            Assert.assertEquals(pathList.size(),
                    stubContext.getReporter().getCounter(CopyMapper.Counter.COPY).getValue());
            Assert.assertEquals(nFiles * FILE_SIZE,
                    stubContext.getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED).getValue());

            testCopyingExistingFiles(fs, copyMapper, context);
            for (Text value : stubContext.getWriter().values()) {
                Assert.assertTrue(value.toString() + " is not skipped", value.toString().startsWith("SKIP:"));
            }
        } catch (Exception e) {
            LOG.error("Unexpected exception: ", e);
            Assert.assertTrue(false);
        }
    }

    private void testCopyingExistingFiles(FileSystem fs, CopyMapper copyMapper,
            Mapper<Text, FileStatus, Text, Text>.Context context) {

        try {
            for (Path path : pathList) {
                copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)),
                        fs.getFileStatus(path), context);
            }

            Assert.assertEquals(nFiles, context.getCounter(CopyMapper.Counter.SKIP).getValue());
        } catch (Exception exception) {
            Assert.assertTrue("Caught unexpected exception:" + exception.getMessage(), false);
        }
    }

    @Test
    public void testMakeDirFailure() {
        try {
            deleteState();
            createSourceData();

            FileSystem fs = cluster.getFileSystem();
            CopyMapper copyMapper = new CopyMapper();
            StubContext stubContext = new StubContext(getConfiguration(), null, 0);
            Mapper<Text, FileStatus, Text, Text>.Context context = stubContext.getContext();

            Configuration configuration = context.getConfiguration();
            String workPath = new Path("hftp://localhost:1234/*/*/*/?/")
                    .makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString();
            configuration.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath);
            copyMapper.setup(context);

            copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), pathList.get(0))),
                    fs.getFileStatus(pathList.get(0)), context);

            Assert.assertTrue("There should have been an exception.", false);
        } catch (Exception ignore) {
        }
    }

    @Test
    public void testIgnoreFailures() {
        doTestIgnoreFailures(true);
        doTestIgnoreFailures(false);
    }

    @Test
    public void testDirToFile() {
        try {
            deleteState();
            createSourceData();

            FileSystem fs = cluster.getFileSystem();
            CopyMapper copyMapper = new CopyMapper();
            StubContext stubContext = new StubContext(getConfiguration(), null, 0);
            Mapper<Text, FileStatus, Text, Text>.Context context = stubContext.getContext();

            mkdirs(SOURCE_PATH + "/src/file");
            touchFile(TARGET_PATH + "/src/file");
            try {
                copyMapper.setup(context);
                copyMapper.map(new Text("/src/file"), fs.getFileStatus(new Path(SOURCE_PATH + "/src/file")),
                        context);
            } catch (IOException e) {
                Assert.assertTrue(e.getMessage().startsWith("Can't replace"));
            }
        } catch (Exception e) {
            LOG.error("Exception encountered ", e);
            Assert.fail("Test failed: " + e.getMessage());
        }
    }

    @Test
    public void testPreserve() {
        try {
            deleteState();
            createSourceData();

            UserGroupInformation tmpUser = UserGroupInformation.createRemoteUser("guest");

            final CopyMapper copyMapper = new CopyMapper();

            final Mapper<Text, FileStatus, Text, Text>.Context context = tmpUser
                    .doAs(new PrivilegedAction<Mapper<Text, FileStatus, Text, Text>.Context>() {
                        @Override
                        public Mapper<Text, FileStatus, Text, Text>.Context run() {
                            try {
                                StubContext stubContext = new StubContext(getConfiguration(), null, 0);
                                return stubContext.getContext();
                            } catch (Exception e) {
                                LOG.error("Exception encountered ", e);
                                throw new RuntimeException(e);
                            }
                        }
                    });

            EnumSet<DistCpOptions.FileAttribute> preserveStatus = EnumSet.allOf(DistCpOptions.FileAttribute.class);

            context.getConfiguration().set(DistCpConstants.CONF_LABEL_PRESERVE_STATUS,
                    DistCpUtils.packAttributes(preserveStatus));

            touchFile(SOURCE_PATH + "/src/file");
            mkdirs(TARGET_PATH);
            cluster.getFileSystem().setPermission(new Path(TARGET_PATH), new FsPermission((short) 511));

            final FileSystem tmpFS = tmpUser.doAs(new PrivilegedAction<FileSystem>() {
                @Override
                public FileSystem run() {
                    try {
                        return FileSystem.get(configuration);
                    } catch (IOException e) {
                        LOG.error("Exception encountered ", e);
                        Assert.fail("Test failed: " + e.getMessage());
                        throw new RuntimeException("Test ought to fail here");
                    }
                }
            });

            tmpUser.doAs(new PrivilegedAction<Integer>() {
                @Override
                public Integer run() {
                    try {
                        copyMapper.setup(context);
                        copyMapper.map(new Text("/src/file"),
                                tmpFS.getFileStatus(new Path(SOURCE_PATH + "/src/file")), context);
                        Assert.fail("Expected copy to fail");
                    } catch (AccessControlException e) {
                        Assert.assertTrue("Got exception: " + e.getMessage(), true);
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                    return null;
                }
            });
        } catch (Exception e) {
            LOG.error("Exception encountered ", e);
            Assert.fail("Test failed: " + e.getMessage());
        }
    }

    @Test
    public void testCopyReadableFiles() {
        try {
            deleteState();
            createSourceData();

            UserGroupInformation tmpUser = UserGroupInformation.createRemoteUser("guest");

            final CopyMapper copyMapper = new CopyMapper();

            final Mapper<Text, FileStatus, Text, Text>.Context context = tmpUser
                    .doAs(new PrivilegedAction<Mapper<Text, FileStatus, Text, Text>.Context>() {
                        @Override
                        public Mapper<Text, FileStatus, Text, Text>.Context run() {
                            try {
                                StubContext stubContext = new StubContext(getConfiguration(), null, 0);
                                return stubContext.getContext();
                            } catch (Exception e) {
                                LOG.error("Exception encountered ", e);
                                throw new RuntimeException(e);
                            }
                        }
                    });

            touchFile(SOURCE_PATH + "/src/file");
            mkdirs(TARGET_PATH);
            cluster.getFileSystem().setPermission(new Path(SOURCE_PATH + "/src/file"),
                    new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ));
            cluster.getFileSystem().setPermission(new Path(TARGET_PATH), new FsPermission((short) 511));

            final FileSystem tmpFS = tmpUser.doAs(new PrivilegedAction<FileSystem>() {
                @Override
                public FileSystem run() {
                    try {
                        return FileSystem.get(configuration);
                    } catch (IOException e) {
                        LOG.error("Exception encountered ", e);
                        Assert.fail("Test failed: " + e.getMessage());
                        throw new RuntimeException("Test ought to fail here");
                    }
                }
            });

            tmpUser.doAs(new PrivilegedAction<Integer>() {
                @Override
                public Integer run() {
                    try {
                        copyMapper.setup(context);
                        copyMapper.map(new Text("/src/file"),
                                tmpFS.getFileStatus(new Path(SOURCE_PATH + "/src/file")), context);
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                    return null;
                }
            });
        } catch (Exception e) {
            LOG.error("Exception encountered ", e);
            Assert.fail("Test failed: " + e.getMessage());
        }
    }

    @Test
    public void testSkipCopyNoPerms() {
        try {
            deleteState();
            createSourceData();

            UserGroupInformation tmpUser = UserGroupInformation.createRemoteUser("guest");

            final CopyMapper copyMapper = new CopyMapper();

            final StubContext stubContext = tmpUser.doAs(new PrivilegedAction<StubContext>() {
                @Override
                public StubContext run() {
                    try {
                        return new StubContext(getConfiguration(), null, 0);
                    } catch (Exception e) {
                        LOG.error("Exception encountered ", e);
                        throw new RuntimeException(e);
                    }
                }
            });

            final Mapper<Text, FileStatus, Text, Text>.Context context = stubContext.getContext();
            EnumSet<DistCpOptions.FileAttribute> preserveStatus = EnumSet.allOf(DistCpOptions.FileAttribute.class);

            context.getConfiguration().set(DistCpConstants.CONF_LABEL_PRESERVE_STATUS,
                    DistCpUtils.packAttributes(preserveStatus));

            touchFile(SOURCE_PATH + "/src/file");
            touchFile(TARGET_PATH + "/src/file");
            cluster.getFileSystem().setPermission(new Path(SOURCE_PATH + "/src/file"),
                    new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ));
            cluster.getFileSystem().setPermission(new Path(TARGET_PATH + "/src/file"),
                    new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ));

            final FileSystem tmpFS = tmpUser.doAs(new PrivilegedAction<FileSystem>() {
                @Override
                public FileSystem run() {
                    try {
                        return FileSystem.get(configuration);
                    } catch (IOException e) {
                        LOG.error("Exception encountered ", e);
                        Assert.fail("Test failed: " + e.getMessage());
                        throw new RuntimeException("Test ought to fail here");
                    }
                }
            });

            tmpUser.doAs(new PrivilegedAction<Integer>() {
                @Override
                public Integer run() {
                    try {
                        copyMapper.setup(context);
                        copyMapper.map(new Text("/src/file"),
                                tmpFS.getFileStatus(new Path(SOURCE_PATH + "/src/file")), context);
                        Assert.assertEquals(stubContext.getWriter().values().size(), 1);
                        Assert.assertTrue(stubContext.getWriter().values().get(0).toString().startsWith("SKIP"));
                        Assert.assertTrue(stubContext.getWriter().values().get(0).toString()
                                .contains(SOURCE_PATH + "/src/file"));
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                    return null;
                }
            });
        } catch (Exception e) {
            LOG.error("Exception encountered ", e);
            Assert.fail("Test failed: " + e.getMessage());
        }
    }

    @Test
    public void testFailCopyWithAccessControlException() {
        try {
            deleteState();
            createSourceData();

            UserGroupInformation tmpUser = UserGroupInformation.createRemoteUser("guest");

            final CopyMapper copyMapper = new CopyMapper();

            final StubContext stubContext = tmpUser.doAs(new PrivilegedAction<StubContext>() {
                @Override
                public StubContext run() {
                    try {
                        return new StubContext(getConfiguration(), null, 0);
                    } catch (Exception e) {
                        LOG.error("Exception encountered ", e);
                        throw new RuntimeException(e);
                    }
                }
            });

            EnumSet<DistCpOptions.FileAttribute> preserveStatus = EnumSet.allOf(DistCpOptions.FileAttribute.class);

            final Mapper<Text, FileStatus, Text, Text>.Context context = stubContext.getContext();

            context.getConfiguration().set(DistCpConstants.CONF_LABEL_PRESERVE_STATUS,
                    DistCpUtils.packAttributes(preserveStatus));

            touchFile(SOURCE_PATH + "/src/file");
            OutputStream out = cluster.getFileSystem().create(new Path(TARGET_PATH + "/src/file"));
            out.write("hello world".getBytes());
            out.close();
            cluster.getFileSystem().setPermission(new Path(SOURCE_PATH + "/src/file"),
                    new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ));
            cluster.getFileSystem().setPermission(new Path(TARGET_PATH + "/src/file"),
                    new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ));

            final FileSystem tmpFS = tmpUser.doAs(new PrivilegedAction<FileSystem>() {
                @Override
                public FileSystem run() {
                    try {
                        return FileSystem.get(configuration);
                    } catch (IOException e) {
                        LOG.error("Exception encountered ", e);
                        Assert.fail("Test failed: " + e.getMessage());
                        throw new RuntimeException("Test ought to fail here");
                    }
                }
            });

            tmpUser.doAs(new PrivilegedAction<Integer>() {
                @Override
                public Integer run() {
                    try {
                        copyMapper.setup(context);
                        copyMapper.map(new Text("/src/file"),
                                tmpFS.getFileStatus(new Path(SOURCE_PATH + "/src/file")), context);
                        Assert.fail("Didn't expect the file to be copied");
                    } catch (AccessControlException ignore) {
                    } catch (Exception e) {
                        if (e.getCause() == null || !(e.getCause() instanceof AccessControlException)) {
                            throw new RuntimeException(e);
                        }
                    }
                    return null;
                }
            });
        } catch (Exception e) {
            LOG.error("Exception encountered ", e);
            Assert.fail("Test failed: " + e.getMessage());
        }
    }

    @Test
    public void testFileToDir() {
        try {
            deleteState();
            createSourceData();

            FileSystem fs = cluster.getFileSystem();
            CopyMapper copyMapper = new CopyMapper();
            StubContext stubContext = new StubContext(getConfiguration(), null, 0);
            Mapper<Text, FileStatus, Text, Text>.Context context = stubContext.getContext();

            touchFile(SOURCE_PATH + "/src/file");
            mkdirs(TARGET_PATH + "/src/file");
            try {
                copyMapper.setup(context);
                copyMapper.map(new Text("/src/file"), fs.getFileStatus(new Path(SOURCE_PATH + "/src/file")),
                        context);
            } catch (IOException e) {
                Assert.assertTrue(e.getMessage().startsWith("Can't replace"));
            }
        } catch (Exception e) {
            LOG.error("Exception encountered ", e);
            Assert.fail("Test failed: " + e.getMessage());
        }
    }

    private void doTestIgnoreFailures(boolean ignoreFailures) {
        try {
            deleteState();
            createSourceData();

            FileSystem fs = cluster.getFileSystem();
            CopyMapper copyMapper = new CopyMapper();
            StubContext stubContext = new StubContext(getConfiguration(), null, 0);
            Mapper<Text, FileStatus, Text, Text>.Context context = stubContext.getContext();

            Configuration configuration = context.getConfiguration();
            configuration.setBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), ignoreFailures);
            configuration.setBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(), true);
            configuration.setBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(), true);
            copyMapper.setup(context);

            for (Path path : pathList) {
                final FileStatus fileStatus = fs.getFileStatus(path);
                if (!fileStatus.isDir()) {
                    fs.delete(path, true);
                    copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)), fileStatus,
                            context);
                }
            }
            if (ignoreFailures) {
                for (Text value : stubContext.getWriter().values()) {
                    Assert.assertTrue(value.toString() + " is not skipped", value.toString().startsWith("FAIL:"));
                }
            }
            Assert.assertTrue("There should have been an exception.", ignoreFailures);
        } catch (Exception e) {
            Assert.assertTrue("Unexpected exception: " + e.getMessage(), !ignoreFailures);
            e.printStackTrace();
        }
    }

    private static void deleteState() throws IOException {
        pathList.clear();
        nFiles = 0;
        cluster.getFileSystem().delete(new Path(SOURCE_PATH), true);
        cluster.getFileSystem().delete(new Path(TARGET_PATH), true);
    }

    @Test
    public void testPreserveBlockSizeAndReplication() {
        testPreserveBlockSizeAndReplicationImpl(true);
        testPreserveBlockSizeAndReplicationImpl(false);
    }

    private void testPreserveBlockSizeAndReplicationImpl(boolean preserve) {
        try {

            deleteState();
            createSourceData();

            FileSystem fs = cluster.getFileSystem();
            CopyMapper copyMapper = new CopyMapper();
            StubContext stubContext = new StubContext(getConfiguration(), null, 0);
            Mapper<Text, FileStatus, Text, Text>.Context context = stubContext.getContext();

            Configuration configuration = context.getConfiguration();
            EnumSet<DistCpOptions.FileAttribute> fileAttributes = EnumSet.noneOf(DistCpOptions.FileAttribute.class);
            if (preserve) {
                fileAttributes.add(DistCpOptions.FileAttribute.BLOCKSIZE);
                fileAttributes.add(DistCpOptions.FileAttribute.REPLICATION);
            }
            configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(),
                    DistCpUtils.packAttributes(fileAttributes));

            copyMapper.setup(context);

            for (Path path : pathList) {
                final FileStatus fileStatus = fs.getFileStatus(path);
                copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)), fileStatus,
                        context);
            }

            // Check that the block-size/replication aren't preserved.
            for (Path path : pathList) {
                final Path targetPath = new Path(path.toString().replaceAll(SOURCE_PATH, TARGET_PATH));
                final FileStatus source = fs.getFileStatus(path);
                final FileStatus target = fs.getFileStatus(targetPath);
                if (!source.isDir()) {
                    Assert.assertTrue(preserve || source.getBlockSize() != target.getBlockSize());
                    Assert.assertTrue(preserve || source.getReplication() != target.getReplication());
                    Assert.assertTrue(!preserve || source.getBlockSize() == target.getBlockSize());
                    Assert.assertTrue(!preserve || source.getReplication() == target.getReplication());
                }
            }
        } catch (Exception e) {
            Assert.assertTrue("Unexpected exception: " + e.getMessage(), false);
            e.printStackTrace();
        }
    }

    private static void changeUserGroup(String user, String group) throws IOException {
        FileSystem fs = cluster.getFileSystem();
        FsPermission changedPermission = new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL);
        for (Path path : pathList)
            if (fs.isFile(path)) {
                fs.setOwner(path, user, group);
                fs.setPermission(path, changedPermission);
            }
    }

    /**
     * If a single file is being copied to a location where the file (of the same
     * name) already exists, then the file shouldn't be skipped.
     */
    @Test
    public void testSingleFileCopy() {
        try {
            deleteState();
            touchFile(SOURCE_PATH + "/1");
            Path sourceFilePath = pathList.get(0);
            Path targetFilePath = new Path(sourceFilePath.toString().replaceAll(SOURCE_PATH, TARGET_PATH));
            touchFile(targetFilePath.toString());

            FileSystem fs = cluster.getFileSystem();
            CopyMapper copyMapper = new CopyMapper();
            StubContext stubContext = new StubContext(getConfiguration(), null, 0);
            Mapper<Text, FileStatus, Text, Text>.Context context = stubContext.getContext();

            context.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH,
                    targetFilePath.getParent().toString()); // Parent directory.
            copyMapper.setup(context);

            final FileStatus sourceFileStatus = fs.getFileStatus(sourceFilePath);

            long before = fs.getFileStatus(targetFilePath).getModificationTime();
            copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), sourceFilePath)),
                    sourceFileStatus, context);
            long after = fs.getFileStatus(targetFilePath).getModificationTime();

            Assert.assertTrue("File should have been skipped", before == after);

            context.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetFilePath.toString()); // Specify the file path.
            copyMapper.setup(context);

            before = fs.getFileStatus(targetFilePath).getModificationTime();
            try {
                Thread.sleep(2);
            } catch (Throwable ignore) {
            }
            copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), sourceFilePath)),
                    sourceFileStatus, context);
            after = fs.getFileStatus(targetFilePath).getModificationTime();

            Assert.assertTrue("File should have been overwritten.", before < after);

        } catch (Exception exception) {
            Assert.fail("Unexpected exception: " + exception.getMessage());
            exception.printStackTrace();
        }
    }

    @Test
    public void testPreserveUserGroup() {
        testPreserveUserGroupImpl(true);
        testPreserveUserGroupImpl(false);
    }

    private void testPreserveUserGroupImpl(boolean preserve) {
        try {

            deleteState();
            createSourceData();
            changeUserGroup("Michael", "Corleone");

            FileSystem fs = cluster.getFileSystem();
            CopyMapper copyMapper = new CopyMapper();
            StubContext stubContext = new StubContext(getConfiguration(), null, 0);
            Mapper<Text, FileStatus, Text, Text>.Context context = stubContext.getContext();

            Configuration configuration = context.getConfiguration();
            EnumSet<DistCpOptions.FileAttribute> fileAttributes = EnumSet.noneOf(DistCpOptions.FileAttribute.class);
            if (preserve) {
                fileAttributes.add(DistCpOptions.FileAttribute.USER);
                fileAttributes.add(DistCpOptions.FileAttribute.GROUP);
                fileAttributes.add(DistCpOptions.FileAttribute.PERMISSION);
            }

            configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(),
                    DistCpUtils.packAttributes(fileAttributes));
            copyMapper.setup(context);

            for (Path path : pathList) {
                final FileStatus fileStatus = fs.getFileStatus(path);
                copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)), fileStatus,
                        context);
            }

            // Check that the user/group attributes are preserved
            // (only) as necessary.
            for (Path path : pathList) {
                final Path targetPath = new Path(path.toString().replaceAll(SOURCE_PATH, TARGET_PATH));
                final FileStatus source = fs.getFileStatus(path);
                final FileStatus target = fs.getFileStatus(targetPath);
                if (!source.isDir()) {
                    Assert.assertTrue(!preserve || source.getOwner().equals(target.getOwner()));
                    Assert.assertTrue(!preserve || source.getGroup().equals(target.getGroup()));
                    Assert.assertTrue(!preserve || source.getPermission().equals(target.getPermission()));
                    Assert.assertTrue(preserve || !source.getOwner().equals(target.getOwner()));
                    Assert.assertTrue(preserve || !source.getGroup().equals(target.getGroup()));
                    Assert.assertTrue(preserve || !source.getPermission().equals(target.getPermission()));
                    Assert.assertTrue(source.isDir() || source.getReplication() != target.getReplication());
                }
            }
        } catch (Exception e) {
            Assert.assertTrue("Unexpected exception: " + e.getMessage(), false);
            e.printStackTrace();
        }
    }
}