Example usage for org.apache.hadoop.fs FileSystem getFileStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getFileStatus.

Prototype

public abstract FileStatus getFileStatus(Path f) throws IOException;

Source Link

Document

Return a file status object that represents the path.

Usage

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

private void testCopyingExistingFiles(FileSystem fs, CopyMapper copyMapper,
        Mapper<Text, FileStatus, NullWritable, Text>.Context context) {

    try {/*from w  w  w.j  a  v a  2s.  co  m*/
        for (Path path : pathList) {
            copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)),
                    fs.getFileStatus(path), context);
        }

        Assert.assertEquals(nFiles, context.getCounter(CopyMapper.Counter.PATHS_SKIPPED).getValue());
    } catch (Exception exception) {
        Assert.assertTrue("Caught unexpected exception:" + exception.getMessage(), false);
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

@Test
public void testMakeDirFailure() {
    try {//from w ww  . jav a2s  .co  m
        deleteState();
        createSourceData();

        FileSystem fs = cluster.getFileSystem();
        CopyMapper copyMapper = new CopyMapper();
        StatusReporter reporter = new StubStatusReporter();
        InMemoryWriter writer = new InMemoryWriter();
        Mapper<Text, FileStatus, NullWritable, Text>.Context context = getMapperContext(copyMapper, reporter,
                writer);

        Configuration configuration = context.getConfiguration();
        String workPath = new Path("hftp://localhost:1234/*/*/*/?/").makeQualified(fs).toString();
        configuration.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workPath);
        copyMapper.setup(context);

        copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), pathList.get(0))),
                fs.getFileStatus(pathList.get(0)), context);

        Assert.assertTrue("There should have been an exception.", false);
    } catch (Exception ignore) {
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

@Test
public void testDirToFile() {
    try {//from  w ww .  ja  v  a2  s .c om
        deleteState();
        createSourceData();

        FileSystem fs = cluster.getFileSystem();
        CopyMapper copyMapper = new CopyMapper();
        StatusReporter reporter = new StubStatusReporter();
        InMemoryWriter writer = new InMemoryWriter();
        Mapper<Text, FileStatus, NullWritable, Text>.Context context = getMapperContext(copyMapper, reporter,
                writer);

        mkdirs(SOURCE_PATH + "/src/file");
        touchFile(TARGET_PATH + "/src/file");
        try {
            copyMapper.setup(context);
            copyMapper.map(new Text("/src/file"), fs.getFileStatus(new Path(SOURCE_PATH + "/src/file")),
                    context);
        } catch (IOException e) {
            Assert.assertTrue(e.getMessage().startsWith("Can't replace"));
        }
    } catch (Exception e) {
        LOG.error("Exception encountered ", e);
        Assert.fail("Test failed: " + e.getMessage());
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

@Test
public void testPreserve() {
    try {//from   w  ww  .  j  a  v  a2  s .c o  m
        deleteState();
        createSourceData();

        UserGroupInformation tmpUser = UserGroupInformation.createRemoteUser("guest");

        final CopyMapper copyMapper = new CopyMapper();

        final Mapper<Text, FileStatus, NullWritable, Text>.Context context = tmpUser
                .doAs(new PrivilegedAction<Mapper<Text, FileStatus, NullWritable, Text>.Context>() {
                    @Override
                    public Mapper<Text, FileStatus, NullWritable, Text>.Context run() {
                        try {
                            StatusReporter reporter = new StubStatusReporter();
                            InMemoryWriter writer = new InMemoryWriter();
                            return getMapperContext(copyMapper, reporter, writer);
                        } catch (Exception e) {
                            LOG.error("Exception encountered ", e);
                            throw new RuntimeException(e);
                        }
                    }
                });

        EnumSet<DistCpOptions.FileAttribute> preserveStatus = EnumSet.allOf(DistCpOptions.FileAttribute.class);

        context.getConfiguration().set(DistCpConstants.CONF_LABEL_PRESERVE_STATUS,
                DistCpUtils.packAttributes(preserveStatus));

        touchFile(SOURCE_PATH + "/src/file.gz");
        mkdirs(TARGET_PATH);
        cluster.getFileSystem().setPermission(new Path(TARGET_PATH), new FsPermission((short) 511));

        final FileSystem tmpFS = tmpUser.doAs(new PrivilegedAction<FileSystem>() {
            @Override
            public FileSystem run() {
                try {
                    return FileSystem.get(configuration);
                } catch (IOException e) {
                    LOG.error("Exception encountered ", e);
                    Assert.fail("Test failed: " + e.getMessage());
                    throw new RuntimeException("Test ought to fail here");
                }
            }
        });

        tmpUser.doAs(new PrivilegedAction<Integer>() {
            @Override
            public Integer run() {
                try {
                    copyMapper.setup(context);
                    copyMapper.map(new Text("/src/file.gz"),
                            tmpFS.getFileStatus(new Path(SOURCE_PATH + "/src/file.gz")), context);
                    Assert.fail("Expected copy to fail");
                } catch (AccessControlException e) {
                    Assert.assertTrue("Got exception: " + e.getMessage(), true);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
                return null;
            }
        });
    } catch (Exception e) {
        LOG.error("Exception encountered ", e);
        Assert.fail("Test failed: " + e.getMessage());
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

@Test
public void testCopyReadableFiles() {
    try {/*from ww  w . ja  va  2  s  . c  om*/
        deleteState();
        createSourceData();

        UserGroupInformation tmpUser = UserGroupInformation.createRemoteUser("guest");

        final CopyMapper copyMapper = new CopyMapper();

        final Mapper<Text, FileStatus, NullWritable, Text>.Context context = tmpUser
                .doAs(new PrivilegedAction<Mapper<Text, FileStatus, NullWritable, Text>.Context>() {
                    @Override
                    public Mapper<Text, FileStatus, NullWritable, Text>.Context run() {
                        try {
                            StatusReporter reporter = new StubStatusReporter();
                            InMemoryWriter writer = new InMemoryWriter();
                            return getMapperContext(copyMapper, reporter, writer);
                        } catch (Exception e) {
                            LOG.error("Exception encountered ", e);
                            throw new RuntimeException(e);
                        }
                    }
                });

        touchFile(SOURCE_PATH + "/src/file.gz");
        mkdirs(TARGET_PATH);
        cluster.getFileSystem().setPermission(new Path(SOURCE_PATH + "/src/file.gz"),
                new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ));
        cluster.getFileSystem().setPermission(new Path(TARGET_PATH), new FsPermission((short) 511));

        final FileSystem tmpFS = tmpUser.doAs(new PrivilegedAction<FileSystem>() {
            @Override
            public FileSystem run() {
                try {
                    return FileSystem.get(configuration);
                } catch (IOException e) {
                    LOG.error("Exception encountered ", e);
                    Assert.fail("Test failed: " + e.getMessage());
                    throw new RuntimeException("Test ought to fail here");
                }
            }
        });

        tmpUser.doAs(new PrivilegedAction<Integer>() {
            @Override
            public Integer run() {
                try {
                    copyMapper.setup(context);
                    copyMapper.map(new Text("/src/file.gz"),
                            tmpFS.getFileStatus(new Path(SOURCE_PATH + "/src/file.gz")), context);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
                return null;
            }
        });
    } catch (Exception e) {
        LOG.error("Exception encountered ", e);
        Assert.fail("Test failed: " + e.getMessage());
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

@Test
public void testSkipCopyNoPerms() {
    try {//from w  w  w  .j a  va  2 s .c  o m
        deleteState();
        createSourceData();

        final InMemoryWriter writer = new InMemoryWriter();
        UserGroupInformation tmpUser = UserGroupInformation.createRemoteUser("guest");

        final CopyMapper copyMapper = new CopyMapper();

        final Mapper<Text, FileStatus, NullWritable, Text>.Context context = tmpUser
                .doAs(new PrivilegedAction<Mapper<Text, FileStatus, NullWritable, Text>.Context>() {
                    @Override
                    public Mapper<Text, FileStatus, NullWritable, Text>.Context run() {
                        try {
                            StatusReporter reporter = new StubStatusReporter();
                            return getMapperContext(copyMapper, reporter, writer);
                        } catch (Exception e) {
                            LOG.error("Exception encountered ", e);
                            throw new RuntimeException(e);
                        }
                    }
                });

        EnumSet<DistCpOptions.FileAttribute> preserveStatus = EnumSet.allOf(DistCpOptions.FileAttribute.class);

        context.getConfiguration().set(DistCpConstants.CONF_LABEL_PRESERVE_STATUS,
                DistCpUtils.packAttributes(preserveStatus));

        touchFile(SOURCE_PATH + "/src/file.gz");
        touchFile(TARGET_PATH + "/src/file.gz");
        cluster.getFileSystem().setPermission(new Path(SOURCE_PATH + "/src/file.gz"),
                new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ));
        cluster.getFileSystem().setPermission(new Path(TARGET_PATH + "/src/file.gz"),
                new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ));

        final FileSystem tmpFS = tmpUser.doAs(new PrivilegedAction<FileSystem>() {
            @Override
            public FileSystem run() {
                try {
                    return FileSystem.get(configuration);
                } catch (IOException e) {
                    LOG.error("Exception encountered ", e);
                    Assert.fail("Test failed: " + e.getMessage());
                    throw new RuntimeException("Test ought to fail here");
                }
            }
        });

        tmpUser.doAs(new PrivilegedAction<Integer>() {
            @Override
            public Integer run() {
                try {
                    copyMapper.setup(context);
                    copyMapper.map(new Text("/src/file.gz"),
                            tmpFS.getFileStatus(new Path(SOURCE_PATH + "/src/file.gz")), context);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
                return null;
            }
        });
    } catch (Exception e) {
        LOG.error("Exception encountered ", e);
        Assert.fail("Test failed: " + e.getMessage());
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

@Test
public void testFailCopyWithAccessControlException() {
    try {//from  www .  j av  a 2s . com
        deleteState();
        createSourceData();

        final InMemoryWriter writer = new InMemoryWriter();
        UserGroupInformation tmpUser = UserGroupInformation.createRemoteUser("guest");

        final CopyMapper copyMapper = new CopyMapper();

        final Mapper<Text, FileStatus, NullWritable, Text>.Context context = tmpUser
                .doAs(new PrivilegedAction<Mapper<Text, FileStatus, NullWritable, Text>.Context>() {
                    @Override
                    public Mapper<Text, FileStatus, NullWritable, Text>.Context run() {
                        try {
                            StatusReporter reporter = new StubStatusReporter();
                            return getMapperContext(copyMapper, reporter, writer);
                        } catch (Exception e) {
                            LOG.error("Exception encountered ", e);
                            throw new RuntimeException(e);
                        }
                    }
                });

        EnumSet<DistCpOptions.FileAttribute> preserveStatus = EnumSet.allOf(DistCpOptions.FileAttribute.class);

        context.getConfiguration().set(DistCpConstants.CONF_LABEL_PRESERVE_STATUS,
                DistCpUtils.packAttributes(preserveStatus));

        touchFile(SOURCE_PATH + "/src/file");
        OutputStream out = cluster.getFileSystem().create(new Path(TARGET_PATH + "/src/file"));
        out.write("hello world".getBytes());
        out.close();
        cluster.getFileSystem().setPermission(new Path(SOURCE_PATH + "/src/file"),
                new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ));
        cluster.getFileSystem().setPermission(new Path(TARGET_PATH + "/src/file"),
                new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ));

        final FileSystem tmpFS = tmpUser.doAs(new PrivilegedAction<FileSystem>() {
            @Override
            public FileSystem run() {
                try {
                    return FileSystem.get(configuration);
                } catch (IOException e) {
                    LOG.error("Exception encountered ", e);
                    Assert.fail("Test failed: " + e.getMessage());
                    throw new RuntimeException("Test ought to fail here");
                }
            }
        });

        tmpUser.doAs(new PrivilegedAction<Integer>() {
            @Override
            public Integer run() {
                try {
                    copyMapper.setup(context);
                    copyMapper.map(new Text("/src/file"),
                            tmpFS.getFileStatus(new Path(SOURCE_PATH + "/src/file")), context);
                    Assert.fail("Didn't expect the file to be copied");
                } catch (AccessControlException ignore) {
                } catch (Exception e) {
                    if (e.getCause() == null || !(e.getCause() instanceof AccessControlException)) {
                        throw new RuntimeException(e);
                    }
                }
                return null;
            }
        });
    } catch (Exception e) {
        LOG.error("Exception encountered ", e);
        Assert.fail("Test failed: " + e.getMessage());
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

@Test
public void testFileToDir() {
    try {/* www  . ja  va  2 s  .c  om*/
        deleteState();
        createSourceData();

        FileSystem fs = cluster.getFileSystem();
        CopyMapper copyMapper = new CopyMapper();
        StatusReporter reporter = new StubStatusReporter();
        InMemoryWriter writer = new InMemoryWriter();
        Mapper<Text, FileStatus, NullWritable, Text>.Context context = getMapperContext(copyMapper, reporter,
                writer);

        touchFile(SOURCE_PATH + "/src/file");
        mkdirs(TARGET_PATH + "/src/file");
        try {
            copyMapper.setup(context);
            copyMapper.map(new Text("/src/file"), fs.getFileStatus(new Path(SOURCE_PATH + "/src/file")),
                    context);
        } catch (IOException e) {
            Assert.assertTrue(e.getMessage().startsWith("Can't replace"));
        }
    } catch (Exception e) {
        LOG.error("Exception encountered ", e);
        Assert.fail("Test failed: " + e.getMessage());
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

private void doTestIgnoreFailures(boolean ignoreFailures) {
    try {//  w ww. ja v a2 s . com
        deleteState();
        createSourceData();

        FileSystem fs = cluster.getFileSystem();
        CopyMapper copyMapper = new CopyMapper();
        StatusReporter reporter = new StubStatusReporter();
        InMemoryWriter writer = new InMemoryWriter();
        Mapper<Text, FileStatus, NullWritable, Text>.Context context = getMapperContext(copyMapper, reporter,
                writer);

        Configuration configuration = context.getConfiguration();
        configuration.setBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), ignoreFailures);
        configuration.setBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(), true);
        configuration.setBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(), true);
        copyMapper.setup(context);

        for (Path path : pathList) {
            final FileStatus fileStatus = fs.getFileStatus(path);
            if (!fileStatus.isDir()) {
                fs.delete(path, true);
                copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)), fileStatus,
                        context);
            }
        }
        if (ignoreFailures) {
            for (Text value : writer.values()) {
                Assert.assertTrue(value.toString() + " is not skipped", value.toString().startsWith("FAIL:"));
            }
        }
        Assert.assertTrue("There should have been an exception.", ignoreFailures);
    } catch (Exception e) {
        Assert.assertTrue("Unexpected exception: " + e.getMessage(), !ignoreFailures);
        e.printStackTrace();
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

private void testPreserveBlockSizeAndReplicationImpl(boolean preserve) {
    try {/*from  w ww .ja v  a 2s .c  o  m*/

        deleteState();
        createSourceData();

        FileSystem fs = cluster.getFileSystem();
        CopyMapper copyMapper = new CopyMapper();
        StatusReporter reporter = new StubStatusReporter();
        InMemoryWriter writer = new InMemoryWriter();
        Mapper<Text, FileStatus, NullWritable, Text>.Context context = getMapperContext(copyMapper, reporter,
                writer);

        Configuration configuration = context.getConfiguration();
        EnumSet<DistCpOptions.FileAttribute> fileAttributes = EnumSet.noneOf(DistCpOptions.FileAttribute.class);
        if (preserve) {
            fileAttributes.add(DistCpOptions.FileAttribute.BLOCKSIZE);
            fileAttributes.add(DistCpOptions.FileAttribute.REPLICATION);
        }
        configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(),
                DistCpUtils.packAttributes(fileAttributes));

        copyMapper.setup(context);

        for (Path path : pathList) {
            final FileStatus fileStatus = fs.getFileStatus(path);
            copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)), fileStatus,
                    context);
        }

        // Check that the block-size/replication aren't preserved.
        for (Path path : pathList) {
            final Path targetPath = new Path(path.toString().replaceAll(SOURCE_PATH, TARGET_PATH));
            final FileStatus source = fs.getFileStatus(path);
            final FileStatus target = fs.getFileStatus(targetPath);
            if (!source.isDir()) {
                Assert.assertTrue(preserve || source.getBlockSize() != target.getBlockSize());
                Assert.assertTrue(preserve || source.getReplication() != target.getReplication());
                Assert.assertTrue(!preserve || source.getBlockSize() == target.getBlockSize());
                Assert.assertTrue(!preserve || source.getReplication() == target.getReplication());
            }
        }
    } catch (Exception e) {
        Assert.assertTrue("Unexpected exception: " + e.getMessage(), false);
        e.printStackTrace();
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »