Example usage for org.apache.hadoop.fs FileSystem getFileStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getFileStatus.

Prototype

public abstract FileStatus getFileStatus(Path f) throws IOException;

Source Link

Document

Return a file status object that represents the path.

Usage

From source file:gobblin.source.extractor.utils.ProxyFsInput.java

License:Apache License

public ProxyFsInput(Path path, FileSystem fs) throws IOException {
    this.len = fs.getFileStatus(path).getLen();
    this.stream = fs.open(path);
}

From source file:gobblin.util.AvroUtils.java

License:Apache License

private static List<FileStatus> getDirectorySchemaHelper(Path directory, FileSystem fs) throws IOException {
    List<FileStatus> files = Lists.newArrayList();
    if (fs.exists(directory)) {
        getAllNestedAvroFiles(fs.getFileStatus(directory), files, fs);
        if (files.size() > 0) {
            Collections.sort(files, FileListUtils.LATEST_MOD_TIME_ORDER);
        }/*  w ww  .j  ava 2 s.  c om*/
    }
    return files;
}

From source file:gobblin.util.commit.DeleteFileCommitStep.java

License:Apache License

public DeleteFileCommitStep(FileSystem fs, Path path, Properties properties) throws IOException {
    this(fs, Lists.newArrayList(fs.getFileStatus(path)), properties, Optional.<Path>absent());
}

From source file:gobblin.util.commit.DeleteFileCommitStep.java

License:Apache License

private static List<FileStatus> toFileStatus(FileSystem fs, Collection<Path> paths) throws IOException {
    List<FileStatus> fileStatuses = Lists.newArrayList();
    for (Path path : paths) {
        fileStatuses.add(fs.getFileStatus(path));
    }//  w ww  . j a v a 2s  . co  m
    return fileStatuses;
}

From source file:gobblin.util.FileListUtils.java

License:Apache License

/**
 * Helper method to list out all files under a specified path. The specified {@link PathFilter} is treated as a file
 * filter, that is it is only applied to file {@link Path}s.
 *///from  w  ww  .j a  v a 2 s  .c o  m
public static List<FileStatus> listFilesRecursively(FileSystem fs, Path path, PathFilter fileFilter)
        throws IOException {
    return listFilesRecursivelyHelper(fs, Lists.<FileStatus>newArrayList(), fs.getFileStatus(path), fileFilter,
            false);
}

From source file:gobblin.util.FileListUtils.java

License:Apache License

/**
 * Helper method to list out all files under a specified path. If applyFilterToDirectories is false, the supplied
 * {@link PathFilter} will only be applied to files.
 *//*from w  ww .j  ava2  s  .co m*/
public static List<FileStatus> listFilesRecursively(FileSystem fs, Path path, PathFilter fileFilter,
        boolean applyFilterToDirectories) throws IOException {
    return listFilesRecursivelyHelper(fs, Lists.<FileStatus>newArrayList(), fs.getFileStatus(path), fileFilter,
            applyFilterToDirectories);
}

From source file:gobblin.util.FileListUtils.java

License:Apache License

/**
 * Method to list out all files, or directory if no file exists, under a specified path.
 * The specified {@link PathFilter} is treated as a file filter, that is it is only applied to file {@link Path}s.
 *//*from w w  w.  ja v a  2s.c o  m*/
public static List<FileStatus> listMostNestedPathRecursively(FileSystem fs, Path path, PathFilter fileFilter)
        throws IOException {
    return listMostNestedPathRecursivelyHelper(fs, Lists.<FileStatus>newArrayList(), fs.getFileStatus(path),
            fileFilter);
}

From source file:gobblin.util.FileListUtils.java

License:Apache License

/**
 * Helper method to list out all paths under a specified path. If the {@link org.apache.hadoop.fs.FileSystem} is
 * unable to list the contents of a relevant directory, will log an error and skip.
 *///from ww w.  ja v  a2  s . c  o m
public static List<FileStatus> listPathsRecursively(FileSystem fs, Path path, PathFilter fileFilter)
        throws IOException {
    return listPathsRecursivelyHelper(fs, Lists.<FileStatus>newArrayList(), fs.getFileStatus(path), fileFilter);
}

From source file:gobblin.util.filesystem.InstrumentedLocalFileSystemTest.java

License:Apache License

@Test
public void testFromInstrumentedScheme() throws Exception {

    File tmpDir = Files.createTempDir();
    tmpDir.deleteOnExit();//ww  w. j  a v a 2s. co m

    FileSystem fs = FileSystem.get(new URI(InstrumentedLocalFileSystem.SCHEME + ":///"), new Configuration());

    Assert.assertTrue(fs instanceof InstrumentedLocalFileSystem);
    Assert.assertTrue(DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem);
    Assert.assertEquals(fs.getFileStatus(new Path("/tmp")).getPath(), new Path("instrumented-file:///tmp"));
    Assert.assertEquals(fs.getUri().getScheme(), "instrumented-file");

    Path basePath = new Path(tmpDir.getAbsolutePath());

    Assert.assertTrue(fs.exists(basePath));

    Path file = new Path(basePath, "file");

    Assert.assertFalse(fs.exists(file));
    fs.create(new Path(basePath, "file"));
    Assert.assertTrue(fs.exists(file));

    Assert.assertEquals(fs.getFileStatus(file).getLen(), 0);
    Assert.assertEquals(fs.listStatus(basePath).length, 1);

    fs.delete(file, false);
    Assert.assertFalse(fs.exists(file));
}

From source file:gobblin.util.filesystem.InstrumentedLocalFileSystemTest.java

License:Apache License

@Test
public void testFromConfigurationOverride() throws Exception {
    Configuration configuration = new Configuration();
    configuration.set("fs.file.impl", InstrumentedLocalFileSystem.class.getName());
    FileSystem fs = FileSystem.newInstance(new URI("file:///"), configuration);
    Assert.assertTrue(fs instanceof InstrumentedLocalFileSystem);
    Assert.assertTrue(DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem);
    Assert.assertEquals(fs.getFileStatus(new Path("/tmp")).getPath(), new Path("file:///tmp"));
    Assert.assertEquals(fs.getUri().getScheme(), "file");
}