Example usage for org.apache.hadoop.fs FileSystem listStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem listStatus.

Prototype

public FileStatus[] listStatus(Path[] files, PathFilter filter) throws FileNotFoundException, IOException

Source Link

Document

Filter files/directories in the given list of paths using user-supplied path filter.

Usage

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static void removeReducerFiles(final String fileName) throws IOException {
    PathFilter filter = new PathFilter() {

        @Override//from ww w  .j a v  a  2  s. c  om
        public boolean accept(Path arg0) {
            if (arg0.getName().contains(fileName))
                return true;
            return false;
        }
    };

    FileSystem fs = FileSystem.get(new Configuration());
    Path path = new Path(fs.getHomeDirectory() + "/relationships");

    FileStatus[] status = fs.listStatus(path, filter);

    for (FileStatus fileStatus : status) {
        fs.delete(new Path(fs.getHomeDirectory() + "/relationships/" + fileStatus.getPath().getName()), true);
    }
}

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static String searchPreProcessing(final String name, Configuration conf, boolean s3) throws IOException {

    PathFilter filter = new PathFilter() {

        @Override//from   w  ww  .  j  av a 2 s .c  om
        public boolean accept(Path arg0) {
            if (arg0.getName().contains(name + "-"))
                return true;
            return false;
        }
    };

    Path path = null;
    FileSystem fs = null;

    if (s3) {
        path = new Path(conf.get("bucket") + preProcessingDir);
        fs = FileSystem.get(path.toUri(), conf);
    } else {
        fs = FileSystem.get(new Configuration());
        path = new Path(fs.getHomeDirectory() + "/" + preProcessingDir);
    }

    FileStatus[] status = fs.listStatus(path, filter);

    if (s3)
        fs.close();

    String preProcessingFile = null;
    boolean aggregatesFound = false;
    String fileName = "";
    for (FileStatus fileStatus : status) {
        fileName = fileStatus.getPath().getName();
        if (!fileName.endsWith(".aggregates"))
            preProcessingFile = fileName;
        else if (fileName.endsWith(".aggregates"))
            aggregatesFound = true;
    }

    if (!aggregatesFound)
        return null;

    return preProcessingFile;
}

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static String searchAggregatesHeader(final String name, Configuration conf, boolean s3)
        throws IOException {

    PathFilter filter = new PathFilter() {

        @Override/*  w  w  w . j  av a  2 s  .c  o m*/
        public boolean accept(Path arg0) {
            if (arg0.getName().contains(name + "-"))
                return true;
            return false;
        }
    };

    Path path = null;
    FileSystem fs = null;

    if (s3) {
        path = new Path(conf.get("bucket") + preProcessingDir);
        fs = FileSystem.get(path.toUri(), conf);
    } else {
        fs = FileSystem.get(new Configuration());
        path = new Path(fs.getHomeDirectory() + "/" + preProcessingDir);
    }

    FileStatus[] status = fs.listStatus(path, filter);

    if (s3)
        fs.close();

    String fileName = "";
    for (FileStatus fileStatus : status) {
        fileName = fileStatus.getPath().getName();
        if (fileName.endsWith(".aggregates"))
            return fileName;
    }

    return null;
}