Example usage for org.apache.hadoop.fs FileSystem listStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem listStatus.

Prototype

public FileStatus[] listStatus(Path[] files) throws FileNotFoundException, IOException

Source Link

Document

Filter files/directories in the given list of paths using default path filter.

Usage

From source file:edu.ecnu.idse.TrajStore.util.FileUtil.java

License:Open Source License

/**
 * function to list files in a certain directory
 * //from  w  ww. j  a  v a 2  s. com
 * @author zzg
 * @param path
 * @return
 * @throws IOException
 */
public static Path[] getFilesListInPath(Path path) throws IOException {
    FileSystem fileSystem = path.getFileSystem(new Configuration());
    FileStatus[] matchingDirs = fileSystem.listStatus(path);
    Path[] pathsArr = new Path[matchingDirs.length];
    for (int i = 0; i < matchingDirs.length; i++) {
        pathsArr[i] = matchingDirs[i].getPath();
    }
    return pathsArr;
}

From source file:edu.indiana.soic.ts.mapreduce.pwd.PairWiseDistance.java

License:Open Source License

public void submitJob() throws IOException {
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);
    FileStatus[] status = fs.listStatus(new Path(vectDir));
    for (int i = 0; i < status.length; i++) {
        String sequenceFile = status[i].getPath().getName();
        String sequenceFileFullPath = vectDir + "/" + sequenceFile;
        try {/*  w w  w . ja  v  a 2 s. c o  m*/
            execJob(conf, sequenceFileFullPath, sequenceFile, interDistDir);
            concatOutput(conf, sequenceFile, interDistDir, distDir);
        } catch (Exception e) {
            String message = "Failed to executed PWD calculation:" + sequenceFileFullPath + " " + interDistDir;
            LOG.info(message, e);
            throw new RuntimeException(message);
        }
    }
}

From source file:edu.indiana.soic.ts.mapreduce.pwd.PairWiseDistance.java

License:Open Source License

public void concatOutput(Configuration conf, String sequenceFile, String distDirIntermediate, String distDir)
        throws IOException {
    FileSystem fs = FileSystem.get(conf);
    Path outDir = new Path(distDirIntermediate + "/" + sequenceFile + "/out");
    FileStatus[] status = fs.listStatus(outDir);
    List<OutFile> outFiles = new ArrayList<OutFile>();
    for (int i = 0; i < status.length; i++) {
        String name = status[i].getPath().getName();
        String split[] = name.split("_");
        if (split.length > 2 && split[0].equals("row")) {
            OutFile o = new OutFile(Integer.parseInt(split[1]), name);
            outFiles.add(o);/* w  w w .  j a  va 2s.co  m*/
        }
    }

    Collections.sort(outFiles);
    String destFile = distDir + "/" + sequenceFile;
    Path outFile = new Path(destFile);
    FSDataOutputStream outputStream = fs.create(outFile);
    for (OutFile o : outFiles) {
        Path inFile = new Path(outDir, o.file);
        FSDataInputStream inputStream = fs.open(inFile);
        IOUtils.copy(inputStream, outputStream);
        inputStream.close();
    }
    outputStream.flush();
    outputStream.close();
}

From source file:edu.isi.mavuno.util.MavunoUtils.java

License:Apache License

public static void recursivelyAddInputPaths(Job job, String path) throws IOException {
    FileSystem fs;
    try {/*from   ww  w .  j  a v a2  s. c  om*/
        fs = FileSystem.get(new URI(path), job.getConfiguration());
    } catch (URISyntaxException e) {
        throw new RuntimeException("Error recursively adding path -- " + path);
    }

    FileStatus[] ls = fs.listStatus(new Path(path));
    for (FileStatus status : ls) {
        // skip anything that starts with an underscore, as it often indicates
        // a log directory or another special type of Hadoop file
        if (status.getPath().getName().startsWith("_")) {
            continue;
        }

        if (status.isDir()) {
            recursivelyAddInputPaths(job, status.getPath().toString());
        } else {
            FileInputFormat.addInputPath(job, status.getPath());
        }
    }
}

From source file:edu.isi.mavuno.util.MavunoUtils.java

License:Apache License

public static FileStatus[] getDirectoryListing(Configuration conf, String path) throws IOException {
    FileSystem fs;
    try {/*from  w w w.  j  ava  2  s  . com*/
        fs = FileSystem.get(new URI(path), conf);
    } catch (URISyntaxException e) {
        throw new RuntimeException("Error getting directory listing -- " + path);
    }
    FileStatus[] files = fs.listStatus(new Path(path));
    return files;
}

From source file:edu.mit.jwi.data.FileProvider.java

License:Creative Commons License

/**
 * Make a File from Path/*ww  w. jav a 2s . c  o m*/
 * 
 * from
 * http://stackoverflow.com/questions/3444313/how-to-convert-a-hadoop-path-
 * object-into-a-java-file-object
 * 
 * @param some_path
 * @param conf
 * @author Mauro Pelucchi
 * @since JWI 2.3.3-hadoop
 * @return
 */
public static File MakeFileFromPath(Path hdfsPath, Configuration conf) {
    try {
        FileSystem fs = FileSystem.get(hdfsPath.toUri(), conf);
        File tempFolder = File.createTempFile(hdfsPath.getName(), "");
        if (!(tempFolder.delete())) {
            throw new IOException("Could not delete temp file: " + tempFolder.getAbsolutePath());
        }

        if (!(tempFolder.mkdir())) {
            throw new IOException("Could not create temp directory: " + tempFolder.getAbsolutePath());
        }
        FileStatus[] status = fs.listStatus(hdfsPath);
        for (int i = 0; i < status.length; i++) {
            System.out.println("------------------------------------");
            if (status[i].isFile()) {
                System.out.println(status[i].getPath());
                fs.copyToLocalFile(false, status[i].getPath(), new Path(tempFolder.getAbsolutePath()));
                //System.out.println(ReadFileFromHdfs(fs, status[i].getPath()));
            }
        }

        tempFolder.deleteOnExit();

        File[] files = tempFolder.listFiles();
        for (int i = 0; i < files.length; i++) {
            System.out.println("------------------------------------");
            System.out.println(files[i].getPath());
            //System.out.println(ReadFile(files[i].getPath()));
            if (files[i].getName().startsWith(".")) {
                System.out.println("Delete --> " + files[i].getPath());
                if (!(files[i].delete())) {
                    throw new IOException("Could not delete temp file: " + files[i].getAbsolutePath());
                }
            }
        }

        return tempFolder;
    } catch (Exception e) {
        e.printStackTrace();
    }
    return null;
}