Example usage for org.apache.hadoop.fs FileSystem listStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem listStatus.

Prototype

public FileStatus[] listStatus(Path[] files) throws FileNotFoundException, IOException 

Source Link

Document

Filter files/directories in the given list of paths using default path filter.

Usage

From source file:corner.services.hadoop.impl.HdfsAccessorProxy.java

License:Apache License

@Override
public List<FileDesc> list(final String path) throws IOException {
    String _path = path;/*  w  w w .jav a 2 s  . co  m*/
    if (path.endsWith("/")) {
        _path = path.substring(0, path.length() - 1);
    }
    Path dstPath = new Path(_path);
    FileSystem dstFs = dstPath.getFileSystem(getConf());
    FileStatus _dstStatus = dstFs.getFileStatus(dstPath);
    if (_dstStatus == null) {
        throw new IllegalArgumentException("The path [" + path + "] dose not exist.");
    }
    if (!_dstStatus.isDir()) {
        throw new IllegalArgumentException("The path [" + path + "] is not dir.");
    }
    FileStatus[] fileStatus = dstFs.listStatus(dstPath);
    if (fileStatus != null && fileStatus.length > 0) {
        List<FileDesc> ret = new LinkedList<FileDesc>();
        for (FileStatus status : fileStatus) {
            ret.add(new FileDesc(_path + "/" + status.getPath().getName(), status.isDir(),
                    new Timestamp(status.getModificationTime()), status.getLen()));
        }
        return ret;
    }
    return null;
}

---

From source file:crunch.MaxTemperature.java

License:Apache License

  public static void main(String[] args) throws Exception {
  String uri = args[0];// w ww .  ja  v a2  s  . co  m
  Configuration conf = new Configuration();
  FileSystem fs = FileSystem.get(URI.create(uri), conf);
    
  Path[] paths = new Path[args.length];
  for (int i = 0; i < paths.length; i++) {
    paths[i] = new Path(args[i]);
  }
    
  FileStatus[] status = fs.listStatus(paths); // XXX Filesystem.listStatus(Path[]) -> FileStatus (see below)
  Path[] listedPaths = FileUtil.stat2Paths(status); // XXX FileStatus to Path[]
  for (Path p : listedPaths) {
    System.out.println(p);
  }
}

---

From source file:de.gesundkrank.wikipedia.hadoop.io.WikiDumpLoader.java

License:Open Source License

/**
 * Returns {@link FileStatus} of the latest dump in the HDFS
 *
 * @param fs       HDFS/* w  w w  . j  av  a 2 s.  c  o  m*/
 * @param basepath Base path of hdfs wikidumps
 * @return
 */
private FileStatus checkLocalDumps(FileSystem fs, Path basepath) {

    long lastLocalChange = 0;
    FileStatus lastLocalDump = null;
    try {
        if (!fs.exists(basepath)) {
            fs.mkdirs(basepath);
            return null;
        }

        FileStatus[] stati = fs.listStatus(basepath);

        for (FileStatus status : stati) {
            long fileChange = status.getModificationTime();
            if (fileChange > lastLocalChange) {
                lastLocalDump = status;
            }
        }
    } catch (IOException e) {
        logger.error(e);
    }
    return lastLocalDump;

}

---

From source file:dz.lab.hdfs.SimpleLocalLs.java

public static void main(String[] args) throws Exception {
    Path path = new Path("/");
    // read location from command line arguments
    if (args.length == 1) {
        path = new Path(args[0]);
    }/* w w  w. ja v  a2s . c  o  m*/
    Configuration conf = new Configuration();
    conf.set("fs.defaultFS", "hdfs://192.168.2.72:8020");

    FileSystem fs = FileSystem.get(conf);

    // list the files and direcotries under the provided path
    FileStatus[] files = fs.listStatus(path);
    for (FileStatus file : files) {
        System.out.println(file.getPath().getName());
    }
}

---

From source file:eagle.service.security.hdfs.HDFSFileSystem.java

License:Apache License

/**
 * Brows the Files for the specific Path
  */*from   w  ww  .  j  av a2 s  . c  o  m*/
 * @param filePath
 * @return listOfFiles
 * @throws Exception 
 */
public List<FileStatus> browse(String filePath) throws Exception {
    LOG.info("HDFS File Path   :  " + filePath + "   and EndPoint  : " + hdfsEndPoint);
    FileSystem hdfsFileSystem = null;
    FileStatus[] listStatus;
    try {
        Configuration config = createConfig();
        hdfsFileSystem = getFileSystem(config);
        Path path = new Path(filePath);
        listStatus = hdfsFileSystem.listStatus(path);
    } catch (Exception ex) {
        LOG.error(" Exception when browsing files for the path " + filePath, ex.getMessage());
        throw new Exception(" Exception When browsing Files in HDFS .. Message :  " + ex.getMessage());
    } finally {
        //Close the file system
        if (hdfsFileSystem != null)
            hdfsFileSystem.close();
    }
    return Arrays.asList(listStatus);
}

---

From source file:eagle.service.security.hdfs.HDFSFileSystem.java

License:Apache License

/**
 * Browse Resources of passed Path and its sub directories
 * Note :: Sub directories only to determine the Sensitivity Type of Root Directories
 * @param filePath/*  w  ww.j a  va  2s.c om*/
 * @return fileSystemResponseObj{ with SubDirectoryMap and ListOf FileStatus}
 * @throws Exception
 */
public HDFSFileSystemResponseEntity browseResources(String filePath) throws Exception {
    LOG.info("HDFS File Path   :  " + filePath + "   and EndPoint  : " + hdfsEndPoint);
    FileSystem hdfsFileSystem = null;
    FileStatus[] listStatus;
    HDFSFileSystemResponseEntity response = new HDFSFileSystemResponseEntity();
    Map<String, List<String>> subdirectoriesMap = new HashMap<String, List<String>>();
    try {
        Configuration config = createConfig();
        hdfsFileSystem = getFileSystem(config);
        Path path = new Path(filePath);
        listStatus = hdfsFileSystem.listStatus(path);
        LOG.info(" Browsing Sub Directories .... ");
        // Browse Sub- directories
        for (FileStatus fileStatus : listStatus) {
            FileStatus[] fileStatusList = null;
            if (fileStatus.isDirectory())
                fileStatusList = hdfsFileSystem.listStatus(new Path(fileStatus.getPath().toUri().getPath()));

            if (fileStatusList != null && fileStatusList.length > 0)
                subdirectoriesMap.put(fileStatus.getPath().toUri().getPath(), /*  Key would be Parent */
                        getSubDirectories(fileStatusList) /*  Value Would be Child Paths */);
        }
        response.setFileList(Arrays.asList(listStatus));
        response.setSubDirectoriesMap(subdirectoriesMap);

    } catch (Exception ex) {
        LOG.error(" Exception when browsing files for the path " + filePath, ex.getMessage());
        throw new Exception(
                " Exception When browsing Files/Directories in HDFS .. Message :  " + ex.getMessage());
    } finally {
        //Close the file system
        if (hdfsFileSystem != null)
            hdfsFileSystem.close();
    }
    return response;
}

---

From source file:eagle.service.security.hdfs.HDFSFileSystem.java

License:Apache License

public static void main(String[] args) throws IOException {
    Configuration config = new Configuration();
    config.set("fs.defaultFS", "hdfs://127.0.0.1:9000");
    FileSystem hdfsFileSystem = FileSystem.get(config);
    Path filePath = new Path("hdfs://127.0.0.1:9000/user");
    //FileStatus[]  listStatus = hdfsFileSystem.listStatus( filePath );
    //System.out.println(status1);
    //RemoteIterator<LocatedFileStatus> locatedFileStatus = hdfsFileSystem.listFiles(filePath, true);
    /*RemoteIterator<LocatedFileStatus> locatedFileStatus = hdfsFileSystem.listLocatedStatus(filePath);
    while( locatedFileStatus.hasNext() )
    {//from  w  w w.  ja  va2 s  . c  o  m
       LocatedFileStatus status = locatedFileStatus.next();
       System.out.println(status.getPath().toUri());
               
    }*/

    FileStatus[] listStatus = hdfsFileSystem.listStatus(filePath);
    for (FileStatus status : listStatus) {
        System.out.println(status.getPath().toUri().getPath());
        //System.out.println(status.getPath().toUri().getPath());
    }
}

---

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsFileOutputCommitter.java

License:Apache License

private void moveTaskOutputsToIRODS(TaskAttemptContext context, FileSystem outfs, Path outDir,
        FileSystem workfs, Path workOutput) throws IOException {
    context.progress();//from   w  ww .java  2  s  .  co m
    if (workfs.isFile(workOutput)) {
        Path finalOutputPath = getFinalPath(outDir, workOutput, this.workPath);
        FSDataOutputStream irods_os = null;
        FSDataInputStream temp_is = null;
        try {
            // commit to iRODS
            irods_os = outfs.create(finalOutputPath, true);
            temp_is = workfs.open(workOutput);

            byte[] buffer = new byte[100 * 1024];
            int bytes_read = 0;

            while ((bytes_read = temp_is.read(buffer)) != -1) {
                irods_os.write(buffer, 0, bytes_read);
            }
        } finally {
            if (temp_is != null) {
                try {
                    temp_is.close();
                } catch (IOException ex) {
                    // ignore exceptions
                }
            }

            // remove temporary file
            try {
                workfs.delete(workOutput, true);
            } catch (IOException ex) {
                // ignore exceptions
            }

            if (irods_os != null) {
                irods_os.close();
            }
        }

        LOG.debug("Moved " + workOutput + " to " + finalOutputPath);
    } else if (workfs.getFileStatus(workOutput).isDir()) {
        FileStatus[] paths = workfs.listStatus(workOutput);
        Path finalOutputPath = getFinalPath(outDir, workOutput, this.workPath);
        outfs.mkdirs(finalOutputPath);
        if (paths != null) {
            for (FileStatus path : paths) {
                moveTaskOutputsToIRODS(context, outfs, outDir, workfs, path.getPath());
            }
        }
    }
}

---

From source file:edu.brown.cs.mapreduce.BenchmarkBase.java

License:Open Source License

public JobConf getJobConf() {
    JobConf jobConf = new JobConf(this.conf, this.benchmarkClass);
    ///*from   ww  w .  ja  v a 2  s . com*/
    // Options
    //
    List<String> otherArgs = new ArrayList<String>();
    for (int i = 0; i < args.length; i++) {
        try {
            //
            // Print property and exit
            //
            if ("-property".equals(args[i])) {
                String prop = jobConf.get(args[i + 1]);
                System.out.println(prop);
                System.exit(0);
                //
                // # of Maps
                //
            } else if ("-m".equals(args[i])) {
                this.num_of_maps = Integer.parseInt(args[++i]);
                //
                // # of Reduces
                //
            } else if ("-r".equals(args[i])) {
                this.num_of_reduces = Integer.parseInt(args[++i]);
                //
                // Enable debug
                //
            } else if ("-debug".equals(args[i])) {
                this.debug = true;
                //
                // Enable single output file for results
                //
            } else if ("-combine".equals(args[i])) {
                this.combine = true;
                //
                // Tell jobs to compress their intermediate output files
                //
            } else if ("-compress".equals(args[i])) {
                this.compress = true;
                //
                // We're using TupleWritable (which has to be in a SequenceFile)
                //
            } else if ("-tuple".equals(args[i])) {
                this.tuple_data = true;
                this.sequence_file = true;
                //
                // Use SequenceFiles for initial input
                //
            } else if ("-sequence".equals(args[i])) {
                this.sequence_file = true;
                //
                // Recursively load directories
                //
            } else if ("-recursive-dirs".equals(args[i])) {
                this.load_directories = true;
                //
                // Job Basename
                //
            } else if ("-basename".equals(args[i])) {
                this.job_name = args[++i];
                //
                // Misc. Properties
                //
            } else if ("-D".equals(args[i].substring(0, 2))) {
                String arg = args[i].substring(2);
                int pos = arg.indexOf('=');
                if (pos == -1) {
                    System.err.println("ERROR: Invalid properties option '" + arg + "'");
                    System.exit(1);
                }
                this.options.put(arg.substring(0, pos), arg.substring(pos + 1));
            } else {
                otherArgs.add(args[i]);
            }
        } catch (NumberFormatException except) {
            System.err.println("ERROR: Integer expected instead of " + args[i]);
            System.exit(1);
        } catch (ArrayIndexOutOfBoundsException except) {
            System.err.println("ERROR: Required parameter missing from " + args[i - 1]);
            System.exit(1);
        }
    } // FOR
      //
      // Make sure there are exactly 2 parameters left.
      //
    if (otherArgs.size() < 2) {
        System.err.println("ERROR: Wrong number of parameters: " + otherArgs.size());
        System.exit(1);
    }

    //
    // Set these flags so the jobs know about them
    //
    if (this.getSequenceFile())
        this.options.put(PROPERTY_SEQUENCEFILE, "true");
    if (this.getTupleData())
        this.options.put(PROPERTY_TUPLEDATA, "true");
    if (this.getDebug())
        this.options.put(PROPERTY_DEBUG, "true");

    FileSystem fs = null;
    try {
        fs = FileSystem.get(conf);
    } catch (Exception ex) {
        ex.printStackTrace();
        System.exit(-1);
    }

    //
    // Input Paths
    //
    int cnt = otherArgs.size() - 1;
    this.input_paths = new ArrayList<Path>();
    for (int ctr = 0; ctr < cnt; ctr++) {
        Path new_path = new Path(otherArgs.get(ctr));
        try {
            if (this.load_directories && fs.getFileStatus(new_path).isDir()) {
                //int limit = 10;
                FileStatus paths[] = fs.listStatus(new_path);
                for (FileStatus p : paths) {
                    this.input_paths.add(p.getPath());
                    FileInputFormat.addInputPath(jobConf, p.getPath());
                    //if (limit-- <= 0) break;
                } // FOR
            } else {
                this.input_paths.add(new_path);
                FileInputFormat.addInputPath(jobConf, new_path);
            }
        } catch (Exception ex) {
            ex.printStackTrace();
            System.exit(-1);
        }
    } // FOR
    if (this.input_paths.isEmpty()) {
        System.err.println(
                "ERROR: No input paths were defined for '" + this.benchmarkClass.getSimpleName() + "'");
        System.exit(-1);
    }

    //
    // Output Paths
    //
    this.output_path = new Path(otherArgs.get(otherArgs.size() - 1));
    FileOutputFormat.setOutputPath(jobConf, this.output_path);

    jobConf.setJobName(this.job_name != null ? this.job_name : this.benchmarkClass.getSimpleName());
    if (this.num_of_maps >= 0)
        jobConf.setNumMapTasks(this.num_of_maps);
    if (this.num_of_reduces >= 0)
        jobConf.setNumReduceTasks(this.num_of_reduces);

    //
    // Set all properties
    //
    for (String key : this.options.keySet()) {
        jobConf.set(key, this.options.get(key));
    }

    return (jobConf);
}

---

From source file:edu.buffalo.cse.dic.mapreduce.WordCount.java

License:Apache License

@Override
public Map<String, Number> start(String inputFile) {
    try {/* w  w  w  .  java2 s  . co m*/
        LinkedHashMap<String, Number> topTen = new LinkedHashMap<>();
        Configuration conf = new Configuration();
        conf.addResource(new Path("/usr/local/hadoop/etc/hadoop/core-site.xml"));
        conf.addResource(new Path("/usr/local/hadoop/etc/hadoop/hdfs-site.xml"));

        FileSystem fs = FileSystem.get(new URI("wordcount"), conf);
        fs.delete(new Path("wordcount"));

        Job job = new Job(conf, "word count");
        job.setJarByClass(WordCount.class);
        job.setMapperClass(TokenizerMapper.class);
        job.setCombinerClass(IntSumReducer.class);
        job.setReducerClass(IntSumReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        FileInputFormat.addInputPath(job, new Path(inputFile));
        FileOutputFormat.setOutputPath(job, new Path("wordcount"));
        job.waitForCompletion(true);
        System.out.println("word count done");

        FileSystem fsa = FileSystem.get(new URI("wordcount"), conf);
        fsa.delete(new Path("wordcountfinal"));

        Job sortJob = new Job(conf, "sort reducer");
        sortJob.setJarByClass(SortReducerOutput.class);
        sortJob.setMapperClass(OutputBreaker.class);
        sortJob.setSortComparatorClass(ReverseComparator.class);
        sortJob.setReducerClass(SortByCount.class);
        sortJob.setOutputKeyClass(IntWritable.class);
        sortJob.setOutputValueClass(Text.class);
        sortJob.setPartitionerClass(TotalOrderPartitioner.class);
        Path partitionFile = new Path("trendcount", "_sortPartitioning");
        TotalOrderPartitioner.setPartitionFile(sortJob.getConfiguration(), partitionFile);
        FileInputFormat.addInputPath(sortJob, new Path("wordcount/part-r-00000"));
        FileOutputFormat.setOutputPath(sortJob, new Path("wordcountfinal"));
        sortJob.waitForCompletion(true);
        System.out.println("sort word count");

        Path output = new Path("wordcountfinal/part-r-00000");
        FileSystem fileSystem = FileSystem.get(output.toUri(), conf);
        FileStatus[] items = fileSystem.listStatus(output);
        for (FileStatus item : items) {
            InputStream stream = null;
            // ignoring files like _SUCCESS
            if (item.getPath().getName().startsWith("_")) {
                continue;
            } else {
                stream = fileSystem.open(item.getPath());
            }
            Scanner scan = new Scanner(stream).useDelimiter("\\n");
            for (int i = 0; i < 10; i++) {
                if (scan.hasNext()) {
                    String data = scan.next();
                    topTen.put(data.split("\\t")[1], Integer.parseInt(data.split("\\t")[0]));
                }
            }
        }
        return topTen;
    } catch (IOException e) {
        e.printStackTrace();
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
    } catch (InterruptedException e) {
        e.printStackTrace();
    } catch (URISyntaxException e) {
        e.printStackTrace();
    }
    return null;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• »