Example usage for org.apache.hadoop.fs FileSystem listStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem listStatus.

Prototype

public FileStatus[] listStatus(Path[] files) throws FileNotFoundException, IOException 

Source Link

Document

Filter files/directories in the given list of paths using default path filter.

Usage

From source file:com.netflix.bdp.s3.S3MultipartOutputCommitter.java

License:Apache License

@Override
public boolean needsTaskCommit(TaskAttemptContext context) throws IOException {
    // check for files on the local FS in the attempt path
    Path attemptPath = getTaskAttemptPath(context);
    FileSystem fs = attemptPath.getFileSystem(context.getConfiguration());

    if (fs.exists(attemptPath)) {
        FileStatus[] stats = fs.listStatus(attemptPath);
        return stats.length > 0;
    }/*from  w  ww .  ja  v  a  2s  . c  o  m*/

    return false;
}

---

From source file:com.netflix.bdp.s3.TestS3MultipartOutputCommitter.java

License:Apache License

@Test
public void testSingleTaskCommit() throws Exception {
    Path file = new Path(commitTask(committer, tac, 1).iterator().next());

    List<String> uploads = committer.results.getUploads();
    Assert.assertEquals("Should initiate one upload", 1, uploads.size());

    Path committedPath = committer.getCommittedTaskPath(tac);
    FileSystem dfs = committedPath.getFileSystem(conf);

    Assert.assertEquals("Should commit to HDFS", getDFS(), dfs);

    FileStatus[] stats = dfs.listStatus(committedPath);
    Assert.assertEquals("Should produce one commit file", 1, stats.length);
    Assert.assertEquals("Should name the commits file with the task ID", "task_job_0001_r_000002",
            stats[0].getPath().getName());

    List<S3Util.PendingUpload> pending = S3Util.readPendingCommits(dfs, stats[0].getPath());
    Assert.assertEquals("Should have one pending commit", 1, pending.size());

    S3Util.PendingUpload commit = pending.get(0);
    Assert.assertEquals("Should write to the correct bucket", BUCKET, commit.getBucketName());
    Assert.assertEquals("Should write to the correct key", KEY_PREFIX + "/" + file.getName(), commit.getKey());

    assertValidUpload(committer.results.getTagsByUpload(), commit);
}

---

From source file:com.netflix.bdp.s3.TestS3MultipartOutputCommitter.java

License:Apache License

@Test
public void testSingleTaskEmptyFileCommit() throws Exception {
    committer.setupTask(tac);/*from w  ww .j av a  2s  .  c om*/

    Path attemptPath = committer.getTaskAttemptPath(tac);

    String rand = UUID.randomUUID().toString();
    writeOutputFile(tac.getTaskAttemptID(), attemptPath, rand, 0);

    committer.commitTask(tac);

    List<String> uploads = committer.results.getUploads();
    Assert.assertEquals("Should initiate one upload", 0, uploads.size());

    Path committedPath = committer.getCommittedTaskPath(tac);
    FileSystem dfs = committedPath.getFileSystem(conf);

    Assert.assertEquals("Should commit to HDFS", getDFS(), dfs);

    FileStatus[] stats = dfs.listStatus(committedPath);
    Assert.assertEquals("Should produce one commit file", 1, stats.length);
    Assert.assertEquals("Should name the commits file with the task ID", "task_job_0001_r_000002",
            stats[0].getPath().getName());

    List<S3Util.PendingUpload> pending = S3Util.readPendingCommits(dfs, stats[0].getPath());
    Assert.assertEquals("Should have no pending commits", 0, pending.size());
}

---

From source file:com.netflix.bdp.s3.TestS3MultipartOutputCommitter.java

License:Apache License

@Test
public void testSingleTaskMultiFileCommit() throws Exception {
    int numFiles = 3;
    Set<String> files = commitTask(committer, tac, numFiles);

    List<String> uploads = committer.results.getUploads();
    Assert.assertEquals("Should initiate multiple uploads", numFiles, uploads.size());

    Path committedPath = committer.getCommittedTaskPath(tac);
    FileSystem dfs = committedPath.getFileSystem(conf);

    Assert.assertEquals("Should commit to HDFS", getDFS(), dfs);

    FileStatus[] stats = dfs.listStatus(committedPath);
    Assert.assertEquals("Should produce one commit file", 1, stats.length);
    Assert.assertEquals("Should name the commits file with the task ID", "task_job_0001_r_000002",
            stats[0].getPath().getName());

    List<S3Util.PendingUpload> pending = S3Util.readPendingCommits(dfs, stats[0].getPath());
    Assert.assertEquals("Should have correct number of pending commits", files.size(), pending.size());

    Set<String> keys = Sets.newHashSet();
    for (S3Util.PendingUpload commit : pending) {
        Assert.assertEquals("Should write to the correct bucket", BUCKET, commit.getBucketName());
        assertValidUpload(committer.results.getTagsByUpload(), commit);
        keys.add(commit.getKey());//from  w  w  w  .  ja  v  a  2s  . com
    }

    Assert.assertEquals("Should write to the correct key", files, keys);
}

---

From source file:com.netflix.bdp.s3mper.cli.MetastoreResolveCommand.java

License:Apache License

@Override
public void execute(Configuration conf, String[] args) throws Exception {
    try {/*  ww  w . ja v  a  2 s. c om*/

        Path path = new Path(args[0]);

        conf.set("s3mper.metastore.deleteMarker.enabled", "true");

        FileSystemMetastore meta = Metastore.getFilesystemMetastore(conf);
        meta.initalize(path.toUri(), conf);

        FileSystem fs = FileSystem.get(path.toUri(), conf);

        Set<String> s3files = new HashSet<String>();

        FileStatus[] s3listing = fs.listStatus(path);

        if (s3listing != null) {
            for (FileStatus f : s3listing) {
                s3files.add(f.getPath().toUri().toString());
            }
        }

        List<FileInfo> files = meta.list(Collections.singletonList(path));

        for (FileInfo f : files) {
            if (!s3files.contains(f.getPath().toUri().toString())) {
                meta.delete(f.getPath());
            }
        }
    } catch (Exception e) {
        System.out.println("Usage: s3mper metastore resolve <path>\n");

        e.printStackTrace();
    }
}

---

From source file:com.netflix.bdp.s3mper.listing.ConsistentListingAspect.java

License:Apache License

private List<Path> recursiveList(FileSystem fs, Path path) throws IOException {
    List<Path> result = new ArrayList<Path>();

    try {/*from w w  w .  ja  v  a2s  .  co  m*/
        result.add(path);

        if (!fs.isFile(path)) {
            FileStatus[] children = fs.listStatus(path);

            if (children == null) {
                return result;
            }

            for (FileStatus child : children) {
                if (child.isDir()) {
                    result.addAll(recursiveList(fs, child.getPath()));
                } else {
                    result.add(child.getPath());
                }
            }
        }
    } catch (Exception e) {
        log.info("A problem occurred recursively deleting path: " + path + " " + e.getMessage());
    }

    return result;
}

---

From source file:com.netflix.bdp.s3mper.listing.DarkloadTest.java

License:Apache License

@Test(expected = FileNotFoundException.class)
public void testDarkloadingDisabled() throws Throwable {
    Configuration noDarkloadConf = new Configuration(conf);
    noDarkloadConf.setBoolean("s3mper.failOnError", true);
    noDarkloadConf.setBoolean("s3mper.darkload", false);
    FileSystem noDarkloadFs = FileSystem.get(testPath.toUri(), noDarkloadConf);
    Path path = new Path(testPath + "/test");
    meta.add(path, false);/* w  w  w.j a va  2 s  . co m*/
    noDarkloadFs.listStatus(path.getParent());
}

---

From source file:com.netflix.suro.sink.localfile.LocalFileSink.java

License:Apache License

/**
 * List all files under the directory. If the file is marked as done, the
 * notice for that file would be sent. Otherwise, it checks the file
 * is not closed properly, the file is marked as done and the notice
 * would be sent. That file would cause EOFException when reading.
 *
 * @param dir/*from  w  ww  .  ja  va2s .  c  o  m*/
 * @return the number of files found in the directory
 */
public int cleanUp(String dir, boolean fetchAll) {
    if (!dir.endsWith("/")) {
        dir += "/";
    }

    int count = 0;

    try {
        FileSystem fs = writer.getFS();
        FileStatus[] files = fs.listStatus(new Path(dir));
        for (FileStatus file : files) {
            if (file.getLen() > 0) {
                String fileName = file.getPath().getName();
                String fileExt = getFileExt(fileName);
                if (fileExt != null && fileExt.equals(done)) {
                    notice.send(dir + fileName);
                    ++count;
                } else if (fileExt != null) {
                    long lastPeriod = new DateTime().minus(rotationPeriod).minus(rotationPeriod).getMillis();
                    if (file.getModificationTime() < lastPeriod) {
                        ++errorClosedFiles;
                        DynamicCounter.increment("closedFileError");
                        log.error(dir + fileName + " is not closed properly!!!");
                        String doneFile = fileName.replace(fileExt, done);
                        writer.setDone(dir + fileName, dir + doneFile);
                        notice.send(dir + doneFile);
                        ++count;
                    } else if (fetchAll) {
                        ++count;
                    }
                }
            }
        }
    } catch (Exception e) {
        log.error("Exception while on cleanUp: " + e.getMessage(), e);
        return Integer.MAX_VALUE; // return non-zero value
    }

    return count;
}

---

From source file:com.ngdata.hbaseindexer.mr.TestUtils.java

License:Apache License

public static void validateSolrServerDocumentCount(File solrHomeDir, FileSystem fs, Path outDir,
        int expectedDocs, int expectedShards) throws IOException, SolrServerException {

    long actualDocs = 0;
    int actualShards = 0;
    for (FileStatus dir : fs.listStatus(outDir)) { // for each shard
        if (dir.getPath().getName().startsWith("part") && dir.isDirectory()) {
            actualShards++;//ww w .j  a  va2s  .c  o m
            EmbeddedSolrServer solr = createEmbeddedSolrServer(solrHomeDir, fs, dir.getPath());

            try {
                SolrQuery query = new SolrQuery();
                query.setQuery("*:*");
                QueryResponse resp = solr.query(query);
                long numDocs = resp.getResults().getNumFound();
                actualDocs += numDocs;
            } finally {
                solr.close();
            }
        }
    }
    assertEquals(expectedShards, actualShards);
    assertEquals(expectedDocs, actualDocs);
}

---

From source file:com.ning.metrics.action.hdfs.reader.HdfsListing.java

License:Apache License

private void findEntries(FileSystem fs, Path p, ImmutableList.Builder<HdfsEntry> entriesBuilder)
        throws IOException {
    final FileStatus[] fileStatuses = fs.listStatus(p);
    if (fileStatuses == null) {
        return;//from w  w w  .java  2  s.c om
    }

    for (final FileStatus s : fileStatuses) {
        if (s.isDir() && recursive) {
            findEntries(fs, s.getPath(), entriesBuilder);
        }

        entriesBuilder.add(new HdfsEntry(fs, s, raw, rowFileContentsIteratorFactory));
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• »