Example usage for org.apache.hadoop.fs FileSystem isDirectory

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem isDirectory.

Prototype

@Deprecated
public boolean isDirectory(Path f) throws IOException 

Source Link

Document

True iff the named path is a directory.

Usage

From source file:org.apache.lens.server.query.LensPersistentResult.java

License:Apache License

@Override
public boolean isHttpResultAvailable() {
    try {//from   w w  w . j  a  va 2  s. c om
        final Path resultPath = new Path(getOutputPath());
        FileSystem fs = resultPath.getFileSystem(conf);
        if (fs.isDirectory(resultPath)) {
            return false;
        }
    } catch (IOException | LensException e) {
        log.warn("Unable to get status for Result Directory", e);
        return false;
    }
    return true;
}

---

From source file:org.apache.lens.server.session.DatabaseResourceService.java

License:Apache License

private void loadDbResourceEntries() throws LensException {
    // Read list of databases in
    FileSystem serverFs = null;

    try {/*from   w  w  w.j a  v a 2 s  .  c  om*/
        String resTopDir = getHiveConf().get(LensConfConstants.DATABASE_RESOURCE_DIR,
                LensConfConstants.DEFAULT_DATABASE_RESOURCE_DIR);
        log.info("Database specific resources at {}", resTopDir);

        Path resTopDirPath = new Path(resTopDir);
        serverFs = FileSystem.newInstance(resTopDirPath.toUri(), getHiveConf());
        if (!serverFs.exists(resTopDirPath)) {
            incrCounter(LOAD_RESOURCES_ERRORS);
            log.warn("Database resource location does not exist - {}. Database jars will not be available",
                    resTopDir);
            return;
        }

        // Look for db dirs
        for (FileStatus dbDir : serverFs.listStatus(resTopDirPath)) {
            Path dbDirPath = dbDir.getPath();
            if (serverFs.isDirectory(dbDirPath)) {
                String dbName = dbDirPath.getName();
                // Get all resources for that db
                findResourcesInDir(serverFs, dbName, dbDirPath);
            } else {
                log.warn("DB resource DIR is not a directory: {}", dbDirPath);
            }
        }

        log.debug("Found resources {}", dbResEntryMap);
    } catch (IOException io) {
        log.error("Error getting list of dbs to load resources from", io);
        throw new LensException(io);
    } finally {
        if (serverFs != null) {
            try {
                serverFs.close();
            } catch (IOException e) {
                log.error("Error closing file system instance", e);
            }
        }
    }
}

---

From source file:org.apache.lens.server.session.DatabaseResourceService.java

License:Apache License

private void findResourcesInDir(FileSystem serverFs, String database, Path dbDirPath) throws IOException {
    // Check if order file is present in the directory
    List<String> jars = new ScannedPaths(dbDirPath, "jar").getFinalPaths();
    if (jars != null && !jars.isEmpty()) {
        log.info("{} picking jar in jar_order: {}", database, jars);
        for (String jar : jars) {
            if (StringUtils.isBlank(jar)) {
                // skipping empty lines. usually the last line could be empty
                continue;
            }//  w  w  w.  j av  a2  s . co  m
            Path jarFilePath = new Path(dbDirPath, jar);
            if (!jar.endsWith(".jar") || !serverFs.exists(jarFilePath)) {
                log.info("Resource skipped {} for db {}", jarFilePath, database);
                continue;
            }
            addResourceEntry(new LensSessionImpl.ResourceEntry("jar", jarFilePath.toUri().toString()),
                    database);
        }
    } else {
        log.info("{} picking jars in file list order", database);
        for (FileStatus dbResFile : serverFs.listStatus(dbDirPath)) {
            // Skip subdirectories
            if (serverFs.isDirectory(dbResFile.getPath())) {
                continue;
            }

            String dbResName = dbResFile.getPath().getName();
            String dbResUri = dbResFile.getPath().toUri().toString();

            if (dbResName.endsWith(".jar")) {
                addResourceEntry(new LensSessionImpl.ResourceEntry("jar", dbResUri), database);
            } else {
                log.info("Resource skipped {} for db {}", dbResFile.getPath(), database);
            }
        }
    }
}

---

From source file:org.apache.lens.server.util.ScannedPaths.java

License:Apache License

/**
 * Method that computes path of resources matching the input path or path regex pattern.
 * If provided path is a directory it additionally checks for the jar_order or glob_order file
 * that imposes ordering of resources and filters out other resources.
 *
 * Updates finalPaths List with matched paths and returns an iterator for matched paths.
 *///from   www .ja  v  a  2s . c o m
private List<String> getMatchedPaths(Path pt, String type) {
    List<String> finalPaths = new ArrayList<>();
    InputStream resourceOrderIStream = null;
    FileSystem fs;

    try {
        fs = pt.getFileSystem(new Configuration());
        if (fs.exists(pt)) {
            if (fs.isFile(pt)) {
                /**
                 * CASE 1 : Direct FILE provided in path
                 **/
                finalPaths.add(pt.toUri().toString());
            } else if (fs.isDirectory(pt)) {
                /**
                 * CASE 2 : DIR provided in path
                 **/
                Path resourceOrderFile;
                FileStatus[] statuses;
                List<String> newMatches;
                List<String> resources;

                resourceOrderFile = new Path(pt, "jar_order");
                /** Add everything in dir if no jar_order or glob_order is present **/
                if (!fs.exists(resourceOrderFile)) {
                    resourceOrderFile = new Path(pt, "glob_order");
                    if (!fs.exists(resourceOrderFile)) {
                        resourceOrderFile = null;
                        /** Get matched resources recursively for all files **/
                        statuses = fs.globStatus(new Path(pt, "*"));
                        if (statuses != null) {
                            for (FileStatus st : statuses) {
                                newMatches = getMatchedPaths(st.getPath(), type);
                                finalPaths.addAll(newMatches);
                            }
                        }
                    }
                }
                if (resourceOrderFile != null) {
                    /** Else get jars as per order specified in jar_order/glob_order **/
                    resourceOrderIStream = fs.open(resourceOrderFile);
                    resources = IOUtils.readLines(resourceOrderIStream, Charset.forName("UTF-8"));
                    for (String resource : resources) {
                        if (StringUtils.isBlank(resource)) {
                            continue;
                        }
                        resource = resource.trim();

                        /** Get matched resources recursively for provided path/pattern **/
                        if (resource.startsWith("/") || resource.contains(":/")) {
                            newMatches = getMatchedPaths(new Path(resource), type);
                        } else {
                            newMatches = getMatchedPaths(new Path(pt, resource), type);
                        }
                        finalPaths.addAll(newMatches);
                    }
                }
            }
        } else {
            /**
             * CASE 3 : REGEX provided in path
             * */
            FileStatus[] statuses = fs.globStatus(Path.getPathWithoutSchemeAndAuthority(pt));
            if (statuses != null) {
                for (FileStatus st : statuses) {
                    List<String> newMatches = getMatchedPaths(st.getPath(), type);
                    finalPaths.addAll(newMatches);
                }
            }
        }
        filterDirsAndJarType(fs, finalPaths);
    } catch (FileNotFoundException fex) {
        log.error("File not found while scanning path. Path: {}, Type: {}", path, type, fex);
    } catch (Exception e) {
        log.error("Exception while initializing PathScanner. Path: {}, Type: {}", path, type, e);
    } finally {
        IOUtils.closeQuietly(resourceOrderIStream);
    }

    return finalPaths;
}

---

From source file:org.apache.lens.server.util.ScannedPaths.java

License:Apache License

/**
 * Skip Dirs from matched regex./*from  w  w  w  .  j a  va2 s .c om*/
 * We are interested only in file resources.
 **/
private void filterDirsAndJarType(FileSystem fs, List<String> matches) {
    try {
        Iterator<String> iter = matches.iterator();
        String path;
        while (iter.hasNext()) {
            path = iter.next();
            if (fs.isDirectory(new Path(path))) {
                iter.remove();
            } else if (type.equalsIgnoreCase("jar") && !path.endsWith(".jar")) {
                iter.remove();
            }
        }
    } catch (IOException e) {
        log.error("Exception while initializing filtering dirs", e);
    }
}

---

From source file:org.apache.metron.dataloads.bulk.HDFSDataPrunerTest.java

License:Apache License

@Test
public void testThrowsIsDirectory() throws Exception {

    FileSystem testFS = mock(FileSystem.class);
    when(testFS.isDirectory(any())).thenThrow(new IOException("Test Exception"));

    HDFSDataPruner pruner = new HDFSDataPruner(yesterday, 30, "file:///",
            dataPath.getAbsolutePath() + "/file-*");
    pruner.fileSystem = testFS;/*from   ww  w .  j  a  v  a2s  . c o  m*/
    HDFSDataPruner.DateFileFilter filter = new HDFSDataPruner.DateFileFilter(pruner, true);
    UnitTestHelper.setLog4jLevel(HDFSDataPruner.class, Level.FATAL);
    try {
        filter.accept(new Path("foo"));
        Assert.fail("Expected Runtime exception, but did not receive one.");
    } catch (RuntimeException e) {

    }
    UnitTestHelper.setLog4jLevel(HDFSDataPruner.class, Level.ERROR);
}

---

From source file:org.apache.metron.dataloads.bulk.HDFSDataPrunerTest.java

License:Apache License

@Test
public void testIgnoresDirectories() throws Exception {

    FileSystem testFS = mock(FileSystem.class);
    when(testFS.isDirectory(any())).thenReturn(true);

    HDFSDataPruner pruner = new HDFSDataPruner(yesterday, 30, "file:///",
            dataPath.getAbsolutePath() + "/file-*");
    pruner.fileSystem = testFS;/*from w w  w .j ava 2s  .c o m*/
    HDFSDataPruner.DateFileFilter filter = new HDFSDataPruner.DateFileFilter(pruner, false);
    assertFalse("Should ignore directories", filter.accept(new Path("/tmp")));

}

---

From source file:org.apache.metron.dataloads.bulk.HDFSDataPrunerTest.java

License:Apache License

@Test
public void testThrowBadFile() throws Exception {

    FileSystem testFS = mock(FileSystem.class);
    when(testFS.isDirectory(any())).thenReturn(false);
    when(testFS.getFileStatus(any())).thenThrow(new IOException("Test Exception"));

    HDFSDataPruner pruner = new HDFSDataPruner(yesterday, 30, "file:///",
            dataPath.getAbsolutePath() + "/file-*");

    pruner.fileSystem = testFS;//from  w  w  w .j  a va 2  s .c o m
    HDFSDataPruner.DateFileFilter filter = new HDFSDataPruner.DateFileFilter(pruner, true);
    UnitTestHelper.setLog4jLevel(HDFSDataPruner.class, Level.FATAL);
    try {
        filter.accept(new Path("foo"));
        Assert.fail("Expected Runtime exception, but did not receive one.");
    } catch (RuntimeException e) {
    }
    UnitTestHelper.setLog4jLevel(HDFSDataPruner.class, Level.ERROR);
}

---

From source file:org.apache.nutch.admin.AdministrationApp.java

License:Apache License

private NutchInstance[] getInstances(Configuration defaultConf, Path instancesRoot) throws IOException {
    //Path[] files = instancesRoot.listFiles();

    // Path[] files =  fs.listPaths(instancesRoot);
    FileSystem fs = FileSystem.get(getConf());
    FileStatus[] filestatuses = fs.listStatus(instancesRoot);
    int len = filestatuses.length;
    Path[] files = new Path[len];
    for (int i = 0; i < len; i++) {
        files[i] = filestatuses[i].getPath();
    }//from   ww  w .java2  s  .c o  m

    ArrayList<NutchInstance> instancesList = new ArrayList<NutchInstance>();
    for (int i = 0; i < files.length; i++) {
        Path folder = files[i];
        if (fs.isDirectory(folder) && !folder.getName().equals("conf")) {
            try {
                instancesList.add(loadNutchInstance(defaultConf, folder));
            } catch (IOException e) {
                LOG.warn("unable to load instance: " + e.toString());
            }
        }
    }
    return (NutchInstance[]) instancesList.toArray(new NutchInstance[instancesList.size()]);
}

---

From source file:org.apache.nutch.admin.AdministrationApp.java

License:Apache License

/**
 * creates an instance object from a instance folder
 * //from   ww  w  . j  a  v  a  2  s .c om
 * @param defaultConf
 * @param folder
 * @return an instance representation of this folder
 * @throws IOException
 *           in case the folder is not a valid instance folder
 */
public static NutchInstance loadNutchInstance(Configuration defaultConf, Path folder) throws IOException {
    Path instanceConfFolder = new Path(folder, "conf");

    Configuration conf = NutchConfiguration.create();
    FileSystem fs = FileSystem.get(conf);
    if (fs.exists(instanceConfFolder) && fs.isDirectory(instanceConfFolder)) {
        Path instanceSiteConf = new Path(instanceConfFolder, "nutch-site.xml");
        if (fs.exists(instanceSiteConf)) {
            Configuration instanceConf = new Configuration(defaultConf);
            instanceConf.addResource(instanceSiteConf.makeQualified(fs));
            return new NutchInstance(folder.getName(), folder, instanceConf);
        }
    }
    throw new IOException("not a valid instance folder: " + folder);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• »