Example usage for org.apache.hadoop.fs FileSystem listStatus

List of usage examples for org.apache.hadoop.fs FileSystem listStatus

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. listStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem listStatus.

Prototype

public FileStatus[] listStatus(Path[] files) throws FileNotFoundException, IOException

Source Link

Document

Filter files/directories in the given list of paths using default path filter.

Usage

From source file:com.practicalHadoop.outputformat.MultpleDirectories.FileOutputCommitter.java

License:Apache License

/**
 * Move all of the files from the work directory to the final output
 * @param context the task context/*w  w w .ja  v a2s. c o  m*/
 * @param fs the output file system
 * @param jobOutputDir the final output direcotry
 * @param taskOutput the work path
 * @throws IOException
 */
private void moveTaskOutputs(TaskAttemptContext context, FileSystem fs, Path jobOutputDir, Path taskOutput)
        throws IOException {
    TaskAttemptID attemptId = context.getTaskAttemptID();
    context.progress();
    if (fs.isFile(taskOutput)) {
        Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, workPath);
        if (!fs.rename(taskOutput, finalOutputPath)) {
            if (!fs.delete(finalOutputPath, true)) {
                throw new IOException("Failed to delete earlier output of task: " + attemptId);
            }
            if (!fs.rename(taskOutput, finalOutputPath)) {
                throw new IOException("Failed to save output of task: " + attemptId);
            }
        }
        LOG.debug("Moved " + taskOutput + " to " + finalOutputPath);
    } else if (fs.getFileStatus(taskOutput).isDir()) {
        FileStatus[] paths = fs.listStatus(taskOutput);
        Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, workPath);
        fs.mkdirs(finalOutputPath);
        if (paths != null) {
            for (FileStatus path : paths) {
                moveTaskOutputs(context, fs, jobOutputDir, path.getPath());
            }
        }
    }
}

From source file:com.rapleaf.hank.hadoop.DomainBuilderOutputCommitter.java

License:Apache License

public static void commitJob(String domainName, JobConf conf) throws IOException {
    Path outputPath = new Path(DomainBuilderProperties.getOutputPath(domainName, conf));
    Path tmpOutputPath = new Path(DomainBuilderProperties.getTmpOutputPath(domainName, conf));
    FileSystem fs = outputPath.getFileSystem(conf);

    // Create outputPath
    fs.mkdirs(outputPath);//from w ww.  jav  a2 s. c  o  m

    // Move temporary output to final output
    LOG.info("Moving temporary output files from: " + tmpOutputPath + " to final output path: " + outputPath);
    FileStatus[] partitions = fs.listStatus(tmpOutputPath);
    for (FileStatus partition : partitions) {
        if (partition.isDir()) {
            FileStatus[] partitionFiles = fs.listStatus(partition.getPath());
            for (FileStatus partitionFile : partitionFiles) {
                Path sourcePath = partitionFile.getPath();
                Path targetPath = new Path(new Path(outputPath, partition.getPath().getName()),
                        partitionFile.getPath().getName());
                LOG.info("Moving: " + sourcePath + " to: " + targetPath);
                if (!fs.mkdirs(targetPath.getParent())) {
                    throw new IOException("Failed at creating directory " + targetPath.getParent());
                }
                if (!fs.rename(sourcePath, targetPath)) {
                    throw new IOException("Failed at renaming " + sourcePath + " to " + targetPath);
                }
            }
        }
    }

    // Finally, cleanup
    cleanupJob(domainName, conf);
}

From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java

License:Open Source License

/**
 * Read rows from the path provide//w  ww  .  j a v  a2 s .c o  m
 * 
 * @param path
 * @param delimiter
 * @param maxToRead
 * @throws RemoteException
 */
@Override
public List<String> select(String path, String delimiter, int maxToRead) throws RemoteException {
    Path p = new Path(path);
    List<String> ans = null;
    HdfsFileChecker fCh = new HdfsFileChecker(p);
    try {
        FileSystem fs = NameNodeVar.getFS();
        if (fCh.isDirectory()) {
            FileStatus[] fsA = fs.listStatus(p);
            int listSize = Math.min(maxToRead, fsA.length);
            ans = new ArrayList<String>(listSize);
            for (int i = 0; i < listSize; ++i) {
                ans.add(fsA[i].getPath().toString());
            }
        } else if (fCh.isFile()) {
            InputStream inS = fs.open(p);
            InputStream in = null;
            InputStream compressedReader = null;
            if (path.endsWith(".bz2") || path.endsWith(".bz")) {
                compressedReader = new BZip2CompressorInputStream(inS);
                in = compressedReader;
            } else if (path.endsWith(".gz")) {
                compressedReader = new GZIPInputStream(inS);
                in = compressedReader;
            } else {
                in = inS;
            }

            LineReader reader = new LineReader(in);
            ans = new ArrayList<String>(maxToRead);
            Text line = new Text();
            int lineNb = 0;
            while (reader.readLine(line) != 0 && lineNb < maxToRead) {
                ans.add(line.toString());
                ++lineNb;
            }
            if (compressedReader != null) {
                compressedReader.close();
            }
            inS.close();
        }
        // fs.close();
    } catch (IOException e) {
        logger.error("Cannot select the file or directory: " + p);
        logger.error(e.getMessage(), e);
    }
    // fCh.close();

    return ans;
}

From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java

License:Open Source License

/**
 * Read a Sequence File/*  w w  w .  j  a v a2s  .  c o  m*/
 * 
 * @param path
 * @param delimiter
 * @param maxToRead
 * @param fields
 * @return List of read rows from the path
 * @throws RemoteException
 */
public List<String> selectSeq(String path, String delimiter, int maxToRead, FieldList fields)
        throws RemoteException {

    Path p = new Path(path);
    List<String> ans = null;
    HdfsFileChecker fCh = new HdfsFileChecker(p);
    try {
        FileSystem fs = NameNodeVar.getFS();
        if (fCh.isDirectory()) {
            FileStatus[] fsA = fs.listStatus(p);
            int listSize = Math.min(maxToRead, fsA.length);
            ans = new ArrayList<String>(listSize);
            for (int i = 0; i < listSize; ++i) {
                ans.add(fsA[i].getPath().toString());
            }
        } else if (fCh.isFile()) {
            FSDataInputStream in = fs.open(p);
            LineReader reader = new LineReader(in);
            ans = new ArrayList<String>(maxToRead);
            Text line = new Text();
            reader.readLine(line);
            int lineNb = 0;
            maxToRead *= fields.getSize();
            int i = 0;
            String toWrite = "";
            logger.debug("delim : " + delimiter);
            while (reader.readLine(line) != 0 && lineNb < maxToRead) {
                reader.readLine(line);
                logger.debug("line : " + line);
                ++lineNb;

                FieldType type = fields.getFieldType(fields.getFieldNames().get(i));
                if (type == FieldType.BOOLEAN) {
                    toWrite += BytesWritable.Comparator.readInt(line.getBytes(), 0);
                } else if (type == FieldType.INT) {
                    toWrite += BytesWritable.Comparator.readInt(line.getBytes(), 0);
                } else if (type == FieldType.FLOAT) {
                    toWrite += BytesWritable.Comparator.readFloat(line.getBytes(), 0);
                } else if (type == FieldType.DOUBLE) {
                    toWrite += BytesWritable.Comparator.readDouble(line.getBytes(), 0);
                } else if (type == FieldType.LONG) {
                    toWrite += BytesWritable.Comparator.readLong(line.getBytes(), 0);
                } else if (type == FieldType.STRING) {
                    toWrite += line.getBytes().toString();
                }
                if ((i + 1) % fields.getSize() == 0) {
                    ans.add(toWrite);
                    toWrite = "";
                } else {
                    toWrite += '\001';
                }
                ++i;
                if (i >= fields.getSize()) {
                    i = 0;
                }

            }
        }
        // fs.close();
    } catch (IOException e) {
        logger.error("Cannot select the file or directory: " + p);
        logger.error(e.getMessage());
    }
    // fCh.close();

    return ans;
}

From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java

License:Open Source License

@Override
public Map<String, Map<String, String>> getChildrenProperties(String pathStr) throws RemoteException {
    Map<String, Map<String, String>> ans = null;
    if (cach.get(getBrowserName()) != null) {
        ans = cach.get(getBrowserName()).get(pathStr);
    }/*from  www . j a v a  2  s.c o  m*/
    if (ans == null) {
        ans = new LinkedHashMap<String, Map<String, String>>();
        Path pathHdfs = new Path(pathStr);
        HdfsFileChecker fCh = new HdfsFileChecker(pathHdfs);
        try {
            FileSystem fs = NameNodeVar.getFS();
            if (fCh.isDirectory()) {
                FileStatus[] fsA = fs.listStatus(pathHdfs);

                for (int i = 0; i < fsA.length; ++i) {
                    String path = pathStr + "/" + fsA[i].getPath().getName();
                    ans.put(path, getProperties(path, fsA[i]));
                }
            } else {
                ans = null;
            }
            // fs.close();
        } catch (IOException e) {
            logger.error("Cannot open the directory: " + pathStr);
            logger.error(e.getMessage());
        }
        if (ans != null) {
            cach.get(getBrowserName()).put(pathStr, ans);
        }
    }
    // fCh.close();

    return ans;
}

From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java

License:Open Source License

/**
 * Change Ownership of a Path//from w  w w  .j a  va2 s.  c  om
 * 
 * @param path
 * @param owner
 * @param group
 * @param recursive
 * @return Error Message
 */
protected String changeOwnership(Path path, String owner, String group, boolean recursive) {
    String error = null;
    try {
        FileSystem fs = NameNodeVar.getFS();
        FileStatus stat = fs.getFileStatus(path);
        if (stat.getOwner().equals(System.getProperty("user.name"))) {
            if (recursive) {
                FileStatus[] fsA = fs.listStatus(path);

                for (int i = 0; i < fsA.length && error == null; ++i) {
                    error = changeOwnership(fs, fsA[i].getPath(), owner, group, recursive);
                }
            }
            if (error == null) {
                fs.setOwner(path, owner, group);
            }
        } else {
            error = LanguageManagerWF.getText("HdfsInterface.changeprop.ownererror",
                    new Object[] { path.toString() });
        }
        // fs.close();
    } catch (IOException e) {
        logger.error("Cannot operate on the file or directory: " + path.toString());
        logger.error(e.getMessage());
        error = LanguageManagerWF.getText("HdfsInterface.changeprop.fileaccess", new Object[] { path });
    }

    if (error != null) {
        logger.debug(error);
    }
    return error;
}

From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java

License:Open Source License

/**
 * Change Ownership of a Path/* w  w  w.  j  a  va2s .c om*/
 * 
 * @param fs
 * @param path
 * @param owner
 * @param group
 * @param recursive
 * @return Error Message
 */
protected String changeOwnership(FileSystem fs, Path path, String owner, String group, boolean recursive) {
    String error = null;
    try {
        FileStatus stat = fs.getFileStatus(path);
        if (stat.getOwner().equals(System.getProperty("user.name"))) {

            if (recursive) {
                FileStatus[] fsA = fs.listStatus(path);

                for (int i = 0; i < fsA.length && error == null; ++i) {
                    error = changeOwnership(fs, fsA[i].getPath(), owner, group, recursive);
                }
            }
            if (error == null) {
                fs.setOwner(path, owner, group);
            }
        } else {
            error = LanguageManagerWF.getText("HdfsInterface.changeprop.ownererror",
                    new Object[] { path.toString() });
        }
        // fs.close();
    } catch (IOException e) {
        logger.error("Cannot operate on the file or directory: " + path.toString());
        logger.error(e.getMessage());
        error = LanguageManagerWF.getText("HdfsInterface.changeprop.fileaccess", new Object[] { path });
    }
    if (error != null) {
        logger.debug(error);
    }
    return error;
}

From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java

License:Open Source License

/**
 * Change the permissions of a path//  ww  w .  j av  a 2  s  .c  o m
 * 
 * @param fs
 * @param path
 * @param permission
 * @param recursive
 * @return Error Message
 */
protected String changePermission(FileSystem fs, Path path, String permission, boolean recursive) {
    String error = null;
    try {
        FileStatus stat = fs.getFileStatus(path);
        if (stat.getOwner().equals(System.getProperty("user.name"))) {
            if (recursive) {
                FileStatus[] child = fs.listStatus(path);
                for (int i = 0; i < child.length && error == null; ++i) {
                    error = changePermission(fs, child[i].getPath(), permission, recursive);
                }
            }
            if (error == null) {
                logger.debug("1 ----- path " + path.getName() + " new perms " + permission);
                fs.setPermission(path, new FsPermission(permission));
            }
        } else {
            error = LanguageManagerWF.getText("HdfsInterface.changeprop.ownererror",
                    new Object[] { path.toString() });
        }
    } catch (IOException e) {
        logger.error("Cannot operate on the file or directory: " + path.toString());
        logger.error(e.getMessage());
        error = LanguageManagerWF.getText("HdfsInterface.changeprop.fileaccess", new Object[] { path });
    }
    if (error != null) {
        logger.debug(error);
    }
    return error;
}

From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java

License:Open Source License

/**
 * Change the permission of a path//from  ww  w  .  ja  v a2  s.  co m
 * 
 * @param path
 * @param permission
 * @param recursive
 * @return Error Message
 */
protected String changePermission(Path path, String permission, boolean recursive) {
    String error = null;
    try {
        logger.debug("1 " + path.getName());
        FileSystem fs = NameNodeVar.getFS();
        FileStatus stat = fs.getFileStatus(path);
        if (stat.getOwner().equals(System.getProperty("user.name"))) {
            FileStatus[] child = fs.listStatus(path);
            if (recursive) {
                logger.debug("children : " + child.length);
                for (int i = 0; i < child.length && error == null; ++i) {
                    error = changePermission(fs, child[i].getPath(), permission, recursive);
                }
            }
            if (error == null) {
                logger.debug("set permissions  : " + path.toString() + " , "
                        + new FsPermission(permission).toString());
                fs.setPermission(path, new FsPermission(permission));
                logger.debug(getProperties(path.getName()));
            }
        } else {
            error = LanguageManagerWF.getText("HdfsInterface.changeprop.ownererror",
                    new Object[] { path.toString() });
        }
        // fs.close();
    } catch (IOException e) {
        logger.error("Cannot operate on the file or directory: " + path.toString());
        logger.error(e.getMessage());
        error = LanguageManagerWF.getText("HdfsInterface.changeprop.fileaccess", new Object[] { path });
    }
    if (error != null) {
        logger.debug(error);
    }
    return error;
}

From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java

License:Open Source License

private String copyInRemote(Channel channel, String lfile, String rfile) {
    String error = null;//  www. j  av  a 2  s  .c o m
    try {
        FileSystem fs = NameNodeVar.getFS();
        Path src = new Path(lfile);

        if (fs.getFileStatus(src).isDir()) {
            FileStatus contents[] = fs.listStatus(src);
            //Make directory in remote
            channel = channel.getSession().openChannel("exec");
            ((ChannelExec) channel).setCommand("mkdir -p " + rfile);
            channel.connect();
            channel.disconnect();
            for (int i = 0; i < contents.length && error == null; i++) {
                String filename = contents[i].getPath().getName();
                error = copyInRemote(channel, lfile + "/" + filename, rfile + "/" + filename);
            }
        } else {
            error = copyFileInRemote(channel, lfile, rfile);
        }

    } catch (Exception e) {
        error = "Unexpected error when copying file accross: " + e.getMessage();
        logger.error(error, e);
    }
    return error;
}