Example usage for org.apache.hadoop.fs FileSystem getFileStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getFileStatus.

Prototype

public abstract FileStatus getFileStatus(Path f) throws IOException;

Source Link

Document

Return a file status object that represents the path.

Usage

From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java

License:Open Source License

/**
 * Change Ownership of a Path/* ww  w . j  a  v  a  2s . com*/
 * 
 * @param fs
 * @param path
 * @param owner
 * @param group
 * @param recursive
 * @return Error Message
 */
protected String changeOwnership(FileSystem fs, Path path, String owner, String group, boolean recursive) {
    String error = null;
    try {
        FileStatus stat = fs.getFileStatus(path);
        if (stat.getOwner().equals(System.getProperty("user.name"))) {

            if (recursive) {
                FileStatus[] fsA = fs.listStatus(path);

                for (int i = 0; i < fsA.length && error == null; ++i) {
                    error = changeOwnership(fs, fsA[i].getPath(), owner, group, recursive);
                }
            }
            if (error == null) {
                fs.setOwner(path, owner, group);
            }
        } else {
            error = LanguageManagerWF.getText("HdfsInterface.changeprop.ownererror",
                    new Object[] { path.toString() });
        }
        // fs.close();
    } catch (IOException e) {
        logger.error("Cannot operate on the file or directory: " + path.toString());
        logger.error(e.getMessage());
        error = LanguageManagerWF.getText("HdfsInterface.changeprop.fileaccess", new Object[] { path });
    }
    if (error != null) {
        logger.debug(error);
    }
    return error;
}

---

From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java

License:Open Source License

/**
 * Change the permissions of a path//  ww w .  java 2 s  . c o  m
 * 
 * @param fs
 * @param path
 * @param permission
 * @param recursive
 * @return Error Message
 */
protected String changePermission(FileSystem fs, Path path, String permission, boolean recursive) {
    String error = null;
    try {
        FileStatus stat = fs.getFileStatus(path);
        if (stat.getOwner().equals(System.getProperty("user.name"))) {
            if (recursive) {
                FileStatus[] child = fs.listStatus(path);
                for (int i = 0; i < child.length && error == null; ++i) {
                    error = changePermission(fs, child[i].getPath(), permission, recursive);
                }
            }
            if (error == null) {
                logger.debug("1 ----- path " + path.getName() + " new perms " + permission);
                fs.setPermission(path, new FsPermission(permission));
            }
        } else {
            error = LanguageManagerWF.getText("HdfsInterface.changeprop.ownererror",
                    new Object[] { path.toString() });
        }
    } catch (IOException e) {
        logger.error("Cannot operate on the file or directory: " + path.toString());
        logger.error(e.getMessage());
        error = LanguageManagerWF.getText("HdfsInterface.changeprop.fileaccess", new Object[] { path });
    }
    if (error != null) {
        logger.debug(error);
    }
    return error;
}

---

From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java

License:Open Source License

/**
 * Change the permission of a path//from  w w  w.j  av a 2 s .c  o  m
 * 
 * @param path
 * @param permission
 * @param recursive
 * @return Error Message
 */
protected String changePermission(Path path, String permission, boolean recursive) {
    String error = null;
    try {
        logger.debug("1 " + path.getName());
        FileSystem fs = NameNodeVar.getFS();
        FileStatus stat = fs.getFileStatus(path);
        if (stat.getOwner().equals(System.getProperty("user.name"))) {
            FileStatus[] child = fs.listStatus(path);
            if (recursive) {
                logger.debug("children : " + child.length);
                for (int i = 0; i < child.length && error == null; ++i) {
                    error = changePermission(fs, child[i].getPath(), permission, recursive);
                }
            }
            if (error == null) {
                logger.debug("set permissions  : " + path.toString() + " , "
                        + new FsPermission(permission).toString());
                fs.setPermission(path, new FsPermission(permission));
                logger.debug(getProperties(path.getName()));
            }
        } else {
            error = LanguageManagerWF.getText("HdfsInterface.changeprop.ownererror",
                    new Object[] { path.toString() });
        }
        // fs.close();
    } catch (IOException e) {
        logger.error("Cannot operate on the file or directory: " + path.toString());
        logger.error(e.getMessage());
        error = LanguageManagerWF.getText("HdfsInterface.changeprop.fileaccess", new Object[] { path });
    }
    if (error != null) {
        logger.debug(error);
    }
    return error;
}

---

From source file:com.redsqirl.workflow.server.connect.HDFSInterface.java

License:Open Source License

private String copyInRemote(Channel channel, String lfile, String rfile) {
    String error = null;//www.  ja v a  2s.c o  m
    try {
        FileSystem fs = NameNodeVar.getFS();
        Path src = new Path(lfile);

        if (fs.getFileStatus(src).isDir()) {
            FileStatus contents[] = fs.listStatus(src);
            //Make directory in remote
            channel = channel.getSession().openChannel("exec");
            ((ChannelExec) channel).setCommand("mkdir -p " + rfile);
            channel.connect();
            channel.disconnect();
            for (int i = 0; i < contents.length && error == null; i++) {
                String filename = contents[i].getPath().getName();
                error = copyInRemote(channel, lfile + "/" + filename, rfile + "/" + filename);
            }
        } else {
            error = copyFileInRemote(channel, lfile, rfile);
        }

    } catch (Exception e) {
        error = "Unexpected error when copying file accross: " + e.getMessage();
        logger.error(error, e);
    }
    return error;
}

---

From source file:com.ricemap.spateDB.core.SpatialSite.java

License:Apache License

public static GlobalIndex<Partition> getGlobalIndex(FileSystem fs, Path dir) throws IOException {
    // Retrieve the master file (the only file with the name _master in it)
    FileStatus[] masterFiles = fs.listStatus(dir, new PathFilter() {
        @Override//from w w  w  .  j  a  va 2s. co  m
        public boolean accept(Path path) {
            return path.getName().contains("_master");
        }
    });
    // Check if the given file is indexed
    if (masterFiles.length == 0)
        return null;
    if (masterFiles.length > 1)
        throw new RuntimeException("Found more than one master file in " + dir);
    Path masterFile = masterFiles[0].getPath();
    ShapeRecordReader<Partition> reader = new ShapeRecordReader<Partition>(fs.open(masterFile), 0,
            fs.getFileStatus(masterFile).getLen());
    CellInfo dummy = new CellInfo();
    Partition partition = new Partition();
    ArrayList<Partition> partitions = new ArrayList<Partition>();
    while (reader.next(dummy, partition)) {
        partitions.add(partition.clone());
    }
    GlobalIndex<Partition> globalIndex = new GlobalIndex<Partition>();
    globalIndex.bulkLoad(partitions.toArray(new Partition[partitions.size()]));
    globalIndex.setCompact(masterFile.getName().endsWith("rtree") || masterFile.getName().endsWith("r+tree"));
    globalIndex.setReplicated(masterFile.getName().endsWith("r+tree") || masterFile.getName().endsWith("grid"));
    return globalIndex;
}

---

From source file:com.ricemap.spateDB.core.SpatialSite.java

License:Apache License

public static boolean isRTree(FileSystem fs, Path path) throws IOException {
    FileStatus file = fs.getFileStatus(path);
    Path fileToCheck;/*from   w w  w  .j a v a  2 s . c  o  m*/
    if (file.isDir()) {
        // Check any cell (e.g., first cell)
        GlobalIndex<Partition> gIndex = getGlobalIndex(fs, path);
        if (gIndex == null)
            return false;
        fileToCheck = new Path(path, gIndex.iterator().next().filename);
    } else {
        fileToCheck = file.getPath();
    }
    InputStream fileIn = fs.open(fileToCheck);

    // Check if file is compressed
    CompressionCodec codec = compressionCodecs.getCodec(fileToCheck);
    Decompressor decompressor = null;
    if (codec != null) {
        decompressor = CodecPool.getDecompressor(codec);
        fileIn = codec.createInputStream(fileIn, decompressor);
    }
    byte[] signature = new byte[RTreeFileMarkerB.length];
    fileIn.read(signature);
    fileIn.close();
    if (decompressor != null) {
        CodecPool.returnDecompressor(decompressor);
    }
    return Arrays.equals(signature, SpatialSite.RTreeFileMarkerB);
}

---

From source file:com.ricemap.spateDB.mapred.IndexedPrism.java

License:Apache License

@SuppressWarnings("unchecked")
@Override/*  w ww.  ja v a2s .c o  m*/
public InputSplit[] getSplits(final JobConf job, int numSplits) throws IOException {
    // Get a list of all input files. There should be exactly two files.
    final Path[] inputFiles = getInputPaths(job);
    GlobalIndex<Partition> gIndexes[] = new GlobalIndex[inputFiles.length];

    BlockFilter blockFilter = null;
    try {
        Class<? extends BlockFilter> blockFilterClass = job.getClass(SpatialSite.FilterClass, null,
                BlockFilter.class);
        if (blockFilterClass != null) {
            // Get all blocks the user wants to process
            blockFilter = blockFilterClass.newInstance();
            blockFilter.configure(job);
        }
    } catch (InstantiationException e1) {
        e1.printStackTrace();
    } catch (IllegalAccessException e1) {
        e1.printStackTrace();
    }

    if (blockFilter != null) {
        // Extract global indexes from input files

        for (int i_file = 0; i_file < inputFiles.length; i_file++) {
            FileSystem fs = inputFiles[i_file].getFileSystem(job);
            gIndexes[i_file] = SpatialSite.getGlobalIndex(fs, inputFiles[i_file]);
        }
    }

    final Vector<CombineFileSplit> matchedSplits = new Vector<CombineFileSplit>();
    if (gIndexes[0] == null || gIndexes[1] == null) {
        // Join every possible pair (Cartesian product)
        BlockLocation[][] fileBlockLocations = new BlockLocation[inputFiles.length][];
        for (int i_file = 0; i_file < inputFiles.length; i_file++) {
            FileSystem fs = inputFiles[i_file].getFileSystem(job);
            FileStatus fileStatus = fs.getFileStatus(inputFiles[i_file]);
            fileBlockLocations[i_file] = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
        }
        LOG.info("Doing a Cartesian product of blocks: " + fileBlockLocations[0].length + "x"
                + fileBlockLocations[1].length);
        for (BlockLocation block1 : fileBlockLocations[0]) {
            for (BlockLocation block2 : fileBlockLocations[1]) {
                FileSplit fsplit1 = new FileSplit(inputFiles[0], block1.getOffset(), block1.getLength(),
                        block1.getHosts());
                FileSplit fsplit2 = new FileSplit(inputFiles[1], block2.getOffset(), block2.getLength(),
                        block2.getHosts());
                CombineFileSplit combinedSplit = (CombineFileSplit) FileSplitUtil.combineFileSplits(job,
                        fsplit1, fsplit2);
                matchedSplits.add(combinedSplit);
            }
        }
    } else {
        // Filter block pairs by the BlockFilter
        blockFilter.selectCellPairs(gIndexes[0], gIndexes[1], new ResultCollector2<Partition, Partition>() {
            @Override
            public void collect(Partition p1, Partition p2) {
                try {
                    List<FileSplit> splits1 = new ArrayList<FileSplit>();
                    Path path1 = new Path(inputFiles[0], p1.filename);
                    splitFile(job, path1, splits1);

                    List<FileSplit> splits2 = new ArrayList<FileSplit>();
                    Path path2 = new Path(inputFiles[1], p2.filename);
                    splitFile(job, path2, splits2);

                    for (FileSplit split1 : splits1) {
                        for (FileSplit split2 : splits2) {
                            matchedSplits.add(
                                    (CombineFileSplit) FileSplitUtil.combineFileSplits(job, split1, split2));
                        }
                    }

                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        });
    }

    LOG.info("Matched " + matchedSplits.size() + " combine splits");

    // Return all matched splits
    return matchedSplits.toArray(new InputSplit[matchedSplits.size()]);
}

---

From source file:com.ricemap.spateDB.mapred.IndexedPrism.java

License:Apache License

public void splitFile(JobConf job, Path path, List<FileSplit> splits) throws IOException {
    NetworkTopology clusterMap = new NetworkTopology();
    FileSystem fs = path.getFileSystem(job);
    FileStatus file = fs.getFileStatus(path);
    long length = file.getLen();
    BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length);
    if (length != 0) {
        long blockSize = file.getBlockSize();
        long splitSize = blockSize;

        long bytesRemaining = length;
        while (((double) bytesRemaining) / splitSize > SPLIT_SLOP) {
            String[] splitHosts = getSplitHosts(blkLocations, length - bytesRemaining, splitSize, clusterMap);
            splits.add(new FileSplit(path, length - bytesRemaining, splitSize, splitHosts));
            bytesRemaining -= splitSize;
        }/*w  w  w.j  av  a  2s.c o m*/

        if (bytesRemaining != 0) {
            splits.add(new FileSplit(path, length - bytesRemaining, bytesRemaining,
                    blkLocations[blkLocations.length - 1].getHosts()));
        }
    } else if (length != 0) {
        String[] splitHosts = getSplitHosts(blkLocations, 0, length, clusterMap);
        splits.add(new FileSplit(path, 0, length, splitHosts));
    } else {
        //Create empty hosts array for zero length files
        splits.add(new FileSplit(path, 0, length, new String[0]));
    }
}

---

From source file:com.ricemap.spateDB.mapred.FileSplitUtil.java

License:Apache License

/**
 * Combines a number of input splits into the given numSplits.
 * @param conf//from w  w w  .ja  v a  2 s . c o  m
 * @param inputSplits
 * @param numSplits
 * @return
 * @throws IOException 
 */
public static InputSplit[] autoCombineSplits(JobConf conf, Vector<FileSplit> inputSplits, int numSplits)
        throws IOException {
    LOG.info("Combining " + inputSplits.size() + " splits into " + numSplits);
    Map<String, Vector<FileSplit>> blocksPerHost = new HashMap<String, Vector<FileSplit>>();
    for (FileSplit fsplit : inputSplits) {
        // Get locations for this split
        final Path path = fsplit.getPath();
        final FileSystem fs = path.getFileSystem(conf);
        BlockLocation[] blockLocations = fs.getFileBlockLocations(fs.getFileStatus(path), fsplit.getStart(),
                fsplit.getLength());
        for (BlockLocation blockLocation : blockLocations) {
            for (String hostName : blockLocation.getHosts()) {
                if (!blocksPerHost.containsKey(hostName))
                    blocksPerHost.put(hostName, new Vector<FileSplit>());
                blocksPerHost.get(hostName).add(fsplit);
            }
        }
    }

    // If the user requested a fewer number of splits, start to combine them
    InputSplit[] combined_splits = new InputSplit[numSplits];
    int splitsAvailable = inputSplits.size();

    for (int i = 0; i < numSplits; i++) {
        // Decide how many splits to combine
        int numSplitsToCombine = splitsAvailable / (numSplits - i);
        Vector<FileSplit> splitsToCombine = new Vector<FileSplit>();
        while (numSplitsToCombine > 0) {
            // Choose the host with minimum number of splits
            Map.Entry<String, Vector<FileSplit>> minEntry = null;
            for (Map.Entry<String, Vector<FileSplit>> entry : blocksPerHost.entrySet()) {
                if (minEntry == null || entry.getValue().size() < minEntry.getValue().size()) {
                    minEntry = entry;
                }
            }
            // Combine all or some of blocks in this host
            for (FileSplit fsplit : minEntry.getValue()) {
                if (!splitsToCombine.contains(fsplit)) {
                    splitsToCombine.add(fsplit);
                    if (--numSplitsToCombine == 0)
                        break;
                }
            }
            if (numSplitsToCombine != 0) {
                // Remove this host so that it is not selected again
                blocksPerHost.remove(minEntry.getKey());
            }
        }

        combined_splits[i] = combineFileSplits(conf, splitsToCombine, 0, splitsToCombine.size());

        for (Map.Entry<String, Vector<FileSplit>> entry : blocksPerHost.entrySet()) {
            entry.getValue().removeAll(splitsToCombine);
        }
        splitsAvailable -= splitsToCombine.size();
    }

    LOG.info("Combined splits " + combined_splits.length);
    return combined_splits;
}

---

From source file:com.ricemap.spateDB.mapred.SpatialInputFormat.java

License:Apache License

protected void listStatus(final FileSystem fs, final Path dir, final List<FileStatus> result,
        BlockFilter filter) throws IOException {
    GlobalIndex<Partition> gindex = SpatialSite.getGlobalIndex(fs, dir);
    FileStatus[] listStatus = fs.listStatus(dir, hiddenFileFilter);
    if (gindex == null) {
        // Add all files under this directory
        for (FileStatus status : listStatus) {
            if (status.isDir()) {
                listStatus(fs, status.getPath(), result, filter);
            } else {
                result.add(status);//w  w  w. ja va 2  s  .co  m
            }
        }
    } else {
        // Use the global index to limit files
        filter.selectCells(gindex, new ResultCollector<Partition>() {
            @Override
            public void collect(Partition partition) {
                try {
                    Path cell_path = new Path(dir, partition.filename);
                    if (!fs.exists(cell_path))
                        LOG.warn("Matched file not found: " + cell_path);
                    result.add(fs.getFileStatus(cell_path));
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        });
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »