Example usage for org.apache.hadoop.fs FileSystem listStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem listStatus.

Prototype

public FileStatus[] listStatus(Path[] files) throws FileNotFoundException, IOException

Source Link

Document

Filter files/directories in the given list of paths using default path filter.

Usage

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>/*from   w  ww  .  j  a  va  2  s.  c  o m*/
 * Clears a complete directory.
 * </p>
 * 
 * @param directory
 *            directory
 */
public static void clearDirectory(String directory) {
    try {
        FileSystem fs = FileSystem.get(conf);
        FileStatus[] directoryContent = fs.listStatus(new Path(directory));
        for (FileStatus content : directoryContent) {
            fs.delete(content.getPath(), true);
        }
    } catch (IOException e) {
    }
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>/*from ww w . j  a v a2  s.  c  o m*/
 * Removes all temporary files and directories in the current working
 * directory.
 * </p>
 */
public static void removeTemporaryFiles() {
    try {
        FileSystem fs = FileSystem.get(conf);
        Path workingDir = new Path(".");
        FileStatus[] files = fs.listStatus(workingDir);
        for (FileStatus file : files) {
            String fileName = file.getPath().toString()
                    .substring(file.getPath().getParent().toString().length() + 1);
            if (fileName.contains("temp"))
                fs.delete(file.getPath(), false);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("unable to remove temporary files: " + e.getMessage());
    }
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>//from   w w w . j a v a2s.  c  o m
 * Checks if any temporary files or directories exist in the current working
 * directory.
 * </p>
 * 
 * @return true if temporary files or directories are available
 */
public static boolean checkForTemporaryFiles() {
    try {
        FileSystem fs = FileSystem.get(conf);
        Path workingDir = new Path(".");
        FileStatus[] files = fs.listStatus(workingDir);
        for (FileStatus file : files) {
            String fileName = file.getPath().toString()
                    .substring(file.getPath().getParent().toString().length() + 1);
            if (fileName.contains("temp"))
                return true;
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("unable to remove temporary files: " + e.getMessage());
    }

    return false;
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>/*  www . j a va  2 s  .c  o  m*/
 * Reads binary cells from a file. A matrix characteristic is created which
 * contains the characteristics of the matrix read from the file and the
 * values.
 * </p>
 * 
 * @param directory
 *            directory containing the matrix
 * @return matrix characteristics
 */
@SuppressWarnings("deprecation")
public static BinaryMatrixCharacteristics readCellsFromSequenceFile(String directory) {
    try {
        FileSystem fs = FileSystem.get(conf);
        FileStatus[] files = fs.listStatus(new Path(directory));

        HashMap<MatrixIndexes, Double> valueMap = new HashMap<MatrixIndexes, Double>();
        int rows = 0;
        int cols = 0;
        MatrixIndexes indexes = new MatrixIndexes();
        MatrixCell value = new MatrixCell();
        for (FileStatus file : files) {
            SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf), file.getPath(), conf);

            while (reader.next(indexes, value)) {
                if (rows < indexes.getRowIndex())
                    rows = (int) indexes.getRowIndex();
                if (cols < indexes.getColumnIndex())
                    cols = (int) indexes.getColumnIndex();

                valueMap.put(new MatrixIndexes(indexes), value.getValue());
            }

            reader.close();
        }

        double[][] values = new double[rows][cols];
        long nonZeros = 0;
        for (MatrixIndexes index : valueMap.keySet()) {
            values[(int) index.getRowIndex() - 1][(int) index.getColumnIndex() - 1] = valueMap.get(index);
            if (valueMap.get(index) != 0)
                nonZeros++;
        }

        return new BinaryMatrixCharacteristics(values, rows, cols, 0, 0, 0, 0, nonZeros);
    } catch (IOException e) {
        e.printStackTrace();
        fail("unable to read sequence file in " + directory);
    }

    return null;
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>/*from ww  w.j  a v  a 2  s  . co  m*/
 * Reads binary blocks from a file. A matrix characteristic is created which
 * contains the characteristics of the matrix read from the file and the
 * values.
 * </p>
 * 
 * @param directory
 *            directory containing the matrix
 * @param rowsInBlock
 *            rows in block
 * @param colsInBlock
 *            columns in block
 * @return matrix characteristics
 */
@SuppressWarnings("deprecation")
public static BinaryMatrixCharacteristics readBlocksFromSequenceFile(String directory, int rowsInBlock,
        int colsInBlock) {
    try {
        FileSystem fs = FileSystem.get(conf);
        FileStatus[] files = fs.listStatus(new Path(directory));

        HashMap<MatrixIndexes, Double> valueMap = new HashMap<MatrixIndexes, Double>();
        int rowsInLastBlock = -1;
        int colsInLastBlock = -1;
        int rows = 0;
        int cols = 0;
        MatrixIndexes indexes = new MatrixIndexes();
        MatrixBlock value = new MatrixBlock();
        for (FileStatus file : files) {
            SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf), file.getPath(), conf);

            while (reader.next(indexes, value)) {
                if (value.getNumRows() < rowsInBlock) {
                    if (rowsInLastBlock == -1)
                        rowsInLastBlock = value.getNumRows();
                    else if (rowsInLastBlock != value.getNumRows())
                        fail("invalid block sizes");
                    rows = (int) ((indexes.getRowIndex() - 1) * rowsInBlock + value.getNumRows());
                } else if (value.getNumRows() == rowsInBlock) {
                    if (rows <= (indexes.getRowIndex() * rowsInBlock + value.getNumRows())) {
                        if (rowsInLastBlock == -1)
                            rows = (int) ((indexes.getRowIndex() - 1) * rowsInBlock + value.getNumRows());
                        else
                            fail("invalid block sizes");
                    }
                } else {
                    fail("invalid block sizes");
                }

                if (value.getNumColumns() < colsInBlock) {
                    if (colsInLastBlock == -1)
                        colsInLastBlock = value.getNumColumns();
                    else if (colsInLastBlock != value.getNumColumns())
                        fail("invalid block sizes");
                    cols = (int) ((indexes.getColumnIndex() - 1) * colsInBlock + value.getNumColumns());
                } else if (value.getNumColumns() == colsInBlock) {
                    if (cols <= (indexes.getColumnIndex() * colsInBlock + value.getNumColumns())) {
                        if (colsInLastBlock == -1)
                            cols = (int) ((indexes.getColumnIndex() - 1) * colsInBlock + value.getNumColumns());
                        else
                            fail("invalid block sizes");
                    }
                } else {
                    fail("invalid block sizes");
                }

                if (value.isInSparseFormat()) {
                    SparseRowsIterator iter = value.getSparseRowsIterator();
                    while (iter.hasNext()) {
                        IJV cell = iter.next();
                        valueMap.put(new MatrixIndexes(((indexes.getRowIndex() - 1) * rowsInBlock + cell.i),
                                (int) ((indexes.getColumnIndex() - 1) * colsInBlock + cell.j)), cell.v);
                    }

                } else {
                    double[] valuesInBlock = value.getDenseArray();
                    for (int i = 0; i < value.getNumRows(); i++) {
                        for (int j = 0; j < value.getNumColumns(); j++) {
                            valueMap.put(
                                    new MatrixIndexes(((indexes.getRowIndex() - 1) * rowsInBlock + i),
                                            (int) ((indexes.getColumnIndex() - 1) * colsInBlock + j)),
                                    valuesInBlock[i * value.getNumColumns() + j]);
                        }
                    }
                }
            }

            reader.close();
        }

        long nonZeros = 0;
        double[][] values = new double[rows][cols];
        for (MatrixIndexes index : valueMap.keySet()) {
            values[(int) index.getRowIndex()][(int) index.getColumnIndex()] = valueMap.get(index);
            if (valueMap.get(index) != 0)
                nonZeros++;
        }

        return new BinaryMatrixCharacteristics(values, rows, cols, rowsInBlock, rowsInLastBlock, colsInBlock,
                colsInLastBlock, nonZeros);
    } catch (IOException e) {
        e.printStackTrace();
        fail("unable to read sequence file in " + directory);
    }

    return null;
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>//from  w  w  w  .  j  av a2 s.  c o  m
 * Returns the path to a file in a directory if it is the only file in the
 * directory.
 * </p>
 * 
 * @param directory
 *            directory containing the file
 * @return path of the file
 */
public static Path getFileInDirectory(String directory) {
    try {
        FileSystem fs = FileSystem.get(conf);
        FileStatus[] files = fs.listStatus(new Path(directory));
        if (files.length != 1)
            throw new IOException("requires exactly one file in directory " + directory);

        return files[0].getPath();
    } catch (IOException e) {
        e.printStackTrace();
        fail("unable to open file in " + directory);
    }

    return null;
}

From source file:com.ibm.crail.hdfs.tools.HdfsIOBenchmark.java

License:Apache License

void getFile() throws Exception, InterruptedException {
    System.out.println("get file, path " + path + ", outstanding " + size + ", loop " + loop);
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);

    Path paths[] = new Path[loop];
    for (int j = 0; j < loop; j++) {
        paths[j] = new Path(path.toString() + "/" + j);
    }//  ww w  .j  av  a  2 s  .  com
    int repfactor = 4;
    for (int k = 0; k < repfactor; k++) {
        long start = System.currentTimeMillis();
        for (int i = 0; i < size; i++) {
            //single operation == loop
            for (int j = 0; j < loop; j++) {
                fs.listStatus(paths[j]);
            }
        }
        long end = System.currentTimeMillis();
        double executionTime = ((double) (end - start));
        double latency = executionTime * 1000.0 / ((double) size);
        System.out.println("execution time [ms] " + executionTime);
        System.out.println("latency [us] " + latency);
    }
    fs.close();
}

From source file:com.ibm.jaql.io.hadoop.DefaultHadoopOutputAdapter.java

License:Apache License

@Override
public JsonValue expand() throws Exception {
    BufferedJsonRecord jr = (BufferedJsonRecord) this.options.getCopy(null);
    BufferedJsonArray ja = new BufferedJsonArray();
    String namenode = null;//  ww  w  . jav a2s. co  m
    String dfsport = null;
    if (this.location != null) {
        Configuration conf = new Configuration();
        namenode = conf.get("fs.default.name");
        dfsport = conf.get("dfs.namenode.http-address");
        FileSystem fs = FileSystem.get(conf);
        FileStatus files = fs.getFileStatus(new Path(this.location));

        if (files.isDir()) {
            StringBuilder sb = new StringBuilder();
            FileStatus[] dirContent = fs.listStatus(new Path(this.location));
            for (FileStatus file : dirContent) {
                if (!file.isDir()) {
                    ja.add(new JsonString(file.getPath().toString()));
                }
            }
        } else {
            ja.add(new JsonString(files.getPath().toString()));
        }
    }
    jr.add(Adapter.LOCATION_NAME, ja);
    jr.add(Adapter.TYPE_NAME, args.get(Adapter.TYPE_NAME));
    jr.add(new JsonString("expanded"), JsonBool.make(true));

    if (namenode != null)
        jr.add(new JsonString("fs.default.name"), new JsonString(namenode));
    if (dfsport != null)
        jr.add(new JsonString("dfs.namenode.http-address"), new JsonString(dfsport));

    return jr;
}

From source file:com.ibm.stocator.fs.swift2d.systemtests.SwiftTestUtils.java

License:Open Source License

/**
 * Deletes all files in a container/*  www.j a  v  a  2  s.  c om*/
 * @param fileSystem
 * @param BaseUri
 * @throws IOException
   */
public static void cleanupAllFiles(FileSystem fileSystem, String BaseUri) throws IOException {
    try {
        if (fileSystem != null) {
            // Clean up generated files
            Path rootDir = new Path(BaseUri);
            FileStatus[] files = fileSystem.listStatus(rootDir);
            for (FileStatus file : files) {
                fileSystem.delete(file.getPath(), false);
            }
        }
    } catch (Exception e) {
        LOG.error("Error in deleting all files.");
    }
}

From source file:com.ibm.stocator.fs.swift2d.systemtests.SwiftTestUtils.java

License:Open Source License

/**
 * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry
 * @param fs filesystem/*from   ww  w.  j  a v a 2  s  . c o  m*/
 * @param dir directory to scan
 * @param subdir full path to look for
 * @throws IOException IO problems
 */
public static void assertListStatusFinds(FileSystem fs, Path dir, Path subdir) throws IOException {
    FileStatus[] stats = fs.listStatus(dir);
    boolean found = false;
    StringBuilder builder = new StringBuilder();
    for (FileStatus stat : stats) {
        builder.append(stat.toString()).append('\n');
        if (stat.getPath().equals(subdir)) {
            found = true;
        }
    }
    assertTrue("Path " + subdir + " not found in directory " + dir + ":" + builder, found);
}