Example usage for org.apache.hadoop.fs FileSystem close

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem close.

Prototype

@Override
public void close() throws IOException 

Source Link

Document

Close this FileSystem instance.

Usage

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static String searchPreProcessing(final String name, Configuration conf, boolean s3) throws IOException {

    PathFilter filter = new PathFilter() {

        @Override//from   ww  w  .j a va2  s.c  o m
        public boolean accept(Path arg0) {
            if (arg0.getName().contains(name + "-"))
                return true;
            return false;
        }
    };

    Path path = null;
    FileSystem fs = null;

    if (s3) {
        path = new Path(conf.get("bucket") + preProcessingDir);
        fs = FileSystem.get(path.toUri(), conf);
    } else {
        fs = FileSystem.get(new Configuration());
        path = new Path(fs.getHomeDirectory() + "/" + preProcessingDir);
    }

    FileStatus[] status = fs.listStatus(path, filter);

    if (s3)
        fs.close();

    String preProcessingFile = null;
    boolean aggregatesFound = false;
    String fileName = "";
    for (FileStatus fileStatus : status) {
        fileName = fileStatus.getPath().getName();
        if (!fileName.endsWith(".aggregates"))
            preProcessingFile = fileName;
        else if (fileName.endsWith(".aggregates"))
            aggregatesFound = true;
    }

    if (!aggregatesFound)
        return null;

    return preProcessingFile;
}

---

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static String searchAggregatesHeader(final String name, Configuration conf, boolean s3)
        throws IOException {

    PathFilter filter = new PathFilter() {

        @Override/*from   www.ja  va 2 s.c  o  m*/
        public boolean accept(Path arg0) {
            if (arg0.getName().contains(name + "-"))
                return true;
            return false;
        }
    };

    Path path = null;
    FileSystem fs = null;

    if (s3) {
        path = new Path(conf.get("bucket") + preProcessingDir);
        fs = FileSystem.get(path.toUri(), conf);
    } else {
        fs = FileSystem.get(new Configuration());
        path = new Path(fs.getHomeDirectory() + "/" + preProcessingDir);
    }

    FileStatus[] status = fs.listStatus(path, filter);

    if (s3)
        fs.close();

    String fileName = "";
    for (FileStatus fileStatus : status) {
        fileName = fileStatus.getPath().getName();
        if (fileName.endsWith(".aggregates"))
            return fileName;
    }

    return null;
}

---

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static String[] searchAggregates(final String name, Configuration conf, boolean s3) throws IOException {

    PathFilter filter = new PathFilter() {

        @Override//from  w ww .  j a  va  2s.c o  m
        public boolean accept(Path arg0) {
            if (arg0.getName().contains("_SUCCESS"))
                return false;
            return true;
        }
    };

    Path path = null;
    FileSystem fs = null;

    if (s3) {
        path = new Path(conf.get("bucket") + aggregatesDir + "/" + name);
        fs = FileSystem.get(path.toUri(), conf);
    } else {
        fs = FileSystem.get(new Configuration());
        path = new Path(fs.getHomeDirectory() + "/" + aggregatesDir + "/" + name);
    }

    FileStatus[] status;

    try {
        status = fs.listStatus(path, filter);
    } catch (FileNotFoundException e) {
        return new String[0];
    }

    if (s3)
        fs.close();

    String[] names = new String[status.length];
    String fileName = "";
    for (int i = 0; i < status.length; i++) {
        fileName = status[i].getPath().getName();
        names[i] = fileName;
    }

    return names;
}

---

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static String[] searchIndex(final String name, Configuration conf, boolean s3) throws IOException {

    PathFilter filter = new PathFilter() {

        @Override/* w  w w  .  j  av  a2  s  .  com*/
        public boolean accept(Path arg0) {
            if (arg0.getName().contains("_SUCCESS"))
                return false;
            return true;
        }
    };

    Path path = null;
    FileSystem fs = null;

    if (s3) {
        path = new Path(conf.get("bucket") + indexDir + "/" + name);
        fs = FileSystem.get(path.toUri(), conf);
    } else {
        fs = FileSystem.get(new Configuration());
        path = new Path(fs.getHomeDirectory() + "/" + indexDir + "/" + name);
    }

    FileStatus[] status;

    try {
        status = fs.listStatus(path, filter);
    } catch (FileNotFoundException e) {
        return new String[0];
    }

    if (s3)
        fs.close();

    String[] names = new String[status.length];
    String fileName = "";
    for (int i = 0; i < status.length; i++) {
        fileName = status[i].getPath().getName();
        names[i] = fileName;
    }

    return names;
}

---

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static String[] searchDataAttributes(final String name, Configuration conf, boolean s3)
        throws IOException {

    PathFilter filter = new PathFilter() {

        @Override//from   w  w w  .  j a v a 2  s.  com
        public boolean accept(Path arg0) {
            if (arg0.getName().contains("_SUCCESS"))
                return false;
            return true;
        }
    };

    Path path = null;
    FileSystem fs = null;

    if (s3) {
        path = new Path(conf.get("bucket") + dataAttributesDir + "/" + name);
        fs = FileSystem.get(path.toUri(), conf);
    } else {
        fs = FileSystem.get(new Configuration());
        path = new Path(fs.getHomeDirectory() + "/" + dataAttributesDir + "/" + name);
    }

    FileStatus[] status;

    try {
        status = fs.listStatus(path, filter);
    } catch (FileNotFoundException e) {
        return new String[0];
    }

    if (s3)
        fs.close();

    String[] names = new String[status.length];
    String fileName = "";
    for (int i = 0; i < status.length; i++) {
        fileName = status[i].getPath().getName();
        names[i] = fileName;
    }

    return names;
}

---

From source file:edu.nyu.vida.data_polygamy.utils.SpatialGraph.java

License:BSD License

public void init(boolean nbhd, Configuration conf) throws IOException {

    String bucket = conf.get("bucket", "");
    Path edgesPath = null;/*from  w w w. j a  va  2  s .c  o m*/
    FileSystem fs = null;

    // reading edges
    if (nbhd)
        edgesPath = new Path(bucket + "neighborhood-graph");
    else
        edgesPath = new Path(bucket + "zipcode-graph");

    if (!bucket.equals(""))
        fs = FileSystem.get(edgesPath.toUri(), conf);
    else
        fs = FileSystem.get(new Configuration());

    BufferedReader buff = new BufferedReader(new InputStreamReader(fs.open(edgesPath)));
    String line = buff.readLine();

    String[] s = Utilities.splitString(line.trim());
    nbNodes = Integer.parseInt(s[0].trim());

    for (int i = 0; i < nbNodes; i++) {
        // assuming that neighborhoods and zip codes are read in order
        //  and the same ids are present in the edges file
        adjacencyList.add(new ArrayList<Integer>());
    }

    line = buff.readLine();
    String[] values;
    ArrayList<Integer> elem;
    Integer id_1, id_2;
    while (line != null) {
        values = line.replace("\n", "").split(" ");
        id_1 = Integer.parseInt(values[0]);
        id_2 = Integer.parseInt(values[1]);

        elem = adjacencyList.get(id_1);
        elem.add(id_2);
        adjacencyList.set(id_1, elem);

        elem = adjacencyList.get(id_2);
        elem.add(id_1);
        adjacencyList.set(id_2, elem);

        line = buff.readLine();
    }
    buff.close();
    fs.close();

    // main BFS - choose random node
    mainOriginNode = random.nextInt(nbNodes);
    bfs(mainOriginNode);

}

---

From source file:edu.uci.ics.asterix.external.adapter.factory.HDFSAdapterFactory.java

License:Apache License

/**
 * Instead of creating the split using the input format, we do it manually
 * This function returns fileSplits (1 per hdfs file block) irrespective of the number of partitions
 * and the produced splits only cover intersection between current files in hdfs and files stored internally
 * in AsterixDB//from   w ww  .java2 s.c o m
 * 1. NoOp means appended file
 * 2. AddOp means new file
 * 3. UpdateOp means the delta of a file
 *
 * @return
 * @throws IOException
 */
protected InputSplit[] getSplits(JobConf conf) throws IOException {
    // Create file system object
    FileSystem fs = FileSystem.get(conf);
    ArrayList<FileSplit> fileSplits = new ArrayList<FileSplit>();
    ArrayList<ExternalFile> orderedExternalFiles = new ArrayList<ExternalFile>();
    // Create files splits
    for (ExternalFile file : files) {
        Path filePath = new Path(file.getFileName());
        FileStatus fileStatus;
        try {
            fileStatus = fs.getFileStatus(filePath);
        } catch (FileNotFoundException e) {
            // file was deleted at some point, skip to next file
            continue;
        }
        if (file.getPendingOp() == ExternalFilePendingOp.PENDING_ADD_OP
                && fileStatus.getModificationTime() == file.getLastModefiedTime().getTime()) {
            // Get its information from HDFS name node
            BlockLocation[] fileBlocks = fs.getFileBlockLocations(fileStatus, 0, file.getSize());
            // Create a split per block
            for (BlockLocation block : fileBlocks) {
                if (block.getOffset() < file.getSize()) {
                    fileSplits.add(new FileSplit(filePath, block.getOffset(),
                            (block.getLength() + block.getOffset()) < file.getSize() ? block.getLength()
                                    : (file.getSize() - block.getOffset()),
                            block.getHosts()));
                    orderedExternalFiles.add(file);
                }
            }
        } else if (file.getPendingOp() == ExternalFilePendingOp.PENDING_NO_OP
                && fileStatus.getModificationTime() == file.getLastModefiedTime().getTime()) {
            long oldSize = 0L;
            long newSize = file.getSize();
            for (int i = 0; i < files.size(); i++) {
                if (files.get(i).getFileName() == file.getFileName()
                        && files.get(i).getSize() != file.getSize()) {
                    newSize = files.get(i).getSize();
                    oldSize = file.getSize();
                    break;
                }
            }

            // Get its information from HDFS name node
            BlockLocation[] fileBlocks = fs.getFileBlockLocations(fileStatus, 0, newSize);
            // Create a split per block
            for (BlockLocation block : fileBlocks) {
                if (block.getOffset() + block.getLength() > oldSize) {
                    if (block.getOffset() < newSize) {
                        // Block interact with delta -> Create a split
                        long startCut = (block.getOffset() > oldSize) ? 0L : oldSize - block.getOffset();
                        long endCut = (block.getOffset() + block.getLength() < newSize) ? 0L
                                : block.getOffset() + block.getLength() - newSize;
                        long splitLength = block.getLength() - startCut - endCut;
                        fileSplits.add(new FileSplit(filePath, block.getOffset() + startCut, splitLength,
                                block.getHosts()));
                        orderedExternalFiles.add(file);
                    }
                }
            }
        }
    }
    fs.close();
    files = orderedExternalFiles;
    return fileSplits.toArray(new FileSplit[fileSplits.size()]);
}

---

From source file:edu.uci.ics.asterix.file.ExternalIndexingOperations.java

License:Apache License

public static ArrayList<ExternalFile> getSnapshotFromExternalFileSystem(Dataset dataset)
        throws AlgebricksException {
    ArrayList<ExternalFile> files = new ArrayList<ExternalFile>();
    ExternalDatasetDetails datasetDetails = (ExternalDatasetDetails) dataset.getDatasetDetails();
    try {//from ww  w  .j  a  v  a2 s  .  co  m
        // Create the file system object
        FileSystem fs = getFileSystemObject(datasetDetails.getProperties());
        // If dataset uses hive adapter, add path to the dataset properties
        if (datasetUsesHiveAdapter(datasetDetails)) {
            HiveAdapterFactory.populateConfiguration(datasetDetails.getProperties());
        }
        // Get paths of dataset
        String path = datasetDetails.getProperties().get(HDFSAdapterFactory.KEY_PATH);
        String[] paths = path.split(",");

        // Add fileStatuses to files
        for (String aPath : paths) {
            FileStatus[] fileStatuses = fs.listStatus(new Path(aPath));
            for (int i = 0; i < fileStatuses.length; i++) {
                int nextFileNumber = files.size();
                if (fileStatuses[i].isDirectory()) {
                    listSubFiles(dataset, fs, fileStatuses[i], files);
                } else {
                    files.add(new ExternalFile(dataset.getDataverseName(), dataset.getDatasetName(),
                            nextFileNumber, fileStatuses[i].getPath().toUri().getPath(),
                            new Date(fileStatuses[i].getModificationTime()), fileStatuses[i].getLen(),
                            ExternalFilePendingOp.PENDING_NO_OP));
                }
            }
        }
        // Close file system
        fs.close();
        if (files.size() == 0) {
            throw new AlgebricksException("File Snapshot retrieved from external file system is empty");
        }
        return files;
    } catch (Exception e) {
        e.printStackTrace();
        throw new AlgebricksException("Unable to get list of HDFS files " + e);
    }
}

---

From source file:edu.umkc.sce.App.java

License:Apache License

public int run(String[] args) throws Exception {
    Configuration conf = getConf();
    GenericOptionsParser parser = new GenericOptionsParser(conf, args);
    args = parser.getRemainingArgs();/* w  w w .j a v  a2s. c  o m*/
    if (args.length != 1) {
        GenericOptionsParser.printGenericCommandUsage(System.out);

        truncate(getAdmin().listTableNamesByNamespace(MY_NAMESPACE));
        System.exit(2);
    }

    String importFile = args[0];
    FileSystem fs = null;
    BufferedReader br = null;
    Model m = null;

    try {
        fs = FileSystem.get(conf);
        Path path = new Path(importFile);
        br = new BufferedReader(new InputStreamReader(fs.open(path)));
        m = createModel();

        Stopwatch sw = new Stopwatch();
        sw.start();
        m.read(br, null, RDFLanguages.strLangNTriples);
        sw.stop();
        System.out.printf("Loading '%s' took %d.\n", importFile, sw.elapsedTime(TimeUnit.MILLISECONDS));
        sw.reset();
        sw.start();
        runTestQuery(m);
        sw.stop();
        System.out.printf("Query '%s' took %d.\n", query, sw.elapsedTime(TimeUnit.MILLISECONDS));
        sw.reset();
        sw.start();

        createStore(m);
        sw.stop();
        System.out.printf("loadHbase took %d.\n", sw.elapsedTime(TimeUnit.MILLISECONDS));
    } finally {
        if (m != null)
            m.close();
        if (br != null)
            br.close();
        if (fs != null)
            fs.close();
    }

    return 0;
}

---

From source file:edu.umkc.sce.VpImport.java

License:Apache License

private void load(Configuration conf, Model model, String importFile) throws IOException {

    FileSystem fs = null;
    BufferedReader br = null;/*ww w  . j  a v  a 2s . co m*/
    try {
        fs = FileSystem.get(conf);
        Path path = new Path(importFile);
        br = new BufferedReader(new InputStreamReader(fs.open(path)));

        model.begin();
        model.read(br, null, RDFLanguages.strLangNTriples);
        model.commit();
    } finally {
        if (br != null)
            br.close();
        if (fs != null)
            fs.close();
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• »