Example usage for org.apache.hadoop.fs FileSystem close

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem close.

Prototype

@Override
public void close() throws IOException

Source Link

Document

Close this FileSystem instance.

Usage

From source file:root.hap.HierarchicalAffinityPropagationJob.java

License:Apache License

/**
 * {@inheritDoc}/*from www  .  j a  v  a  2  s .c om*/
 */
@Override
public int run(String[] args) throws Exception {

    constructParameterList();

    if (parseArguments(args) == null) {
        return -1;
    }

    initializeConfigurationParameters();

    printJobHeader();

    Configuration conf = getConf();

    URI workingURI = new URI(conf.get("fs.default.name"));
    URI inputURI = new URI(inputDirectory);

    FileSystem workingFS = FileSystem.get(workingURI, conf);
    FileSystem inputFS = FileSystem.get(inputURI, conf);

    if (workingDirectory == null) {
        workingDirectory = workingFS.getWorkingDirectory().toString();
    }

    // Check to see if input directory exists
    Path input = new Path(inputDirectory);
    if (!inputFS.exists(input)) {
        System.err.println("Input Directory does not exist.");
        System.exit(2);
    }

    Path simMatPath = new Path(inputDirectory + simMatFileName);
    if (!inputFS.exists(simMatPath)) {
        System.err.println("Similarity Matrix File does not exist.");
        System.exit(2);
    }

    int iterations = Integer.valueOf(numIterations);
    for (int i = 0; i < iterations; i++) {

        RDargs[1] = workingDirectory + RD_File + i;
        RDargs[3] = workingDirectory + AD_File + i;
        RDargs[11] = i + "";

        // If this is the first run, read from initial input.
        if (i == 0) {
            RDargs[1] = inputDirectory;
        }

        System.out.println();
        System.out.println("----------------------");
        System.out.println("Updating Responsibilty");
        System.out.println("----------------------");
        System.out.println("\tInput: " + RDargs[1]);
        System.out.println("\tOutput: " + RDargs[3]);
        System.out.println("\tIteration: " + (i + 1) + " of " + numIterations);
        System.out.println();
        ToolRunner.run(conf, HRD, RDargs);

        // Delete input directory to remove wasted space.
        // Preserve the initial input, though.
        if (i > 0) {
            workingFS.delete(new Path(RDargs[1]), true);
        }

        ADargs[1] = workingDirectory + AD_File + i;
        ADargs[3] = workingDirectory + RD_File + (i + 1);
        ADargs[11] = i + "";

        if (i == iterations - 1) {
            ADargs[3] = workingDirectory + CD_File;
        }

        System.out.println();
        System.out.println("---------------------");
        System.out.println("Updating Availability");
        System.out.println("---------------------");
        System.out.println("\tInput: " + ADargs[1]);
        System.out.println("\tOutput: " + ADargs[3]);
        System.out.println("\tIteration: " + (i + 1) + " of " + numIterations);
        System.out.println();
        ToolRunner.run(conf, HAD, ADargs);

        workingFS.delete(new Path(ADargs[1]), true);

    }

    CDargs[1] = workingDirectory + CD_File;

    System.out.println();
    System.out.println("---------------------");
    System.out.println("Extracting Clusters");
    System.out.println("---------------------");
    System.out.println("\tInput: " + CDargs[1]);
    System.out.println("\tOutput: " + CDargs[3]);
    System.out.println();
    ToolRunner.run(conf, HCD, CDargs);

    workingFS.delete(new Path(CDargs[1]), true);

    workingFS.close();
    inputFS.close();
    return 0;
}

From source file:root.input.InputJob.java

License:Apache License

/**
 * Writes a timestamp to file./*from w  ww. j  a  v  a  2 s.co m*/
 * 
 * @param conf the configuration object to work with
 * @param filename the location to write to
 * @param time the timestamp to write
 * @return
 */
protected boolean writeTimestamp(Configuration conf, String filename, long time) {

    boolean success = true;

    try {
        URI localURI = new URI(conf.get("fs.default.name"));
        FileSystem localFS = FileSystem.get(localURI, conf);

        Path timestamp = new Path(filename);

        if (localFS.exists(timestamp)) {
            localFS.delete(timestamp, true);
        }

        FSDataOutputStream out = localFS.create(timestamp);

        out.writeBytes("HAP duration: " + time + "\n");

        Iterator<Map.Entry<String, String>> iter = conf.iterator();
        while (iter.hasNext()) {
            Map.Entry<String, String> entry = iter.next();
            String key = entry.getKey();
            String val = entry.getValue();

            if (key.startsWith(CONF_PREFIX)) {
                key = key.substring(CONF_PREFIX.length());
                out.writeBytes(key);
                out.writeBytes("\t");
                out.writeBytes(val);
                out.writeBytes("\n");
            }
        }

        out.flush();
        out.close();

        localFS.close();
    } catch (IOException e) {
        System.err.println(e);
        success = false;
    } catch (URISyntaxException e) {
        System.err.println(e);
        success = false;
    }

    return success;
}

From source file:sharedsidefunctions.CopyFromLocal.java

License:Apache License

public static void copyFromLocal(Configuration conf, String diamond, String query, String dataBase,
        String userName) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    fs.moveFromLocalFile(new Path(dataBase + ".dmnd"), new Path("/user/" + userName + "/Hamond"));
    fs.copyFromLocalFile(new Path(query), new Path("/user/" + userName + "/Hamond"));
    //        fs.copyFromLocalFile(new Path(System.getProperty("user.dir")+"/diamond"), new Path(userName));
    fs.copyFromLocalFile(new Path(diamond), new Path("/user/" + userName + "/Hamond"));

    //close file system
    fs.close();
}

From source file:sharedsidefunctions.MakeHamondHDFSdir.java

License:Apache License

public static void makedir(Configuration conf, String userName) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    Path p1 = new Path("/user/" + userName + "/Hamond");
    if (fs.exists(p1)) {
        fs.delete(p1, true);/* www. j ava 2  s .c  o  m*/
    }
    fs.mkdirs(p1);
    //make dir for all outputs
    Path p2 = new Path("/user/" + userName + "/Hamond/output");
    fs.mkdirs(p2);

    fs.close();
}

From source file:tv.icntv.grade.film.core.AbstractJob.java

License:Apache License

protected Path[] getPaths(String[] files) throws IOException {
    List<Path> paths = Lists.newArrayList();
    FileSystem fileSystem = null;
    try {/*from  www.  j  a v  a2s . c  om*/
        fileSystem = FileSystem.get(super.getConf());
        for (String file : files) {
            Path p = new Path(file);
            if (fileSystem.exists(p)) {
                paths.add(p);
            }
        }
    } catch (Exception e) {

    } finally {
        if (null != fileSystem) {
            fileSystem.close();
        }
    }
    return paths.toArray(new Path[paths.size()]);
}

From source file:tv.icntv.grade.film.utils.HadoopUtils.java

License:Apache License

public static boolean isExist(Path path) throws IOException {
    FileSystem fileSystem = null;
    try {/*w  w w .  j  a  va2  s.com*/
        fileSystem = FileSystem.get(configuration);
        return fileSystem.exists(path);
    } catch (Exception e) {
        return false;
    } finally {
        if (null != fileSystem) {
            fileSystem.close();
        }
    }
}

From source file:tv.icntv.grade.film.utils.HadoopUtils.java

License:Apache License

public static void deleteIfExist(String path) throws IOException {
    FileSystem fileSystem = null;
    try {/*from   w ww.  j  av a  2  s.  co  m*/
        fileSystem = FileSystem.get(configuration);
        Path p = new Path(path);
        if (fileSystem.exists(p)) {
            fileSystem.delete(p, true);
        }
    } catch (IOException e) {
        e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
    } finally {
        if (null != fileSystem) {
            fileSystem.close();
        }
    }

}

From source file:tv.icntv.grade.film.utils.HadoopUtils.java

License:Apache License

public static void save(Path path, Long size) throws IOException {
    FileSystem fileSystem = null;
    FSDataOutputStream out = null;/*from  w w w  . j  a va  2 s . c om*/
    try {

        fileSystem = FileSystem.get(configuration);
        if (fileSystem.exists(path)) {
            fileSystem.delete(path);
            out = fileSystem.create(path);
            out.writeLong(size);
            out.flush();
        }

    } catch (Exception e) {
        return;
    } finally {
        Closeables.close(out, true);
        if (null != fileSystem) {
            fileSystem.close();
        }
    }
}

From source file:tv.icntv.grade.film.utils.HadoopUtils.java

License:Apache License

public static long count(Path path, PathFilter filter) throws Exception {
    FileSystem fileSystem = null;

    try {//from w w  w  .j ava 2 s.  c  om
        fileSystem = FileSystem.get(configuration);
        FileStatus[] fs = fileSystem.listStatus(path, filter);
        long count = 0;
        for (FileStatus f : fs) {
            SequenceFile.Reader frequentPatternsReader = new SequenceFile.Reader(fileSystem, f.getPath(),
                    configuration);
            Text key = new Text();
            while (frequentPatternsReader.next(key)) {
                count++;
            }
            frequentPatternsReader.close();
        }
        return count;
    } catch (Exception e) {
        throw e;
    } finally {
        //            Closeables.close(br,true);
        if (null != fileSystem) {
            fileSystem.close();
        }
    }
}

From source file:tv.icntv.log.crawl.store.HdfsDefaultStore.java

License:Apache License

@Override
public boolean createDirectory(String directoryName) {
    if (isNull(directoryName)) {
        logger.info("directory path={} null", directoryName);
        return false;
    }//w  w w. j a v  a2  s  . c o  m
    FileSystem fileSystem = null;
    try {
        Path path = new Path(directoryName);
        fileSystem = FileSystem.get(configuration);
        if (!fileSystem.exists(path)) {
            logger.info("create directory {}", directoryName);
            return fileSystem.mkdirs(path);
        }
        logger.info("directory path={} exist", directoryName);
        return true;
    } catch (IOException e) {
        e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
        return false;
    } finally {
        if (null != fileSystem) {
            try {
                fileSystem.close();
            } catch (IOException e) {
                e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
            }
        }
    }
}