Example usage for org.apache.hadoop.fs FileSystem close

List of usage examples for org.apache.hadoop.fs FileSystem close

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. close

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem close.

Prototype

@Override
public void close() throws IOException

Source Link

Document

Close this FileSystem instance.

Usage

From source file:org.apache.kylin.rest.service.PerformService.java

License:Apache License

private List<String[]> readHdfsFile(String filePath) throws IOException {
    List<String[]> allRows = null;
    CSVReader reader = null;/*www  . jav  a 2s.c  o  m*/
    FileSystem fs = null;
    Configuration conf = new Configuration();

    try {
        fs = FileSystem.newInstance(conf);
        FSDataInputStream inputStream = fs.open(new Path(filePath));
        reader = new CSVReader(new InputStreamReader(inputStream), '|');

        //Read all rows at once
        allRows = reader.readAll();

    } catch (IOException e) {
        logger.info("failed to read hdfs file:", e);
    } finally {
        fs.close();
    }
    return allRows;
}

From source file:org.apache.lens.server.session.DatabaseResourceService.java

License:Apache License

private void loadDbResourceEntries() throws LensException {
    // Read list of databases in
    FileSystem serverFs = null;

    try {/*from ww w.j  a  v a  2  s.  com*/
        String resTopDir = getHiveConf().get(LensConfConstants.DATABASE_RESOURCE_DIR,
                LensConfConstants.DEFAULT_DATABASE_RESOURCE_DIR);
        log.info("Database specific resources at {}", resTopDir);

        Path resTopDirPath = new Path(resTopDir);
        serverFs = FileSystem.newInstance(resTopDirPath.toUri(), getHiveConf());
        if (!serverFs.exists(resTopDirPath)) {
            incrCounter(LOAD_RESOURCES_ERRORS);
            log.warn("Database resource location does not exist - {}. Database jars will not be available",
                    resTopDir);
            return;
        }

        // Look for db dirs
        for (FileStatus dbDir : serverFs.listStatus(resTopDirPath)) {
            Path dbDirPath = dbDir.getPath();
            if (serverFs.isDirectory(dbDirPath)) {
                String dbName = dbDirPath.getName();
                // Get all resources for that db
                findResourcesInDir(serverFs, dbName, dbDirPath);
            } else {
                log.warn("DB resource DIR is not a directory: {}", dbDirPath);
            }
        }

        log.debug("Found resources {}", dbResEntryMap);
    } catch (IOException io) {
        log.error("Error getting list of dbs to load resources from", io);
        throw new LensException(io);
    } finally {
        if (serverFs != null) {
            try {
                serverFs.close();
            } catch (IOException e) {
                log.error("Error closing file system instance", e);
            }
        }
    }
}

From source file:org.apache.mahout.fpm.bigfim.BigFIMDriver.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    if (args.length < 1) {
        System.out.println("Please specify: [configuration file]");
        return -1;
    }/*from   www .  ja v a2s .c  o  m*/
    for (String arg : args) {
        System.out.println(arg);
    }
    Config config = new Config();
    if (args[0].startsWith("s3n")) {
        Path path = new Path(args[0]);
        FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
        fs.open(path);
        config.readConfig(new InputStreamReader(fs.open(path)));
        fs.close();
    } else {
        config.readConfig(args[0]);
    }
    if (!config.isValid()) {
        System.out.println("Config file is invalid!");
        Config.printHelp();
        return -1;
    }

    config.printConfig();

    String inputDir = config.getInputFile();
    String outputDir = config.getOutputDir();
    cleanDirs(new String[] { outputDir });
    long start = System.currentTimeMillis();
    long nrLines = startAprioriPhase(inputDir, outputDir, config);
    startCreatePrefixGroups(inputDir, outputDir, config, nrLines);
    startMining(outputDir, config);
    long end = System.currentTimeMillis();

    System.out.println("[Eclat]: Total time: " + (end - start) / 1000 + "s");

    // getAvgNumberOfPrefixes(tmpDir2);
    // if (!config.getWriteSets()) {
    // getNumberOfItemsets(config.getOutputFile());
    // }
    return 0;
}

From source file:org.apache.mahout.fpm.bigfim.BigFIMDriver.java

License:Apache License

private static void startMining(String outputDir, Config config)
        throws IOException, ClassNotFoundException, InterruptedException {
    String inputFilesDir = outputDir + separator + "pg" + separator;
    String outputFile = outputDir + separator + OFis;
    System.out.println("[StartMining]: input: " + inputFilesDir + ", output: " + outputFile);

    Configuration conf = new Configuration();
    setConfigurationValues(conf, config);

    Job job = new Job(conf, "Start Mining");
    job.setJarByClass(BigFIMDriver.class);

    job.setOutputKeyClass(Text.class);

    if (config.getWriteSets()) {
        job.setOutputValueClass(Text.class);
        job.setMapperClass(EclatMinerMapper.class);
        job.setReducerClass(EclatMinerReducer.class);
    } else {//from w w  w .jav a  2 s .c o  m
        job.setOutputValueClass(LongWritable.class);
        job.setMapperClass(EclatMinerMapperSetCount.class);
        job.setReducerClass(EclatMinerReducerSetCount.class);
    }

    job.setInputFormatClass(NoSplitSequenceFileInputFormat.class);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);

    job.setNumReduceTasks(1);

    List<Path> inputPaths = new ArrayList<Path>();

    FileSystem fs = FileSystem.get(conf);
    FileStatus[] listStatus = fs.globStatus(new Path(inputFilesDir + "bucket*"));
    fs.close();
    for (FileStatus fstat : listStatus) {
        inputPaths.add(fstat.getPath());
    }

    FileInputFormat.setInputPaths(job, inputPaths.toArray(new Path[inputPaths.size()]));
    FileOutputFormat.setOutputPath(job, new Path(outputFile));

    long start = System.currentTimeMillis();
    job.waitForCompletion(true);
    long end = System.currentTimeMillis();
    System.out.println("Job Mining took " + (end - start) / 1000 + "s");
}

From source file:org.apache.mahout.fpm.disteclat.DistEclatDriver.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    if (args.length < 1) {
        System.out.println("Please specify: [configuration file]");
        return -1;
    }/*from www. j  av  a  2  s . c om*/
    for (String arg : args) {
        System.out.println(arg);
    }
    Config config = new Config();
    if (args[0].startsWith("s3n")) {
        Path path = new Path(args[0]);
        FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
        fs.open(path);
        config.readConfig(new InputStreamReader(fs.open(path)));
        fs.close();
    } else {
        config.readConfig(args[0]);
    }
    if (!config.isValid()) {
        System.out.println("Config file is invalid!");
        Config.printHelp();
        return -1;
    }
    config.printConfig();

    String tmpDir1 = config.getOutputDir() + separator + "tmp1" + separator;
    String tmpDir2 = config.getOutputDir() + separator + "prefixes" + separator;

    long start = System.currentTimeMillis();
    cleanDirs(new String[] { config.getOutputDir(), tmpDir1, tmpDir2 });
    startItemReading(config.getInputFile(), tmpDir1, config);
    startPrefixComputation(tmpDir1, tmpDir2, config);
    startMining(tmpDir2, config.getOutputDir(), config);
    long end = System.currentTimeMillis();

    System.out.println("[Eclat]: Total time: " + (end - start) / 1000 + "s");

    return 0;
}

From source file:org.apache.mahout.fpm.disteclat.DistEclatDriver.java

License:Apache License

/**
 * Starts the third MapReduce cycle. Each mapper reads the prefix groups assigned to it and computes the collection of
 * closed sets. All information is reported to the reducer which finally writes the output to disk.
 * //from w ww .j av a2 s.  c om
 * 
 * @param inputDir
 * @param outputDir
 * @param config
 * @throws IOException
 * @throws InterruptedException
 * @throws ClassNotFoundException
 * @throws URISyntaxException
 */
private static void startMining(String inputDir, String outputDir, Config config)
        throws IOException, InterruptedException, ClassNotFoundException, URISyntaxException {

    String inputFilesDir = inputDir;
    String outputFile = outputDir + separator + OFis;
    System.out.println("[StartMining]: input: " + inputFilesDir + ", output: " + outputFile);

    Configuration conf = new Configuration();
    setConfigurationValues(conf, config);

    Job job = new Job(conf, "Start Mining");
    job.setJarByClass(DistEclatDriver.class);

    job.setOutputKeyClass(Text.class);

    if (config.getWriteSets()) {
        job.setOutputValueClass(Text.class);
        job.setMapperClass(EclatMinerMapper.class);
        job.setReducerClass(EclatMinerReducer.class);
    } else {
        job.setOutputValueClass(LongWritable.class);
        job.setMapperClass(EclatMinerMapperSetCount.class);
        job.setReducerClass(EclatMinerReducerSetCount.class);
    }

    job.setInputFormatClass(NoSplitSequenceFileInputFormat.class);

    List<Path> inputPaths = new ArrayList<Path>();

    FileSystem fs = FileSystem.get(conf);
    FileStatus[] listStatus = fs.globStatus(new Path(inputFilesDir + "bucket*"));
    for (FileStatus fstat : listStatus) {
        inputPaths.add(fstat.getPath());
    }

    FileInputFormat.setInputPaths(job, inputPaths.toArray(new Path[inputPaths.size()]));
    FileOutputFormat.setOutputPath(job, new Path(outputFile));

    job.setOutputFormatClass(TextOutputFormat.class);

    job.setNumReduceTasks(1);

    long start = System.currentTimeMillis();
    job.waitForCompletion(true);
    long end = System.currentTimeMillis();
    System.out.println("[Mining]: Took " + (end - start) / 1000 + "s");
    fs.close();
}

From source file:org.apache.mele.embedded.HadoopQueueEmbedded.java

License:Apache License

private void removeAllFiles() throws IOException {
    FileSystem fileSystem = newFileSystem(_file);
    fileSystem.delete(_file, false);//from   w  w w .j av  a2  s  .  c  o  m
    _readerPointer.removeFile(fileSystem);
    for (Path path : _ackFiles) {
        fileSystem.delete(path, false);
    }
    fileSystem.close();
}

From source file:org.apache.mele.embedded.HadoopQueueEmbedded.java

License:Apache License

private boolean ackCheck() throws IOException {
    LOG.info("Starting ack check");
    BitSet bitSet = new BitSet();
    FileSystem fileSystem = null;
    try {/*from ww  w. j av  a2s  . c om*/
        _ackLock.lock();
        _ackOutputStream.close();
        fileSystem = newFileSystem(_file);
        FileStatus fileStatus = fileSystem.getFileStatus(_file);
        long dataLength = fileStatus.getLen();
        long totalAckLength = getTotalAckLength(fileSystem);
        if (!couldContainAllAcks(totalAckLength)) {
            LOG.info("Existing early [" + totalAckLength + "] because [" + totalAckLength % 12 + "]");
            return false;
        }
        for (Path ackFile : _ackFiles) {
            LOG.info("Starting ack check for file [" + ackFile + "]");
            DFSInputStream inputStream = null;
            try {
                inputStream = getDFS(fileSystem.open(ackFile));
                long length = inputStream.getFileLength();
                DataInputStream dataInputStream = new DataInputStream(inputStream);
                while (length > 0) {
                    int pos = (int) dataInputStream.readLong();
                    // @TODO check position
                    // 4 bytes for storing the length of the message
                    int len = dataInputStream.readInt() + 4;
                    bitSet.set(pos, pos + len);
                    length -= 12;
                }
                if (bitSet.cardinality() == dataLength) {
                    return true;
                }
            } finally {
                if (inputStream != null) {
                    inputStream.close();
                }
            }
        }
        return false;
    } finally {
        reopenAckFile(fileSystem);
        _ackLock.unlock();
        if (fileSystem != null) {
            fileSystem.close();
        }
    }
}

From source file:org.apache.nutch.parse.ParseData.java

License:Apache License

public static void main(String argv[]) throws Exception {
    String usage = "ParseData (-local | -dfs <namenode:port>) recno segment";

    if (argv.length < 3) {
        System.out.println("usage:" + usage);
        return;/*from   w  w w  .  ja va 2  s.  c  o m*/
    }

    Options opts = new Options();
    Configuration conf = NutchConfiguration.create();

    GenericOptionsParser parser = new GenericOptionsParser(conf, opts, argv);

    String[] remainingArgs = parser.getRemainingArgs();
    FileSystem fs = FileSystem.get(conf);

    try {
        int recno = Integer.parseInt(remainingArgs[0]);
        String segment = remainingArgs[1];

        Path file = new Path(segment, DIR_NAME);
        System.out.println("Reading from file: " + file);

        ArrayFile.Reader parses = new ArrayFile.Reader(fs, file.toString(), conf);

        ParseData parseDatum = new ParseData();
        parses.get(recno, parseDatum);

        System.out.println("Retrieved " + recno + " from file " + file);
        System.out.println(parseDatum);

        parses.close();
    } finally {
        fs.close();
    }
}

From source file:org.apache.nutch.parse.ParseText.java

License:Apache License

public static void main(String argv[]) throws Exception {
    String usage = "ParseText (-local | -dfs <namenode:port>) recno segment";

    if (argv.length < 3) {
        System.out.println("usage:" + usage);
        return;/*from w ww  .ja  v  a  2s.co  m*/
    }
    Options opts = new Options();
    Configuration conf = NutchConfiguration.create();

    GenericOptionsParser parser = new GenericOptionsParser(conf, opts, argv);

    String[] remainingArgs = parser.getRemainingArgs();

    FileSystem fs = FileSystem.get(conf);
    try {
        int recno = Integer.parseInt(remainingArgs[0]);
        String segment = remainingArgs[1];
        String filename = new Path(segment, ParseText.DIR_NAME).toString();

        ParseText parseText = new ParseText();
        ArrayFile.Reader parseTexts = new ArrayFile.Reader(fs, filename, conf);

        parseTexts.get(recno, parseText);
        System.out.println("Retrieved " + recno + " from file " + filename);
        System.out.println(parseText);
        parseTexts.close();
    } finally {
        fs.close();
    }
}