Example usage for org.apache.hadoop.fs FileSystem close

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem close.

Prototype

@Override
public void close() throws IOException 

Source Link

Document

Close this FileSystem instance.

Usage

From source file:com.cloudera.flume.handlers.hdfs.TestDFSWrite.java

License:Apache License

@Test
public void testHDFSSequenceFileWrite() throws IOException {
    FlumeConfiguration conf = FlumeConfiguration.get();

    Path path = new Path("file:///tmp/testfile");
    FileSystem hdfs = path.getFileSystem(conf);
    hdfs.deleteOnExit(path);//from  w  w w .  j a  v a2  s  .  c  o  m

    Event e = new EventImpl("EVENT".getBytes());

    Writer w = SequenceFile.createWriter(hdfs, conf, path, WriteableEventKey.class, WriteableEvent.class);

    // writing
    w.append(new WriteableEventKey(e), new WriteableEvent(e));
    w.close();

    FileStatus stats = hdfs.getFileStatus(path);
    assertTrue(stats.getLen() > 0);

    // reading
    SequenceFile.Reader r = new SequenceFile.Reader(hdfs, path, conf);
    WriteableEventKey k = new WriteableEventKey();
    WriteableEvent evt = new WriteableEvent();
    r.next(k, evt);

    assertEquals(evt.getTimestamp(), e.getTimestamp());
    assertEquals(evt.getNanos(), e.getNanos());
    assertEquals(evt.getPriority(), e.getPriority());
    assertTrue(Arrays.equals(evt.getBody(), e.getBody()));

    hdfs.close();
}

---

From source file:com.cloudera.flume.handlers.hdfs.TestDFSWrite.java

License:Apache License

@Test
public void testHDFSEventSink() throws IOException, InterruptedException {
    FlumeConfiguration conf = FlumeConfiguration.get();
    String str = "file:///tmp/testfile";
    Path path = new Path(str);
    DFSEventSink sink = new DFSEventSink(str);

    FileSystem hdfs = path.getFileSystem(conf);

    sink.open();/*ww w  .j  a  v a 2s .  c  o  m*/
    Event e = new EventImpl("EVENT".getBytes());
    sink.append(e);
    sink.close();

    FileStatus stats = hdfs.getFileStatus(path);
    assertTrue(stats.getLen() > 0);

    // // reading
    SequenceFile.Reader r = new SequenceFile.Reader(hdfs, path, conf);
    WriteableEventKey k = new WriteableEventKey();
    WriteableEvent evt = new WriteableEvent();
    r.next(k, evt);

    assertEquals(evt.getTimestamp(), e.getTimestamp());
    assertEquals(evt.getNanos(), e.getNanos());
    assertEquals(evt.getPriority(), e.getPriority());
    assertTrue(Arrays.equals(evt.getBody(), e.getBody()));

    hdfs.close();
}

---

From source file:com.cloudera.flume.handlers.hdfs.TestDFSWrite.java

License:Apache License

/**
 * Test that pathnames are being correctly substituted.
 * //from  www  .ja v  a  2s  .  c o m
 * @throws InterruptedException
 */
@Test
public void testTaggedWrites() throws IOException, InterruptedException {
    FlumeConfiguration conf = FlumeConfiguration.get();
    String template = "file:///tmp/testfile-%{mytag}";
    String real1 = "file:///tmp/testfile-one";
    String real2 = "file:///tmp/testfile-two";

    DFSEventSink sink = new DFSEventSink(template);

    Event e1 = new EventImpl("EVENT1".getBytes());
    e1.set("mytag", "one".getBytes());
    Event e2 = new EventImpl("EVENT2".getBytes());
    e2.set("mytag", "two".getBytes());
    sink.open();
    sink.append(e1);
    sink.append(e2);
    sink.close();

    Path path1 = new Path(real1);
    Path path2 = new Path(real2);

    FileSystem hdfs = path1.getFileSystem(conf);
    FileStatus stats = hdfs.getFileStatus(path1);
    assertTrue("File " + real1 + " not found but expected", stats.getLen() > 0);
    stats = hdfs.getFileStatus(new Path(real2));
    assertTrue("File " + real2 + " not found but expected", stats.getLen() > 0);

    SequenceFile.Reader r = new SequenceFile.Reader(hdfs, path1, conf);
    WriteableEventKey k = new WriteableEventKey();
    WriteableEvent evt = new WriteableEvent();
    r.next(k, evt);

    assertEquals(evt.getTimestamp(), e1.getTimestamp());
    assertEquals(evt.getNanos(), e1.getNanos());
    assertEquals(evt.getPriority(), e1.getPriority());
    assertTrue(Arrays.equals(evt.getBody(), e1.getBody()));

    r = new SequenceFile.Reader(hdfs, path2, conf);
    k = new WriteableEventKey();
    evt = new WriteableEvent();
    r.next(k, evt);

    assertEquals(evt.getTimestamp(), e2.getTimestamp());
    assertEquals(evt.getNanos(), e2.getNanos());
    assertEquals(evt.getPriority(), e2.getPriority());
    assertTrue(Arrays.equals(evt.getBody(), e2.getBody()));

    hdfs.close();
}

---

From source file:com.cloudera.flume.handlers.hive.MarkerStore.java

License:Apache License

private boolean runHiveMarkerQueries() {
    boolean queryStatus = true;
    FileSystem hdfs;
    FSDataInputStream in;/*w  w w  .  j  a  va 2 s.c o  m*/
    dstPath = new Path(hiveMarkerFolder);
    LOG.info("DSTPATH: " + dstPath);
    try {
        hdfs = dstPath.getFileSystem(conf);
        if (hdfs.exists(dstPath)) {
            FileStatus[] fileListing = hdfs.listStatus(dstPath);
            for (FileStatus fs : fileListing) {
                if (!fs.isDir()) {
                    LOG.info("File marker path: " + fs.getPath());
                    in = hdfs.open(fs.getPath());
                    byte[] fileData = new byte[(int) fs.getLen()];
                    in.readFully(fileData);
                    String[] splitTab = new String(fileData).split("\t");
                    if (splitTab.length == 2) {
                        dstPath = new Path(splitTab[0]);
                        FileSystem hiveFile = dstPath.getFileSystem(conf);
                        if (hiveFile.exists(dstPath)) {
                            LOG.info("marker file data: " + splitTab[1]);
                            if (runHiveQuery(splitTab[1])) {
                                LOG.info("Marker query is successful");
                                in.close();
                                cleanMarkerFile(fs.getPath().toString());
                            } else {
                                LOG.info("Error running marker query, marker point not deleted");
                                queryStatus = false;
                            }

                        } else {
                            LOG.info("marker points to invalid hive file location, deleting the marker");
                            in.close();
                            cleanMarkerFile(fs.getPath().toString());
                        }
                    }
                    //in.close();
                }
            }
        }
        hdfs.close();
    } catch (IOException e) {
        LOG.error("ERROR running runMarkerQueries:" + e.getMessage());
    }

    return queryStatus;
}

---

From source file:com.cloudera.flume.handlers.hive.MarkerStore.java

License:Apache License

public boolean mergeFiles(String folder, Path file, String hiveOutputLocation) {
    FileSystem hdfs;
    FSDataInputStream in;/*from ww w  .  j  a  va2s . c  om*/
    FSDataOutputStream out;
    List<Path> fileCollection = new ArrayList<Path>();
    dstPath = new Path(folder);
    LOG.info("mergeFiles DSTPATH: " + dstPath);
    try {
        hdfs = dstPath.getFileSystem(conf);

        if (hdfs.exists(dstPath)) {
            FileStatus[] fileListing = hdfs.listStatus(dstPath);
            LOG.error("Creating file @: " + hiveOutputLocation);
            out = hdfs.create(new Path(hiveOutputLocation));

            in = hdfs.open(file);
            byte[] fileData = new byte[(int) hdfs.getFileStatus(file).getLen()];
            in.readFully(fileData);
            out.write(fileData);

            for (FileStatus fs : fileListing) {
                if (!fs.isDir()) {
                    LOG.info("mergeFiles File marker path: " + fs.getPath());
                    fileCollection.add(fs.getPath());
                    in = hdfs.open(fs.getPath());
                    fileData = new byte[(int) fs.getLen()];
                    in.readFully(fileData);
                    out.write(fileData);
                }
            }
            out.close();
        }

        hdfs.close();
        LOG.error("Written file: " + hiveOutputLocation);

        //lets start the purge process, delete all files except the merged file
        hdfs = dstPath.getFileSystem(conf);
        for (Path p : fileCollection) {
            if (hdfs.delete(p, false)) {
                LOG.error("Successfully deleted: " + p);
            } else {
                LOG.error("Error deleting file: " + p);
            }
        }

    } catch (IOException e) {
        LOG.error("ERROR running runMarkerQueries:" + e.getMessage());
    }
    LOG.error("mergeFiles Done merging files");
    return false;
}

---

From source file:com.cloudera.flume.PerfHdfsIO.java

License:Apache License

@Test
public void testCopy() throws IOException, InterruptedException {

    Benchmark b = new Benchmark("hdfs seqfile copy");
    b.mark("begin");

    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();//from   w ww .  j a  v a  2 s.c o m
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
    sink.open();
    b.mark("localdisk_write_started");

    EventUtil.dumpAll(mem, sink);

    b.mark("local_disk_write done");

    sink.close();

    FlumeConfiguration conf = FlumeConfiguration.get();
    Path src = new Path(tmp.getAbsolutePath());
    Path dst = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = dst.getFileSystem(conf);
    hdfs.deleteOnExit(dst);

    b.mark("hdfs_copy_started");
    hdfs.copyFromLocalFile(src, dst);
    b.mark("hdfs_copy_done");
    hdfs.close();
    b.done();
}

---

From source file:com.cloudera.flume.PerfHdfsIO.java

License:Apache License

@Test
public void testDirectWrite() throws IOException, InterruptedException {

    Benchmark b = new Benchmark("hdfs seqfile write");
    b.mark("begin");

    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    FlumeConfiguration conf = FlumeConfiguration.get();
    Path path = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = path.getFileSystem(conf);
    hdfs.deleteOnExit(path);/*w ww.  java  2  s .com*/

    Writer w = SequenceFile.createWriter(hdfs, conf, path, WriteableEventKey.class, WriteableEvent.class);
    b.mark("hdfs_fileopen_started");

    Event e = null;
    while ((e = mem.next()) != null) {
        // writing
        w.append(new WriteableEventKey(e), new WriteableEvent(e));
    }
    w.close();
    b.mark("seqfile_hdfs_write");

    hdfs.close();
    b.done();
}

---

From source file:com.cloudera.HioBench.java

License:Apache License

public static void main(String[] args) throws Exception {
    options = new Options();
    final Configuration conf = new Configuration();
    if (options.dumpConf) {
        Configuration.dumpConfiguration(conf, new PrintWriter(System.out));
    }/*from   w  ww .  j a  v a2 s .  com*/
    final FileSystem fs = FileSystem.get(new URI(options.hdfsUri), conf);
    fs.setVerifyChecksum(!options.skipChecksum);

    if (!fs.exists(options.filePath)) {
        System.out.println("no file at " + options.filePath + "; writing " + "new file now with length "
                + options.nGigsInFile + " gigs...");
        writeFile(fs);
        System.out.println("done.");
    } else if (fs.getLength(options.filePath) != options.nBytesInFile) {
        System.out.println("existing file " + options.filename + " has length " + fs.getLength(options.filePath)
                + ", but we wanted length " + options.nBytesInFile + ".  Re-creating.");
        writeFile(fs);
        System.out.println("done.");
    } else {
        System.out.println(
                "using existing file at " + options.filePath + " of length " + options.nGigsInFile + " gigs.");
    }

    long nanoStart = System.nanoTime();
    WorkerThread threads[] = new WorkerThread[options.nThreads];
    for (int i = 0; i < options.nThreads; i++) {
        threads[i] = new WorkerThread(i == 0, fs, WorkerThread.createBenchReader(options, i));
    }
    for (int i = 0; i < options.nThreads; i++) {
        threads[i].start();
    }
    for (int i = 0; i < options.nThreads; i++) {
        threads[i].join();
    }
    for (int i = 0; i < options.nThreads; i++) {
        Throwable t = threads[i].getException();
        if (t != null) {
            System.err.println("there were exceptions.  Aborting.");
            System.exit(1);
        }
    }
    long nanoEnd = System.nanoTime();
    fs.close();
    long totalIo = options.nThreads;
    totalIo *= options.nBytesToRead;
    float nanoDiff = nanoEnd - nanoStart;
    float seconds = nanoDiff / 1000000000;
    System.out.println(String.format("Using %d threads, read %s in %f seconds", options.nThreads,
            prettyPrintByteSize(totalIo), seconds));
    float rate = totalIo / seconds;
    System.out.println("Average rate was " + prettyPrintByteSize(rate) + "/s");
}

---

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void testGet() throws Exception {
    Configuration conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    FileSystem fs = FileSystem.get(getJettyURL().toURI(), conf);
    Assert.assertNotNull(fs);//from   w  w  w .java  2s.co  m
    Assert.assertEquals(fs.getUri(), getJettyURL().toURI());
    fs.close();
}

---

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void testOpen() throws Exception {
    FileSystem fs = FileSystem.get(getHadoopConf());
    Path path = new Path(getHadoopTestDir(), "foo.txt");
    OutputStream os = fs.create(path);
    os.write(1);/* w w w  .  ja v a  2s. c om*/
    os.close();
    fs.close();
    Configuration conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    fs = FileSystem.get(getJettyURL().toURI(), conf);
    InputStream is = fs.open(new Path(path.toUri().getPath()));
    Assert.assertEquals(is.read(), 1);
    is.close();
    fs.close();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• »