Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import org.apache.hadoop.hdfs.*; import junit.framework.TestCase; import java.io.*; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.permission.*; import java.net.*; /** * This class tests that the Fuse module for DFS can mount properly * and does a few simple commands: * mkdir * rmdir * ls * cat * * cp and touch are purposely not tested because they won't work with the current module * */ public class TestFuseDFS extends TestCase { /** * mount the fuse file system using assumed fuse module library installed in /usr/local/lib or somewhere else on your * pre-existing LD_LIBRARY_PATH * */ static Process fuse_process; static String fuse_cmd; static private void mount(String mountpoint, URI dfs) throws IOException, InterruptedException { String cp = System.getProperty("java.class.path"); Runtime r = Runtime.getRuntime(); fuse_cmd = System.getProperty("build.test") + "/../fuse_dfs"; String libhdfs = System.getProperty("build.test") + "/../../../libhdfs/"; String arch = System.getProperty("os.arch"); String jvm = System.getProperty("java.home") + "/lib/" + arch + "/server"; String lp = System.getProperty("LD_LIBRARY_PATH") + ":" + "/usr/local/lib:" + libhdfs + ":" + jvm; System.err.println("LD_LIBRARY_PATH=" + lp); String cmd[] = { fuse_cmd, "dfs://" + dfs.getHost() + ":" + String.valueOf(dfs.getPort()), mountpoint, "-obig_writes", "-odebug", "-oentry_timeout=0.1", "-oattribute_timeout=0.1", "-ousetrash", "rw", "-oinitchecks", "-ordbuffer=32768" }; final String[] envp = { "CLASSPATH=" + cp, "LD_LIBRARY_PATH=" + lp, "PATH=" + "/usr/bin:/bin" }; // ensure the mount point is not currently mounted Process p = r.exec("fusermount -u " + mountpoint); p.waitFor(); // clean up the mount point p = r.exec("rm -rf " + mountpoint); assertTrue(p.waitFor() == 0); // make the mount point if needed p = r.exec("mkdir -p " + mountpoint); assertTrue(p.waitFor() == 0); // mount fuse to the mount point fuse_process = r.exec(cmd, envp); // give DFS a chance to come up try { Thread.sleep(3000); } catch (Exception e) { } } /** * unmounts fuse for before shutting down. */ static private void umount(String mpoint) throws IOException, InterruptedException { Runtime r = Runtime.getRuntime(); Process p = r.exec("fusermount -u " + mpoint); p.waitFor(); } /** * Set things up - create mini dfs cluster and mount the fuse filesystem. */ public TestFuseDFS() throws IOException, InterruptedException { } static private MiniDFSCluster cluster; static private DistributedFileSystem fileSys; final static private String mpoint; static { mpoint = System.getProperty("build.test") + "/mnt"; System.runFinalizersOnExit(true); startStuff(); } static public void startStuff() { try { Configuration conf = new Configuration(); conf.setBoolean("dfs.permissions", false); cluster = new MiniDFSCluster(conf, 1, true, null); fileSys = (DistributedFileSystem) cluster.getFileSystem(); assertTrue(fileSys.getFileStatus(new Path("/")).isDir()); mount(mpoint, fileSys.getUri()); } catch (Exception e) { e.printStackTrace(); } } public void setUp() { } /** * use shell to create a dir and then use filesys to see it exists. */ public void testMkdir() throws IOException, InterruptedException, Exception { try { // First create a new directory with mkdirs Path path = new Path("/foo"); Runtime r = Runtime.getRuntime(); String cmd = "mkdir -p " + mpoint + path.toString(); Process p = r.exec(cmd); assertTrue(p.waitFor() == 0); // check it is there assertTrue(fileSys.getFileStatus(path).isDir()); // check again through the shell String lsCmd = "ls " + mpoint + path.toString(); p = r.exec(lsCmd); assertTrue(p.waitFor() == 0); } catch (Exception e) { e.printStackTrace(); throw e; } } /** * use shell to create a dir and then use filesys to see it exists. */ public void testWrites() throws IOException, InterruptedException { try { // write a hello file File file = new File(mpoint, "hello.txt"); FileOutputStream f = new FileOutputStream(file); String s = "hello "; f.write(s.getBytes()); s = "world"; f.write(s.getBytes()); f.flush(); f.close(); try { Thread.sleep(1000); } catch (Exception e) { } // check the file exists. Path myPath = new Path("/hello.txt"); assertTrue(fileSys.exists(myPath)); // check the data is ok FileInputStream fi = new FileInputStream(new File(mpoint, "hello.txt")); byte b[] = new byte[12]; int length = fi.read(b, 0, 12); assertTrue(length > 0); String s2 = new String(b, 0, length); assertEquals("hello world", s2); } catch (Exception e) { e.printStackTrace(); } finally { } } /** * Test ls for dir already created in testMkdDir also tests bad ls */ public void testLs() throws IOException, InterruptedException { try { // First create a new directory with mkdirs Runtime r = Runtime.getRuntime(); // mkdir Process p = r.exec("mkdir -p " + mpoint + "/test/mkdirs"); assertTrue(p.waitFor() == 0); // ls p = r.exec("ls " + mpoint + "/test/mkdirs"); assertTrue(p.waitFor() == 0); // ls non-existant directory p = r.exec("ls " + mpoint + "/test/mkdirsNotThere"); int res = p.waitFor(); assertFalse(res == 0); } catch (Exception e) { e.printStackTrace(); } } /** * Remove a dir using the shell and use filesys to see it no longer exists. */ public void testRmdir() throws IOException, InterruptedException { try { // First create a new directory with mkdirs Runtime r = Runtime.getRuntime(); Process p = r.exec("mkdir -p " + mpoint + "/test/rmdir"); assertTrue(p.waitFor() == 0); Path myPath = new Path("/test/rmdir"); assertTrue(fileSys.exists(myPath)); // remove it p = r.exec("rmdir " + mpoint + "/test/rmdir"); assertTrue(p.waitFor() == 0); // check it is not there assertFalse(fileSys.exists(myPath)); Path trashPath = new Path("/user/root/.Trash/Current/test/rmdir"); assertTrue(fileSys.exists(trashPath)); // make it again to test trashing same thing twice p = r.exec("mkdir -p " + mpoint + "/test/rmdir"); assertTrue(p.waitFor() == 0); assertTrue(fileSys.exists(myPath)); // remove it p = r.exec("rmdir " + mpoint + "/test/rmdir"); assertTrue(p.waitFor() == 0); // check it is not there assertFalse(fileSys.exists(myPath)); trashPath = new Path("/user/root/.Trash/Current/test/rmdir.1"); assertTrue(fileSys.exists(trashPath)); } catch (Exception e) { e.printStackTrace(); } } /** * use shell to create a dir and then use filesys to see it exists. */ public void testDF() throws IOException, InterruptedException, Exception { try { // First create a new directory with mkdirs Path path = new Path("/foo"); Runtime r = Runtime.getRuntime(); String cmd = "mkdir -p " + mpoint + path.toString(); Process p = r.exec(cmd); assertTrue(p.waitFor() == 0); File f = new File(mpoint + "/foo"); DistributedFileSystem.DiskStatus d = fileSys.getDiskStatus(); long fileUsedBlocks = (f.getTotalSpace() - f.getFreeSpace()) / (64 * 1024 * 1024); long dfsUsedBlocks = (long) Math.ceil((double) d.getDfsUsed() / (64 * 1024 * 1024)); assertTrue(fileUsedBlocks == dfsUsedBlocks); assertTrue(d.getCapacity() == f.getTotalSpace()); } catch (Exception e) { e.printStackTrace(); throw e; } } /** * use shell to create a dir and then use filesys to see it exists. */ public void testChown() throws IOException, InterruptedException, Exception { try { // First create a new directory with mkdirs Path path = new Path("/foo"); Runtime r = Runtime.getRuntime(); String cmd = "mkdir -p " + mpoint + path.toString(); Process p = r.exec(cmd); assertTrue(p.waitFor() == 0); // check it is there assertTrue(fileSys.getFileStatus(path).isDir()); FileStatus foo = fileSys.getFileStatus(path); System.err.println("DEBUG:owner=" + foo.getOwner()); cmd = "chown nobody " + mpoint + path.toString(); p = r.exec(cmd); assertTrue(p.waitFor() == 0); // cmd = "chgrp nobody " + mpoint + path.toString(); // p = r.exec(cmd); // assertTrue(p.waitFor() == 0); foo = fileSys.getFileStatus(path); System.err.println("DEBUG:owner=" + foo.getOwner()); assertTrue(foo.getOwner().equals("nobody")); assertTrue(foo.getGroup().equals("nobody")); } catch (Exception e) { e.printStackTrace(); throw e; } } /** * use shell to create a dir and then use filesys to see it exists. */ public void testChmod() throws IOException, InterruptedException, Exception { try { // First create a new directory with mkdirs Path path = new Path("/foo"); Runtime r = Runtime.getRuntime(); String cmd = "mkdir -p " + mpoint + path.toString(); Process p = r.exec(cmd); assertTrue(p.waitFor() == 0); // check it is there assertTrue(fileSys.getFileStatus(path).isDir()); cmd = "chmod 777 " + mpoint + path.toString(); p = r.exec(cmd); assertTrue(p.waitFor() == 0); FileStatus foo = fileSys.getFileStatus(path); FsPermission perm = foo.getPermission(); assertTrue(perm.toShort() == 0777); } catch (Exception e) { e.printStackTrace(); throw e; } } /** * use shell to create a dir and then use filesys to see it exists. */ public void testUtimes() throws IOException, InterruptedException, Exception { try { // First create a new directory with mkdirs Path path = new Path("/utimetest"); Runtime r = Runtime.getRuntime(); String cmd = "touch " + mpoint + path.toString(); Process p = r.exec(cmd); assertTrue(p.waitFor() == 0); // check it is there assertTrue(fileSys.exists(path)); FileStatus foo = fileSys.getFileStatus(path); long oldTime = foo.getModificationTime(); try { Thread.sleep(1000); } catch (Exception e) { } cmd = "touch " + mpoint + path.toString(); p = r.exec(cmd); assertTrue(p.waitFor() == 0); try { Thread.sleep(1000); } catch (Exception e) { } foo = fileSys.getFileStatus(path); long newTime = foo.getModificationTime(); assertTrue(newTime > oldTime); } catch (Exception e) { e.printStackTrace(); throw e; } finally { } } /** * * Test dfs_read on a file size that will trigger multiple internal reads. * First, just check raw size reading is ok and then check with smaller reads * including checking the validity of the data read. * */ public void testReads() throws IOException, InterruptedException { try { // First create a new directory with mkdirs Runtime r = Runtime.getRuntime(); Process p; // create the file Path myPath = new Path("/test/hello.reads"); FSDataOutputStream s = fileSys.create(myPath); String hello = "hello world!"; int written = 0; int mycount = 0; while (written < 1024 * 9) { s.writeUTF(hello); s.writeInt(mycount++); written += hello.length() + 4; } s.close(); // check it exists assertTrue(fileSys.exists(myPath)); FileStatus foo = fileSys.getFileStatus(myPath); assertTrue(foo.getLen() >= 9 * 1024); { // cat the file DataInputStream is = new DataInputStream(new FileInputStream(mpoint + "/test/hello.reads")); byte buf[] = new byte[4096]; // test reading 0 length assertTrue(is.read(buf, 0, 0) == 0); // test real reads assertTrue(is.read(buf, 0, 1024) == 1024); assertTrue(is.read(buf, 0, 4096) == 4096); assertTrue(is.read(buf, 0, 4096) == 4096); is.close(); } { DataInputStream is = new DataInputStream(new FileInputStream(mpoint + "/test/hello.reads")); int read = 0; int counter = 0; try { while (true) { String s2 = DataInputStream.readUTF(is); int s3 = is.readInt(); assertTrue(s2.equals(hello)); assertTrue(s3 == counter++); read += hello.length() + 4; } } catch (EOFException e) { assertTrue(read >= 9 * 1024); } } // check reading an empty file for EOF { // create the file myPath = new Path("/test/hello.reads2"); s = fileSys.create(myPath); s.close(); FSDataInputStream fs = fileSys.open(myPath); assertEquals(-1, fs.read()); FileInputStream f = new FileInputStream(mpoint + "/test/hello.reads2"); assertEquals(-1, f.read()); } } catch (Exception e) { e.printStackTrace(); } finally { } } /** * Use filesys to create the hello world! file and then cat it and see its contents are correct. */ public void testCat() throws IOException, InterruptedException { try { // First create a new directory with mkdirs Runtime r = Runtime.getRuntime(); Process p = r.exec("rm -rf " + mpoint + "/test/hello"); assertTrue(p.waitFor() == 0); // create the file Path myPath = new Path("/test/hello"); FSDataOutputStream s = fileSys.create(myPath); String hello = "hello world!"; s.writeUTF(hello); s.writeInt(1033); s.close(); // check it exists assertTrue(fileSys.exists(myPath)); // cat the file DataInputStream is = new DataInputStream(new FileInputStream(mpoint + "/test/hello")); String s2 = DataInputStream.readUTF(is); int s3 = is.readInt(); assertTrue(s2.equals(hello)); assertTrue(s3 == 1033); } catch (Exception e) { e.printStackTrace(); } finally { } } /** * Use filesys to create the hello world! file and then cat it and see its contents are correct. */ public void testAppends() throws IOException, InterruptedException { try { // First create a new directory with mkdirs Runtime r = Runtime.getRuntime(); { FileOutputStream os = new FileOutputStream(mpoint + "/appends"); String hello = "hello"; os.write(hello.getBytes()); os.flush(); os.close(); } // check it exists Path myPath = new Path("/appends"); assertTrue(fileSys.exists(myPath)); try { Thread.sleep(1000); } catch (Exception e) { } FileStatus foo = fileSys.getFileStatus(myPath); File f = new File(mpoint + "/appends"); assertTrue(f.length() > 0); { FileOutputStream os = new FileOutputStream(mpoint + "/appends", true); String hello = " world!"; os.write(hello.getBytes()); os.flush(); os.close(); } // cat the file FileInputStream is = new FileInputStream(mpoint + "/appends"); byte b[] = new byte[1024]; int len = is.read(b); assertTrue(len > 0); String s2 = new String(b, 0, len); assertTrue(s2.equals("hello world!")); } catch (Exception e) { e.printStackTrace(); } finally { } } public void testDone() throws IOException { close(); } /** * Unmount and close */ protected void tearDown() throws Exception { } /** * Unmount and close */ protected void finalize() throws Throwable { close(); } public void close() { try { int length; // print out the fuse debug output { do { InputStream i = fuse_process.getInputStream(); byte b[] = new byte[i.available()]; length = i.read(b); System.err.println("read x bytes: " + length); System.err.write(b, 0, b.length); } while (length > 0); } do { InputStream i = fuse_process.getErrorStream(); byte b[] = new byte[i.available()]; length = i.read(b); System.err.println("read x bytes: " + length); System.err.write(b, 0, b.length); } while (length > 0); umount(mpoint); fuse_process.destroy(); fuse_process = null; if (fileSys != null) { fileSys.close(); fileSys = null; } if (cluster != null) { cluster.shutdown(); cluster = null; } } catch (Exception e) { } } };