Java tutorial
/* Copyright 2011 m6d.com Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package com.hdfs.concat.crush; import static java.lang.String.format; import static java.lang.System.currentTimeMillis; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; import java.io.DataInputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.cli.UnrecognizedOptionException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.SequenceFile.Reader; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ToolRunner; import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import com.hdfs.concat.crush.Crush; import com.hdfs.concat.crush.MapperCounter; @SuppressWarnings("deprecation") public class CrushTest { @Rule public final TemporaryFolder tmp = new TemporaryFolder(); private JobConf job; private FileSystem fileSystem; private String javaIoTmpDir; @Before public void setupJob() throws IOException { job = new JobConf(false); job.set("fs.default.name", "file:///"); job.set("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem"); job.setInt("mapred.reduce.tasks", 5); job.setLong("dfs.block.size", 50); FileSystem delegate = FileSystem.get(job); fileSystem = new SortingFileSystem(delegate); /* * Set the working directory so that all relative paths are rooted in the tmp dir. This will keep the file system clean of * temporary test files. */ FileSystem.get(job).setWorkingDirectory(new Path(tmp.getRoot().getAbsolutePath())); } @Before public void setJavaIoTmpDir() { javaIoTmpDir = System.setProperty("java.io.tmpdir", tmp.getRoot().getAbsolutePath()); } @After public void restoreJavaIoTmpDir() { System.setProperty("java.io.tmpdir", javaIoTmpDir); } private void run(String... args) throws Exception { ToolRunner.run(job, new Crush(), args); } @Test public void backwardsCompatibleInvocationBadSrcDir() throws Exception { try { run("does-not-exist", tmp.getRoot().getAbsolutePath(), "80"); fail(); } catch (IOException e) { if (!e.getMessage().contains("does-not-exist")) { throw e; } } } @Test public void backwardsCompatibleInvocationBadNumberOfTasks() throws Exception { try { run(tmp.newFolder("in").getAbsolutePath(), tmp.newFolder("out").getAbsolutePath(), "not a number"); fail(); } catch (NumberFormatException e) { if (!e.getMessage().contains("not a number")) { throw e; } } } @Test public void backwardsCompatibleInvocationNegativeTasks() throws Exception { try { run(tmp.newFolder("in").getAbsolutePath(), tmp.newFolder("out").getAbsolutePath(), "-1"); fail(); } catch (UnrecognizedOptionException e) { if (!e.getMessage().contains("-1")) { throw e; } } } @Test public void backwardsCompatibleInvocationZeroTasks() throws Exception { try { run(tmp.newFolder("in").getAbsolutePath(), tmp.newFolder("out").getAbsolutePath(), "0"); fail(); } catch (IllegalArgumentException e) { if (!e.getMessage().contains("0")) { throw e; } } } @Test public void backwardsCompatibleInvocationHugeTasks() throws Exception { try { run(tmp.newFolder("in").getAbsolutePath(), tmp.newFolder("out").getAbsolutePath(), "4001"); fail(); } catch (IllegalArgumentException e) { if (!e.getMessage().contains("4001")) { throw e; } } } @Test public void backwardsCompatibleInvocationBadSrcDirWithType() throws Exception { try { run("does-not-exist", tmp.getRoot().getAbsolutePath(), "80", "TEXT"); fail(); } catch (IOException e) { if (!e.getMessage().contains("does-not-exist")) { throw e; } } } @Test public void backwardsCompatibleInvocationBadNumberOfTasksWithType() throws Exception { try { run(tmp.newFolder("in").getAbsolutePath(), tmp.newFolder("out").getAbsolutePath(), "not a number", "TEXT"); fail(); } catch (NumberFormatException e) { if (!e.getMessage().contains("not a number")) { throw e; } } } @Test public void backwardsCompatibleInvocationNegativeTasksWithType() throws Exception { try { run(tmp.newFolder("in").getAbsolutePath(), tmp.newFolder("out").getAbsolutePath(), "-1", "TEXT"); fail(); } catch (UnrecognizedOptionException e) { if (!e.getMessage().contains("-1")) { throw e; } } } @Test public void backwardsCompatibleInvocationZeroTasksWithType() throws Exception { try { run(tmp.newFolder("in").getAbsolutePath(), tmp.newFolder("out").getAbsolutePath(), "0", "TEXT"); fail(); } catch (IllegalArgumentException e) { if (!e.getMessage().contains("0")) { throw e; } } } @Test public void backwardsCompatibleInvocationHugeHugeTasksWithType() throws Exception { try { run(tmp.newFolder("in").getAbsolutePath(), tmp.newFolder("out").getAbsolutePath(), "4001", "TEXT"); fail(); } catch (IllegalArgumentException e) { if (!e.getMessage().contains("4001")) { throw e; } } } @Test public void backwardsCompatibleInvocationBadType() throws Exception { try { run(tmp.newFolder("in").getAbsolutePath(), tmp.newFolder("out").getAbsolutePath(), "80", "NEITHER_TEXT_OR_SEQUENCE"); fail(); } catch (IllegalArgumentException e) { if (!e.getMessage().contains("NEITHER_TEXT_OR_SEQUENCE")) { throw e; } } } @Test public void invocationBadSrcDir() throws Exception { try { run("--threshold=0.9", "does-not-exist", tmp.getRoot().getAbsolutePath(), "20101116123015"); fail(); } catch (IOException e) { if (!e.getMessage().contains("does-not-exist")) { throw e; } } } @Test public void invocationBadTimestamp() throws Exception { try { run("--threshold=0.9", tmp.newFolder("in").getAbsolutePath(), new File(tmp.getRoot(), "out").getAbsolutePath(), "not a number"); fail(); } catch (IllegalArgumentException e) { if (!e.getMessage().contains("not a number")) { throw e; } } } @Test public void invocationShortTimestamp() throws Exception { try { run(tmp.newFolder("in").getAbsolutePath(), new File(tmp.getRoot(), "out").getAbsolutePath(), "2010111612301"); fail(); } catch (IllegalArgumentException e) { if (!e.getMessage().contains("2010111612301")) { throw e; } } } @Test public void invocationLongTimestamp() throws Exception { try { run("--threshold=0.5", tmp.newFolder("in").getAbsolutePath(), new File(tmp.getRoot(), "out").getAbsolutePath(), "201011161230150"); fail(); } catch (IllegalArgumentException e) { if (!e.getMessage().contains("201011161230150")) { throw e; } } } @Test public void dirWithNoMatchingRegex() throws Exception { /* * Create a non-empty directory. */ File src = tmp.newFolder("src"); tmp.newFolder("src/foo"); tmp.newFile("src/foo/file"); try { run("--regex", ".+/in", "--replacement", "foo", "--input-format", "org.apache.hadoop.mapred.TextInputFormat", "--output-format", "org.apache.hadoop.mapred.TextOutputFormat", "--threshold", "0.5", "--max-file-blocks", "100", src.getAbsolutePath(), "out", "20101116123015"); fail(); } catch (IllegalArgumentException e) { if (!e.getMessage().contains("src/foo")) { throw e; } } } @Test public void bucketing() throws Exception { File in = tmp.newFolder("in"); Counters expectedCounters = new Counters(); List<String> expectedBucketFiles = new ArrayList<String>(); /* * Create a hierarchy of directories. Directories are distinguished by a trailing slash in these comments. * * 1/ * 1.1/ * file1 10 bytes * file2 20 bytes * file3 30 bytes * file4 41 bytes * file5 15 bytes * file6 30 bytes * file7 20 bytes * 1.2/ * file1 20 bytes * file2 10 bytes * 1.3/ * 2/ * file1 70 bytes * file2 30 bytes * file3 25 bytes * file4 30 bytes * file5 35 bytes * 2.1/ * file1 10 bytes * 2.2/ * file1 25 bytes * file2 15 bytes * file3 35 bytes * 2.3/ * file1 41 bytes * file2 10 bytes * 2.4/ * 2.4.1/ * file1 100 bytes * file2 30 bytes * 2.4.2/ * file1 20 bytes * file2 20 bytes * file3 10 bytes */ /* * in contains 2 dirs and no files so it is skipped. * * in/ * 1/ * 2/ */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_SKIPPED, 1); tmp.newFolder("in/1"); File dir2 = tmp.newFolder("in/2"); /* * in/1 contains three dirs and no files so it is skipped. * * in/ * 1/ * 1.1/ * 1.2/ * 1.3/ */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_SKIPPED, 1); File dir1_1 = tmp.newFolder("in/1/1.1"); File dir1_2 = tmp.newFolder("in/1/1.2"); tmp.newFolder("in/1/1.3"); /* * in/2 contains five files and four dirs. * * in/ * 2/ * file1 70 bytes * file2 30 bytes * file3 25 bytes * file4 30 bytes * file5 35 bytes * 2.1/ * 2.2/ * 2.3/ * 2.4/ * * 0 1 2 * file5 35 file2 30 file4 30 * file3 25 * * Buckets 0 and 2 have a single file each so they are ignored. */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_ELIGIBLE, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 5); expectedCounters.incrCounter(MapperCounter.FILES_ELIGIBLE, 2); expectedCounters.incrCounter(MapperCounter.FILES_SKIPPED, 3); File dir2_1 = tmp.newFolder("in/2/2.1"); File dir2_2 = tmp.newFolder("in/2/2.2"); File dir2_3 = tmp.newFolder("in/2/2.3"); tmp.newFolder("in/2/2.4"); createFile(dir2, "file1", 70); createFile(dir2, "file2", 30); createFile(dir2, "file3", 25); createFile(dir2, "file4", 30); createFile(dir2, "file5", 35); expectedBucketFiles .add(format("%s %s", dir2.getAbsolutePath() + "-1", new File(dir2, "file2").getAbsolutePath())); expectedBucketFiles .add(format("%s %s", dir2.getAbsolutePath() + "-1", new File(dir2, "file3").getAbsolutePath())); /* * in/1/1.1 contains seven files and no dirs. * * in/ * 1/ * 1.1/ * file1 10 bytes * file2 20 bytes * file3 30 bytes * file4 41 bytes * file5 15 bytes * file6 30 bytes * file7 20 bytes * * 0 1 2 * file3 30 file6 30 file2 20 * file5 15 file1 10 file7 20 * * file4 is > 50 * 0.8 so it is ignored. */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_ELIGIBLE, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 7); expectedCounters.incrCounter(MapperCounter.FILES_ELIGIBLE, 6); expectedCounters.incrCounter(MapperCounter.FILES_SKIPPED, 1); createFile(dir1_1, "file1", 10); createFile(dir1_1, "file2", 20); createFile(dir1_1, "file3", 30); createFile(dir1_1, "file4", 41); createFile(dir1_1, "file5", 15); createFile(dir1_1, "file6", 30); createFile(dir1_1, "file7", 20); expectedBucketFiles.add( format("%s %s", dir1_1.getAbsolutePath() + "-0", new File(dir1_1, "file3").getAbsolutePath())); expectedBucketFiles.add( format("%s %s", dir1_1.getAbsolutePath() + "-0", new File(dir1_1, "file5").getAbsolutePath())); expectedBucketFiles.add( format("%s %s", dir1_1.getAbsolutePath() + "-1", new File(dir1_1, "file6").getAbsolutePath())); expectedBucketFiles.add( format("%s %s", dir1_1.getAbsolutePath() + "-1", new File(dir1_1, "file1").getAbsolutePath())); expectedBucketFiles.add( format("%s %s", dir1_1.getAbsolutePath() + "-2", new File(dir1_1, "file2").getAbsolutePath())); expectedBucketFiles.add( format("%s %s", dir1_1.getAbsolutePath() + "-2", new File(dir1_1, "file7").getAbsolutePath())); /* * in/1/1.2 contains to files. * * in/ * 1/ * 1.2/ * file1 20 bytes * file2 10 bytes */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_ELIGIBLE, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 2); expectedCounters.incrCounter(MapperCounter.FILES_ELIGIBLE, 2); createFile(dir1_2, "file1", 20); createFile(dir1_2, "file2", 10); expectedBucketFiles.add( format("%s %s", dir1_2.getAbsolutePath() + "-0", new File(dir1_2, "file1").getAbsolutePath())); expectedBucketFiles.add( format("%s %s", dir1_2.getAbsolutePath() + "-0", new File(dir1_2, "file2").getAbsolutePath())); /* * in/1/1.3 is empty. * * in/ * 1/ * 1.3/ */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_SKIPPED, 1); tmp.newFolder("in/1/1.3"); /* * in/2/2.1 contains on file. * * in/ * 2/ * 2.1/ * file1 10 bytes * * Single file dirs are ignored. */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_SKIPPED, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 1); expectedCounters.incrCounter(MapperCounter.FILES_SKIPPED, 1); createFile(dir2_1, "file1", 10); /* * in/2/2.2 contains three files. * * in/ * 2/ * 2.2/ * file1 25 bytes * file2 15 bytes * file3 35 bytes * * 0 1 * file3 35 file1 25 * file2 15 * * Bucket 0 with a single file is ignored. */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_ELIGIBLE, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 3); expectedCounters.incrCounter(MapperCounter.FILES_ELIGIBLE, 2); expectedCounters.incrCounter(MapperCounter.FILES_SKIPPED, 1); createFile(dir2_2, "file1", 25); createFile(dir2_2, "file2", 15); createFile(dir2_2, "file3", 35); expectedBucketFiles.add( format("%s %s", dir2_2.getAbsolutePath() + "-1", new File(dir2_2, "file1").getAbsolutePath())); expectedBucketFiles.add( format("%s %s", dir2_2.getAbsolutePath() + "-1", new File(dir2_2, "file2").getAbsolutePath())); /* * in/2/2.3 contains 2 files. * * in/ * 2/ * 2.3/ * file1 41 bytes * file2 10 bytes * * file1 is too big and leaving file2 as a single file, which is also ignored. */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_SKIPPED, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 2); expectedCounters.incrCounter(MapperCounter.FILES_SKIPPED, 2); createFile(dir2_3, "file1", 41); createFile(dir2_3, "file2", 10); /* * in/2/2.4 contains two sub directories and no files. * * in/ * 2/ * 2.4/ * 2.4.1/ * 2.4.2/ */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_SKIPPED, 1); tmp.newFolder("in/2/2.4"); File dir2_4_1 = tmp.newFolder("in/2/2.4/2.4.1"); File dir2_4_2 = tmp.newFolder("in/2/2.4/2.4.2"); /* * in/ * 2/ * 2.4/ * 2.4.1/ * file1 100 bytes * file2 30 bytes */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_SKIPPED, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 2); expectedCounters.incrCounter(MapperCounter.FILES_SKIPPED, 2); createFile(dir2_4_1, "file1", 100); createFile(dir2_4_1, "file2", 30); /* * in/ * 2/ * 2.4/ * 2.4.2/ * file1 20 bytes * file2 20 bytes * file3 10 bytes * 0 * file1 20 * file2 20 * file3 10 */ expectedCounters.incrCounter(MapperCounter.DIRS_FOUND, 1); expectedCounters.incrCounter(MapperCounter.DIRS_ELIGIBLE, 1); expectedCounters.incrCounter(MapperCounter.FILES_FOUND, 3); expectedCounters.incrCounter(MapperCounter.FILES_ELIGIBLE, 3); createFile(dir2_4_2, "file1", 20); createFile(dir2_4_2, "file2", 20); createFile(dir2_4_2, "file3", 10); expectedBucketFiles.add(format("%s %s", dir2_4_2.getAbsolutePath() + "-0", new File(dir2_4_2, "file1").getAbsolutePath())); expectedBucketFiles.add(format("%s %s", dir2_4_2.getAbsolutePath() + "-0", new File(dir2_4_2, "file2").getAbsolutePath())); expectedBucketFiles.add(format("%s %s", dir2_4_2.getAbsolutePath() + "-0", new File(dir2_4_2, "file3").getAbsolutePath())); Crush crush = new Crush(); crush.setConf(job); crush.setFileSystem(fileSystem); /* * Call these in the same order that run() does. */ crush.createJobConfAndParseArgs("--compress=none", "--max-file-blocks=1", in.getAbsolutePath(), new File(tmp.getRoot(), "out").getAbsolutePath(), "20101124171730"); crush.writeDirs(); /* * Verify bucket contents. */ List<String> actualBucketFiles = new ArrayList<String>(); Text key = new Text(); Text value = new Text(); Reader reader = new Reader(FileSystem.get(job), crush.getBucketFiles(), job); while (reader.next(key, value)) { actualBucketFiles.add(format("%s\t%s", key, value)); } reader.close(); Collections.sort(expectedBucketFiles); Collections.sort(actualBucketFiles); assertThat(actualBucketFiles, equalTo(expectedBucketFiles)); /* * Verify the partition map. */ Reader partitionMapReader = new Reader(FileSystem.get(job), crush.getPartitionMap(), job); IntWritable partNum = new IntWritable(); Map<String, Integer> actualPartitions = new HashMap<String, Integer>(); while (partitionMapReader.next(key, partNum)) { actualPartitions.put(key.toString(), partNum.get()); } partitionMapReader.close(); /* * These crush files need to allocated into 5 partitions: * * in/2-1 55 bytes * in/1/1.1-0 45 bytes * in/1/1.1-2 40 bytes * in/1/1.1-1 40 bytes * in/1/1.2-0 30 bytes * in/2/2.2-1 40 bytes * in/2/2.4/2.4.2-0 50 bytes * * 0 1 2 3 4 * in/2-1 55 in/2/2.4/2.4.2-0 50 in/1/1.1-0 45 in/1/1.1-2 40 in/1/1.1-1 40 * in/2/2.2-1 40 in/1/1.2-0 39 */ Map<String, Integer> expectedPartitions = new HashMap<String, Integer>(); //TODO: this may not be deterministic due to jvm/hashmap/filesystem expectedPartitions.put(dir2.getAbsolutePath() + "-1", 0); expectedPartitions.put(dir2_4_2.getAbsolutePath() + "-0", 1); expectedPartitions.put(dir1_1.getAbsolutePath() + "-0", 2); expectedPartitions.put(dir1_1.getAbsolutePath() + "-2", 4); expectedPartitions.put(dir2_2.getAbsolutePath() + "-1", 3); expectedPartitions.put(dir1_1.getAbsolutePath() + "-1", 3); expectedPartitions.put(dir1_2.getAbsolutePath() + "-0", 4); assertThat(actualPartitions, equalTo(expectedPartitions)); /* * Verify counters. */ Counters actualCounters = new Counters(); DataInputStream countersStream = FileSystem.get(job).open(crush.getCounters()); actualCounters.readFields(countersStream); countersStream.close(); assertThat(actualCounters, equalTo(expectedCounters)); } /** * Returns a qualified file status, just like {@link FileSystem#listStatus(Path)} does. */ private static FileStatus createFile(File dir, String fileName, int size) { File file = new File(dir, fileName); try { FileOutputStream os = new FileOutputStream(file); os.write(new byte[size]); os.close(); } catch (IOException e) { throw new RuntimeException(e); } return new FileStatus(size, false, 3, 1024 * 1024 * 60, currentTimeMillis(), new Path("hdfs://hostname.pvt:12345" + file.getAbsolutePath())); } /** * This exists only so we can impose a specific order on the files that are listed. */ private static class SortingFileSystem extends FileSystem { private final FileSystem delegate; public SortingFileSystem(FileSystem delegate) { super(); this.delegate = delegate; } @Override public FileStatus[] listStatus(Path arg0) throws IOException { FileStatus[] contents = delegate.listStatus(arg0); Arrays.sort(contents); return contents; } @Override public FSDataOutputStream append(Path arg0, int arg1, Progressable arg2) throws IOException { return delegate.append(arg0, arg1, arg2); } @Override public FSDataOutputStream append(Path f, int bufferSize) throws IOException { return delegate.append(f, bufferSize); } @Override public FSDataOutputStream append(Path f) throws IOException { return delegate.append(f); } @Override public void close() throws IOException { delegate.close(); } @Override public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException { delegate.completeLocalOutput(fsOutputFile, tmpLocalFile); } @Override public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, Path dst) throws IOException { delegate.copyFromLocalFile(delSrc, overwrite, src, dst); } @Override public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs, Path dst) throws IOException { delegate.copyFromLocalFile(delSrc, overwrite, srcs, dst); } @Override public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException { delegate.copyFromLocalFile(delSrc, src, dst); } @Override public void copyFromLocalFile(Path src, Path dst) throws IOException { delegate.copyFromLocalFile(src, dst); } @Override public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { delegate.copyToLocalFile(delSrc, src, dst); } @Override public void copyToLocalFile(Path src, Path dst) throws IOException { delegate.copyToLocalFile(src, dst); } @Override public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, Progressable progress) throws IOException { return delegate.create(f, overwrite, bufferSize, progress); } @Override public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return delegate.create(f, overwrite, bufferSize, replication, blockSize, progress); } @Override public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize) throws IOException { return delegate.create(f, overwrite, bufferSize, replication, blockSize); } @Override public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize) throws IOException { return delegate.create(f, overwrite, bufferSize); } @Override public FSDataOutputStream create(Path f, boolean overwrite) throws IOException { return delegate.create(f, overwrite); } @Override public FSDataOutputStream create(Path arg0, FsPermission arg1, boolean arg2, int arg3, short arg4, long arg5, Progressable arg6) throws IOException { return delegate.create(arg0, arg1, arg2, arg3, arg4, arg5, arg6); } @Override public FSDataOutputStream create(Path f, Progressable progress) throws IOException { return delegate.create(f, progress); } @Override public FSDataOutputStream create(Path f, short replication, Progressable progress) throws IOException { return delegate.create(f, replication, progress); } @Override public FSDataOutputStream create(Path f, short replication) throws IOException { return delegate.create(f, replication); } @Override public FSDataOutputStream create(Path f) throws IOException { return delegate.create(f); } @Override public boolean createNewFile(Path f) throws IOException { return delegate.createNewFile(f); } @Override public boolean delete(Path arg0, boolean arg1) throws IOException { return delegate.delete(arg0, arg1); } @Override public boolean delete(Path arg0) throws IOException { return delegate.delete(arg0); } @Override public boolean deleteOnExit(Path f) throws IOException { return delegate.deleteOnExit(f); } @Override public boolean equals(Object obj) { return delegate.equals(obj); } @Override public boolean exists(Path arg0) throws IOException { return delegate.exists(arg0); } @Override public long getBlockSize(Path f) throws IOException { return delegate.getBlockSize(f); } @Override public Configuration getConf() { return delegate.getConf(); } @Override public ContentSummary getContentSummary(Path arg0) throws IOException { return delegate.getContentSummary(arg0); } @Override public long getDefaultBlockSize() { return delegate.getDefaultBlockSize(); } @Override public short getDefaultReplication() { return delegate.getDefaultReplication(); } @Override public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException { return delegate.getFileBlockLocations(file, start, len); } @Override public FileChecksum getFileChecksum(Path f) throws IOException { return delegate.getFileChecksum(f); } @Override public FileStatus getFileStatus(Path arg0) throws IOException { return delegate.getFileStatus(arg0); } @Override public Path getHomeDirectory() { return delegate.getHomeDirectory(); } @Override public long getLength(Path f) throws IOException { return delegate.getLength(f); } @Override public String getName() { return delegate.getName(); } @Override public short getReplication(Path src) throws IOException { return delegate.getReplication(src); } @Override public URI getUri() { return delegate.getUri(); } @Override public long getUsed() throws IOException { return delegate.getUsed(); } @Override public Path getWorkingDirectory() { return delegate.getWorkingDirectory(); } @Override public FileStatus[] globStatus(Path arg0, PathFilter arg1) throws IOException { return delegate.globStatus(arg0, arg1); } @Override public FileStatus[] globStatus(Path pathPattern) throws IOException { return delegate.globStatus(pathPattern); } @Override public int hashCode() { return delegate.hashCode(); } @Override public void initialize(URI name, Configuration conf) throws IOException { delegate.initialize(name, conf); } @Override public boolean isDirectory(Path arg0) throws IOException { return delegate.isDirectory(arg0); } @Override public boolean isFile(Path arg0) throws IOException { return delegate.isFile(arg0); } @Override public FileStatus[] listStatus(Path f, PathFilter filter) throws IOException { return delegate.listStatus(f, filter); } @Override public FileStatus[] listStatus(Path[] arg0, PathFilter arg1) throws IOException { return delegate.listStatus(arg0, arg1); } @Override public FileStatus[] listStatus(Path[] files) throws IOException { return delegate.listStatus(files); } @Override public Path makeQualified(Path path) { return delegate.makeQualified(path); } @Override public boolean mkdirs(Path arg0, FsPermission arg1) throws IOException { return delegate.mkdirs(arg0, arg1); } @Override public boolean mkdirs(Path f) throws IOException { return delegate.mkdirs(f); } @Override public void moveFromLocalFile(Path src, Path dst) throws IOException { delegate.moveFromLocalFile(src, dst); } @Override public void moveFromLocalFile(Path[] srcs, Path dst) throws IOException { delegate.moveFromLocalFile(srcs, dst); } @Override public void moveToLocalFile(Path src, Path dst) throws IOException { delegate.moveToLocalFile(src, dst); } @Override public FSDataInputStream open(Path arg0, int arg1) throws IOException { return delegate.open(arg0, arg1); } @Override public FSDataInputStream open(Path f) throws IOException { return delegate.open(f); } @Override public boolean rename(Path arg0, Path arg1) throws IOException { return delegate.rename(arg0, arg1); } @Override public void setConf(Configuration conf) { if (null != delegate) { delegate.setConf(conf); } } @Override public void setOwner(Path p, String username, String groupname) throws IOException { delegate.setOwner(p, username, groupname); } @Override public void setPermission(Path p, FsPermission permission) throws IOException { delegate.setPermission(p, permission); } @Override public boolean setReplication(Path src, short replication) throws IOException { return delegate.setReplication(src, replication); } @Override public void setTimes(Path p, long mtime, long atime) throws IOException { delegate.setTimes(p, mtime, atime); } @Override public void setVerifyChecksum(boolean verifyChecksum) { delegate.setVerifyChecksum(verifyChecksum); } @Override public void setWorkingDirectory(Path arg0) { delegate.setWorkingDirectory(arg0); } @Override public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException { return delegate.startLocalOutput(fsOutputFile, tmpLocalFile); } @Override public String toString() { return delegate.toString(); } } }