Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.fs.slive; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.DataInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Random; import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.slive.ArgumentParser.ParsedOutput; import org.apache.hadoop.fs.slive.Constants.OperationType; import org.apache.hadoop.fs.slive.DataVerifier.VerifyOutput; import org.apache.hadoop.fs.slive.DataWriter.GenerateOutput; import org.apache.hadoop.util.ToolRunner; import org.junit.Before; import org.junit.Test; /** * Junit 4 test for slive */ public class TestSlive { private static final Log LOG = LogFactory.getLog(TestSlive.class); private static final Random rnd = new Random(1L); private static final String TEST_DATA_PROP = "test.build.data"; private static Configuration getBaseConfig() { Configuration conf = new Configuration(); return conf; } /** gets the test write location according to the coding guidelines */ private static File getWriteLoc() { String writeLoc = System.getProperty(TEST_DATA_PROP, "build/test/data/"); return new File(writeLoc, "slive"); } /** gets where the MR job places its data + output + results */ private static File getFlowLocation() { return new File(getWriteLoc(), "flow"); } /** gets the test directory which is created by the mkdir op */ private static File getTestDir() { return new File(getWriteLoc(), "slivedir"); } /** * gets the test file location * which is used for reading, appending and created */ private static File getTestFile() { return new File(getWriteLoc(), "slivefile"); } /** * gets the rename file which is used in combination * with the test file to do a rename operation */ private static File getTestRenameFile() { return new File(getWriteLoc(), "slivefile1"); } /** gets the MR result file name */ private static File getResultFile() { return new File(getWriteLoc(), "sliveresfile"); } private static File getImaginaryFile() { return new File(getWriteLoc(), "slivenofile"); } /** gets the test program arguments used for merging and main MR running */ private String[] getTestArgs(boolean sleep) { List<String> args = new LinkedList<String>(); // setup the options { args.add("-" + ConfigOption.WRITE_SIZE.getOpt()); args.add("1M,2M"); args.add("-" + ConfigOption.OPS.getOpt()); args.add(Constants.OperationType.values().length + ""); args.add("-" + ConfigOption.MAPS.getOpt()); args.add("2"); args.add("-" + ConfigOption.REDUCES.getOpt()); args.add("2"); args.add("-" + ConfigOption.APPEND_SIZE.getOpt()); args.add("1M,2M"); args.add("-" + ConfigOption.BLOCK_SIZE.getOpt()); args.add("1M,2M"); args.add("-" + ConfigOption.REPLICATION_AM.getOpt()); args.add("1,1"); if (sleep) { args.add("-" + ConfigOption.SLEEP_TIME.getOpt()); args.add("10,10"); } args.add("-" + ConfigOption.RESULT_FILE.getOpt()); args.add(getResultFile().toString()); args.add("-" + ConfigOption.BASE_DIR.getOpt()); args.add(getFlowLocation().toString()); args.add("-" + ConfigOption.DURATION.getOpt()); args.add("10"); args.add("-" + ConfigOption.DIR_SIZE.getOpt()); args.add("10"); args.add("-" + ConfigOption.FILES.getOpt()); args.add("10"); } return args.toArray(new String[args.size()]); } @Test public void testFinder() throws Exception { ConfigExtractor extractor = getTestConfig(false); PathFinder fr = new PathFinder(extractor, rnd); // should only be able to select 10 files // attempt for a given amount of iterations int maxIterations = 10000; Set<Path> files = new HashSet<Path>(); for (int i = 0; i < maxIterations; i++) { files.add(fr.getFile()); } assertTrue(files.size() == 10); Set<Path> dirs = new HashSet<Path>(); for (int i = 0; i < maxIterations; i++) { dirs.add(fr.getDirectory()); } assertTrue(dirs.size() == 10); } @Test public void testSelection() throws Exception { ConfigExtractor extractor = getTestConfig(false); WeightSelector selector = new WeightSelector(extractor, rnd); // should be 1 of each type - uniform int expected = OperationType.values().length; Operation op = null; Set<String> types = new HashSet<String>(); FileSystem fs = FileSystem.get(extractor.getConfig()); while (true) { op = selector.select(1, 1); if (op == null) { break; } // doesn't matter if they work or not op.run(fs); types.add(op.getType()); } assertEquals(types.size(), expected); } // gets the config merged with the arguments private ConfigExtractor getTestConfig(boolean sleep) throws Exception { ArgumentParser parser = new ArgumentParser(getTestArgs(sleep)); ParsedOutput out = parser.parse(); assertTrue(!out.shouldOutputHelp()); ConfigMerger merge = new ConfigMerger(); Configuration cfg = merge.getMerged(out, getBaseConfig()); ConfigExtractor extractor = new ConfigExtractor(cfg); return extractor; } @Before public void ensureDeleted() throws Exception { rDelete(getTestFile()); rDelete(getTestDir()); rDelete(getTestRenameFile()); rDelete(getResultFile()); rDelete(getFlowLocation()); rDelete(getImaginaryFile()); } /** cleans up a file or directory recursively if need be */ private void rDelete(File place) throws Exception { if (place.isFile()) { LOG.info("Deleting file " + place); assertTrue(place.delete()); } else if (place.isDirectory()) { deleteDir(place); } } /** deletes a dir and its contents */ private void deleteDir(File dir) throws Exception { String fns[] = dir.list(); // delete contents first for (String afn : fns) { File fn = new File(dir, afn); rDelete(fn); } LOG.info("Deleting directory " + dir); // now delete the dir assertTrue(dir.delete()); } @Test public void testArguments() throws Exception { ConfigExtractor extractor = getTestConfig(true); assertEquals(extractor.getOpCount().intValue(), Constants.OperationType.values().length); assertEquals(extractor.getMapAmount().intValue(), 2); assertEquals(extractor.getReducerAmount().intValue(), 2); Range<Long> apRange = extractor.getAppendSize(); assertEquals(apRange.getLower().intValue(), Constants.MEGABYTES * 1); assertEquals(apRange.getUpper().intValue(), Constants.MEGABYTES * 2); Range<Long> wRange = extractor.getWriteSize(); assertEquals(wRange.getLower().intValue(), Constants.MEGABYTES * 1); assertEquals(wRange.getUpper().intValue(), Constants.MEGABYTES * 2); Range<Long> bRange = extractor.getBlockSize(); assertEquals(bRange.getLower().intValue(), Constants.MEGABYTES * 1); assertEquals(bRange.getUpper().intValue(), Constants.MEGABYTES * 2); String resfile = extractor.getResultFile(); assertEquals(resfile, getResultFile().toString()); int durationMs = extractor.getDurationMilliseconds(); assertEquals(durationMs, 10 * 1000); } @Test public void testDataWriting() throws Exception { long byteAm = 100; File fn = getTestFile(); DataWriter writer = new DataWriter(rnd); FileOutputStream fs = new FileOutputStream(fn); GenerateOutput ostat = writer.writeSegment(byteAm, fs); LOG.info(ostat); fs.close(); assertTrue(ostat.getBytesWritten() == byteAm); DataVerifier vf = new DataVerifier(); FileInputStream fin = new FileInputStream(fn); VerifyOutput vfout = vf.verifyFile(byteAm, new DataInputStream(fin)); LOG.info(vfout); fin.close(); assertEquals(vfout.getBytesRead(), byteAm); assertTrue(vfout.getChunksDifferent() == 0); } @Test public void testRange() { Range<Long> r = new Range<Long>(10L, 20L); assertEquals(r.getLower().longValue(), 10L); assertEquals(r.getUpper().longValue(), 20L); } @Test public void testCreateOp() throws Exception { // setup a valid config ConfigExtractor extractor = getTestConfig(false); final Path fn = new Path(getTestFile().getCanonicalPath()); CreateOp op = new CreateOp(extractor, rnd) { protected Path getCreateFile() { return fn; } }; runOperationOk(extractor, op, true); } @Test public void testOpFailures() throws Exception { ConfigExtractor extractor = getTestConfig(false); final Path fn = new Path(getImaginaryFile().getCanonicalPath()); ReadOp rop = new ReadOp(extractor, rnd) { protected Path getReadFile() { return fn; } }; runOperationBad(extractor, rop); DeleteOp dop = new DeleteOp(extractor, rnd) { protected Path getDeleteFile() { return fn; } }; runOperationBad(extractor, dop); RenameOp reop = new RenameOp(extractor, rnd) { protected SrcTarget getRenames() { return new SrcTarget(fn, fn); } }; runOperationBad(extractor, reop); AppendOp aop = new AppendOp(extractor, rnd) { protected Path getAppendFile() { return fn; } }; runOperationBad(extractor, aop); } private void runOperationBad(ConfigExtractor cfg, Operation op) throws Exception { FileSystem fs = FileSystem.get(cfg.getConfig()); List<OperationOutput> data = op.run(fs); assertTrue(!data.isEmpty()); boolean foundFail = false; for (OperationOutput d : data) { if (d.getMeasurementType().equals(ReportWriter.FAILURES)) { foundFail = true; } if (d.getMeasurementType().equals(ReportWriter.NOT_FOUND)) { foundFail = true; } } assertTrue(foundFail); } private void runOperationOk(ConfigExtractor cfg, Operation op, boolean checkOk) throws Exception { FileSystem fs = FileSystem.get(cfg.getConfig()); List<OperationOutput> data = op.run(fs); assertTrue(!data.isEmpty()); if (checkOk) { boolean foundSuc = false; boolean foundOpCount = false; boolean foundTime = false; for (OperationOutput d : data) { assertTrue(!d.getMeasurementType().equals(ReportWriter.FAILURES)); if (d.getMeasurementType().equals(ReportWriter.SUCCESSES)) { foundSuc = true; } if (d.getMeasurementType().equals(ReportWriter.OP_COUNT)) { foundOpCount = true; } if (d.getMeasurementType().equals(ReportWriter.OK_TIME_TAKEN)) { foundTime = true; } } assertTrue(foundSuc); assertTrue(foundOpCount); assertTrue(foundTime); } } @Test public void testDelete() throws Exception { ConfigExtractor extractor = getTestConfig(false); final Path fn = new Path(getTestFile().getCanonicalPath()); // ensure file created before delete CreateOp op = new CreateOp(extractor, rnd) { protected Path getCreateFile() { return fn; } }; runOperationOk(extractor, op, true); // now delete DeleteOp dop = new DeleteOp(extractor, rnd) { protected Path getDeleteFile() { return fn; } }; runOperationOk(extractor, dop, true); } @Test public void testRename() throws Exception { ConfigExtractor extractor = getTestConfig(false); final Path src = new Path(getTestFile().getCanonicalPath()); final Path tgt = new Path(getTestRenameFile().getCanonicalPath()); // ensure file created before rename CreateOp op = new CreateOp(extractor, rnd) { protected Path getCreateFile() { return src; } }; runOperationOk(extractor, op, true); RenameOp rop = new RenameOp(extractor, rnd) { protected SrcTarget getRenames() { return new SrcTarget(src, tgt); } }; runOperationOk(extractor, rop, true); } @Test public void testMRFlow() throws Exception { ConfigExtractor extractor = getTestConfig(false); SliveTest s = new SliveTest(getBaseConfig()); int ec = ToolRunner.run(s, getTestArgs(false)); assertTrue(ec == 0); String resFile = extractor.getResultFile(); File fn = new File(resFile); assertTrue(fn.exists()); // can't validate completely since operations may fail (mainly anyone but // create +mkdir) since they may not find there files } @Test public void testRead() throws Exception { ConfigExtractor extractor = getTestConfig(false); final Path fn = new Path(getTestFile().getCanonicalPath()); // ensure file created before read CreateOp op = new CreateOp(extractor, rnd) { protected Path getCreateFile() { return fn; } }; runOperationOk(extractor, op, true); ReadOp rop = new ReadOp(extractor, rnd) { protected Path getReadFile() { return fn; } }; runOperationOk(extractor, rop, true); } @Test public void testSleep() throws Exception { ConfigExtractor extractor = getTestConfig(true); SleepOp op = new SleepOp(extractor, rnd); runOperationOk(extractor, op, true); } @Test public void testList() throws Exception { // ensure dir made ConfigExtractor extractor = getTestConfig(false); final Path dir = new Path(getTestDir().getCanonicalPath()); MkdirOp op = new MkdirOp(extractor, rnd) { protected Path getDirectory() { return dir; } }; runOperationOk(extractor, op, true); // list it ListOp lop = new ListOp(extractor, rnd) { protected Path getDirectory() { return dir; } }; runOperationOk(extractor, lop, true); } @Test public void testBadChunks() throws Exception { File fn = getTestFile(); int byteAm = 10000; FileOutputStream fout = new FileOutputStream(fn); byte[] bytes = new byte[byteAm]; rnd.nextBytes(bytes); fout.write(bytes); fout.close(); // attempt to read it DataVerifier vf = new DataVerifier(); VerifyOutput vout = new VerifyOutput(0, 0, 0, 0); DataInputStream in = null; try { in = new DataInputStream(new FileInputStream(fn)); vout = vf.verifyFile(byteAm, in); } catch (Exception e) { } finally { if (in != null) in.close(); } assertTrue(vout.getChunksSame() == 0); } @Test public void testMkdir() throws Exception { ConfigExtractor extractor = getTestConfig(false); final Path dir = new Path(getTestDir().getCanonicalPath()); MkdirOp op = new MkdirOp(extractor, rnd) { protected Path getDirectory() { return dir; } }; runOperationOk(extractor, op, true); } @Test public void testSelector() throws Exception { ConfigExtractor extractor = getTestConfig(false); RouletteSelector selector = new RouletteSelector(rnd); List<OperationWeight> sList = new LinkedList<OperationWeight>(); Operation op = selector.select(sList); assertTrue(op == null); CreateOp cop = new CreateOp(extractor, rnd); sList.add(new OperationWeight(cop, 1.0d)); AppendOp aop = new AppendOp(extractor, rnd); sList.add(new OperationWeight(aop, 0.01d)); op = selector.select(sList); assertTrue(op == cop); } @Test public void testAppendOp() throws Exception { // setup a valid config ConfigExtractor extractor = getTestConfig(false); // ensure file created before append final Path fn = new Path(getTestFile().getCanonicalPath()); CreateOp op = new CreateOp(extractor, rnd) { protected Path getCreateFile() { return fn; } }; runOperationOk(extractor, op, true); // local file system (ChecksumFileSystem) currently doesn't support append - // but we'll leave this test here anyways but can't check the results.. AppendOp aop = new AppendOp(extractor, rnd) { protected Path getAppendFile() { return fn; } }; runOperationOk(extractor, aop, false); } }