Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapred.gridmix; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.Random; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import static org.junit.Assert.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; public class TestFilePool { static final Log LOG = LogFactory.getLog(TestFileQueue.class); static final int NFILES = 26; static final Path base = getBaseDir(); static Path getBaseDir() { try { final Configuration conf = new Configuration(); final FileSystem fs = FileSystem.getLocal(conf).getRaw(); return new Path(System.getProperty("test.build.data", "/tmp"), "testFilePool").makeQualified(fs); } catch (IOException e) { fail(); } return null; } @BeforeClass public static void setup() throws IOException { final Configuration conf = new Configuration(); final FileSystem fs = FileSystem.getLocal(conf).getRaw(); fs.delete(base, true); final Random r = new Random(); final long seed = r.nextLong(); r.setSeed(seed); LOG.info("seed: " + seed); fs.mkdirs(base); for (int i = 0; i < NFILES; ++i) { Path file = base; for (double d = 0.6; d > 0.0; d *= 0.8) { if (r.nextDouble() < d) { file = new Path(base, Integer.toString(r.nextInt(3))); continue; } break; } OutputStream out = null; try { out = fs.create(new Path(file, "" + (char) ('A' + i))); final byte[] b = new byte[1024]; Arrays.fill(b, (byte) ('A' + i)); for (int len = ((i % 13) + 1) * 1024; len > 0; len -= 1024) { out.write(b); } } finally { if (out != null) { out.close(); } } } } @AfterClass public static void cleanup() throws IOException { final Configuration conf = new Configuration(); final FileSystem fs = FileSystem.getLocal(conf).getRaw(); fs.delete(base, true); } @Test public void testUnsuitable() throws Exception { try { final Configuration conf = new Configuration(); // all files 13k or less conf.setLong(FilePool.GRIDMIX_MIN_FILE, 14 * 1024); final FilePool pool = new FilePool(conf, base); pool.refresh(); } catch (IOException e) { return; } fail(); } @Test public void testPool() throws Exception { final Random r = new Random(); final Configuration conf = new Configuration(); conf.setLong(FilePool.GRIDMIX_MIN_FILE, 3 * 1024); final FilePool pool = new FilePool(conf, base); pool.refresh(); final ArrayList<FileStatus> files = new ArrayList<FileStatus>(); // ensure 1k, 2k files excluded final int expectedPoolSize = (NFILES / 2 * (NFILES / 2 + 1) - 6) * 1024; assertEquals(expectedPoolSize, pool.getInputFiles(Long.MAX_VALUE, files)); assertEquals(NFILES - 4, files.size()); // exact match files.clear(); assertEquals(expectedPoolSize, pool.getInputFiles(expectedPoolSize, files)); // match random within 12k files.clear(); final long rand = r.nextInt(expectedPoolSize); assertTrue("Missed: " + rand, (NFILES / 2) * 1024 > rand - pool.getInputFiles(rand, files)); // all files conf.setLong(FilePool.GRIDMIX_MIN_FILE, 0); pool.refresh(); files.clear(); assertEquals((NFILES / 2 * (NFILES / 2 + 1)) * 1024, pool.getInputFiles(Long.MAX_VALUE, files)); } void checkSplitEq(FileSystem fs, CombineFileSplit split, long bytes) throws Exception { long splitBytes = 0L; HashSet<Path> uniq = new HashSet<Path>(); for (int i = 0; i < split.getNumPaths(); ++i) { splitBytes += split.getLength(i); assertTrue(split.getLength(i) <= fs.getFileStatus(split.getPath(i)).getLen()); assertFalse(uniq.contains(split.getPath(i))); uniq.add(split.getPath(i)); } assertEquals(bytes, splitBytes); } @Test public void testStriper() throws Exception { final Random r = new Random(); final Configuration conf = new Configuration(); final FileSystem fs = FileSystem.getLocal(conf).getRaw(); conf.setLong(FilePool.GRIDMIX_MIN_FILE, 3 * 1024); final FilePool pool = new FilePool(conf, base) { @Override public BlockLocation[] locationsFor(FileStatus stat, long start, long len) throws IOException { return new BlockLocation[] { new BlockLocation() }; } }; pool.refresh(); final int expectedPoolSize = (NFILES / 2 * (NFILES / 2 + 1) - 6) * 1024; final InputStriper striper = new InputStriper(pool, expectedPoolSize); int last = 0; for (int i = 0; i < expectedPoolSize; last = Math.min(expectedPoolSize - i, r.nextInt(expectedPoolSize))) { checkSplitEq(fs, striper.splitFor(pool, last, 0), last); i += last; } final InputStriper striper2 = new InputStriper(pool, expectedPoolSize); checkSplitEq(fs, striper2.splitFor(pool, expectedPoolSize, 0), expectedPoolSize); } }