Java tutorial
/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.addthis.hydra.data.io; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Random; import com.addthis.basis.test.SlowTest; import com.addthis.basis.util.LessBytes; import com.addthis.basis.util.LessFiles; import com.addthis.codec.annotations.FieldConfig; import com.addthis.codec.codables.Codable; import com.addthis.hydra.data.io.DiskBackedList2.ItemCodec; import com.google.common.io.Files; import org.apache.commons.io.output.ByteArrayOutputStream; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; @Category(SlowTest.class) public class TestDiskBackedList2 { private final TestValueCodec codec = new TestValueCodec(); public static class TestValueCodec implements ItemCodec<TestValue> { @Override public TestValue decode(byte[] row) throws IOException { ByteArrayInputStream in = new ByteArrayInputStream(row); TestValue tv = new TestValue(); tv.value = LessBytes.readLong(in); tv.randomBytes = LessBytes.readBytes(in); return tv; } @Override public byte[] encode(TestValue row) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); LessBytes.writeLong(row.value, out); LessBytes.writeBytes(row.randomBytes, out); return out.toByteArray(); } } public static final class TestValue implements Codable { private static final Random random = new Random(1234); @FieldConfig(codable = true) private long value; @FieldConfig(codable = true) private byte[] randomBytes; public TestValue() { } public TestValue(long val) { this.value = val; this.randomBytes = new byte[random.nextInt() & 0xff]; random.nextBytes(randomBytes); } @Override public boolean equals(Object o) { return (o instanceof TestValue) && ((TestValue) o).value == value; } @Override public String toString() { return "TV:" + value + ":" + randomBytes.length; } } @Test public void addRemoveSizeTest() throws IOException { Random random = new Random(58); DiskBackedList2<TestValue> dbl = new DiskBackedList2<>(codec); try { int numEntries = 1000; int[] vals = new int[numEntries]; for (int i = 0; i < numEntries; i++) { vals[i] = random.nextInt(3 * numEntries); } // Add a bunch of entries and make sure they all get set correctly for (int i = 0; i < numEntries; i++) { dbl.add(new TestValue(vals[i])); Assert.assertEquals(vals[i], dbl.get(i).value); } // Make sure resulting size is correct Assert.assertEquals(numEntries, dbl.size()); // Remove half of the entries, check if new size is correct for (int i = (numEntries / 2); i > 0; i--) { dbl.remove(i); } Assert.assertEquals(numEntries - numEntries / 2, dbl.size()); dbl.clear(); } finally { LessFiles.deleteDir(dbl.getDirectory()); } } @Test public void addAtIndexTest() throws IOException { List<Integer> ints = new ArrayList<>(Arrays.asList(1, 2, 3, 4, 5)); DiskBackedList2<TestValue> dbl = new DiskBackedList2<>(codec); try { // Add a bunch of entries and make sure they all get set correctly for (Integer z : ints) { dbl.add(new TestValue(z)); } ints.add(2, 100); dbl.add(2, new TestValue(100)); int i = 0; for (TestValue tv : dbl) { Assert.assertEquals(new TestValue(ints.get(i++)), tv); } dbl.clear(); } finally { if (dbl != null) { LessFiles.deleteDir(dbl.getDirectory()); } } } @Test public void canHoldBigLoadTest() throws IOException { DiskBackedList2<TestValue> dbl = new DiskBackedList2<>(codec); try { int numEntries = 1000000; // Add a large number of entries to make sure we don't overflow heap for (int i = 0; i < numEntries; i++) { dbl.add(new TestValue(i)); } // Make sure resulting size is correct Assert.assertEquals(numEntries, dbl.size()); // Make sure a random read from the middle gives the correct value int middleEntry = numEntries / 2; TestValue tv = dbl.get(middleEntry); Assert.assertEquals(middleEntry, tv.value); dbl.clear(); } finally { if (dbl != null) { LessFiles.deleteDir(dbl.getDirectory()); } } } @Test public void iteratorTest() throws IOException { // Test whether the iterator has the correct set of values and no extras DiskBackedList2<TestValue> dbl = new DiskBackedList2<>(codec); try { int numEntries = 10000; for (int i = 0; i < numEntries; i++) { dbl.add(new TestValue(i)); } Iterator<TestValue> listIter = dbl.listIterator(); for (int i = 0; i < numEntries; i++) { Assert.assertEquals(new TestValue(i), listIter.next()); } Assert.assertEquals(false, listIter.hasNext()); dbl.clear(); } finally { if (dbl != null) { LessFiles.deleteDir(dbl.getDirectory()); } } } @Test public void sortTest() throws IOException { // Manually sort a bunch of values, then make sure dbl.sort reproduces the same list DiskBackedList2<TestValue> dbl = new DiskBackedList2<>(codec); try { Random random = new Random(1984); int numEntries = 20000; int[] vals = new int[numEntries]; for (int i = 0; i < numEntries; i++) { vals[i] = random.nextInt(3 * numEntries); } int[] sortedVals = vals.clone(); Arrays.sort(sortedVals); for (int val : vals) { dbl.add(new TestValue(val)); } Comparator<TestValue> comp = new Comparator<TestValue>() { @Override public int compare(TestValue o1, TestValue o2) { return (int) (o1.value - o2.value); } }; dbl.sort(comp); int i = 0; for (TestValue tv : dbl) { Assert.assertEquals(sortedVals[i++], tv.value); } dbl.clear(); } finally { if (dbl != null) { LessFiles.deleteDir(dbl.getDirectory()); } } } @Test public void dontExceedMaxDiskSpaceTest() throws IOException { System.setProperty("max.total.query.size.bytes", "200"); DiskBackedList2<TestValue> dbl = new DiskBackedList2<>(codec, 50, Files.createTempDir()); try { boolean failed = false; for (int i = 0; i < 100; i++) { try { dbl.add(new TestValue(i)); } catch (RuntimeException ex) { System.out.println(ex); failed = true; break; } } Assert.assertEquals(true, failed); System.clearProperty("max.total.query.size.bytes"); dbl.clear(); } finally { if (dbl != null) { LessFiles.deleteDir(dbl.getDirectory()); } } } @Test public void runStressTests() throws IOException { // This test can be used to measure sort runtimes // stressTest(16000000,10000000); } public void stressTest(int chunkSize, int numEntries) throws IOException { long startTime = System.currentTimeMillis(); DiskBackedList2<TestValue> dbl = new DiskBackedList2<>(codec, chunkSize, Files.createTempDir()); try { int seed = 58; Random random = new Random(seed); for (int i = 0; i < numEntries; i++) { dbl.add(new TestValue(random.nextInt(3 * numEntries))); } Comparator<TestValue> comp = new Comparator<TestValue>() { @Override public int compare(TestValue o1, TestValue o2) { return (int) (o1.value - o2.value); } }; dbl.sort(comp); long totalTime = System.currentTimeMillis() - startTime; System.out .println("chunkSize " + chunkSize + " numEntries " + numEntries + " took " + totalTime + " ms"); dbl.clear(); } finally { if (dbl != null) { LessFiles.deleteDir(dbl.getDirectory()); } } } @Test public void saveToDirectoryTest() throws IOException { Random random = new Random(451); int numEntries = 100; int sizeBytes = 3000; File directory = Files.createTempDir(); try { DiskBackedList2<TestValue> dbl = new DiskBackedList2<>(codec, sizeBytes, directory); for (int i = 0; i < numEntries; i++) { dbl.add(new TestValue(random.nextInt(2101))); } dbl.saveAllChunksToDirectory(); DiskBackedList2<TestValue> anotherdbl = new DiskBackedList2<>(codec, sizeBytes, directory); anotherdbl.loadAllChunksFromDirectory(); Assert.assertEquals(dbl.size(), anotherdbl.size()); for (int i = 0; i < numEntries; i++) { Assert.assertEquals(dbl.get(i).value, anotherdbl.get(i).value); } dbl.clear(); } finally { if (directory != null) { LessFiles.deleteDir(directory); } } } }