Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapred.gridmix; import java.io.IOException; import java.util.Arrays; import java.util.Random; import org.junit.Test; import static org.junit.Assert.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.io.WritableUtils; public class TestGridmixRecord { private static final Log LOG = LogFactory.getLog(TestGridmixRecord.class); static void lengthTest(GridmixRecord x, GridmixRecord y, int min, int max) throws Exception { final Random r = new Random(); final long seed = r.nextLong(); r.setSeed(seed); LOG.info("length: " + seed); final DataInputBuffer in = new DataInputBuffer(); final DataOutputBuffer out1 = new DataOutputBuffer(); final DataOutputBuffer out2 = new DataOutputBuffer(); for (int i = min; i < max; ++i) { setSerialize(x, r.nextLong(), i, out1); // check write assertEquals(i, out1.getLength()); // write to stream x.write(out2); // check read in.reset(out1.getData(), 0, out1.getLength()); y.readFields(in); assertEquals(i, x.getSize()); assertEquals(i, y.getSize()); } // check stream read in.reset(out2.getData(), 0, out2.getLength()); for (int i = min; i < max; ++i) { y.readFields(in); assertEquals(i, y.getSize()); } } static void randomReplayTest(GridmixRecord x, GridmixRecord y, int min, int max) throws Exception { final Random r = new Random(); final long seed = r.nextLong(); r.setSeed(seed); LOG.info("randReplay: " + seed); final DataOutputBuffer out1 = new DataOutputBuffer(); for (int i = min; i < max; ++i) { final int s = out1.getLength(); x.setSeed(r.nextLong()); x.setSize(i); x.write(out1); assertEquals(i, out1.getLength() - s); } final DataInputBuffer in = new DataInputBuffer(); in.reset(out1.getData(), 0, out1.getLength()); final DataOutputBuffer out2 = new DataOutputBuffer(); // deserialize written records, write to separate buffer for (int i = min; i < max; ++i) { final int s = in.getPosition(); y.readFields(in); assertEquals(i, in.getPosition() - s); y.write(out2); } // verify written contents match assertEquals(out1.getLength(), out2.getLength()); // assumes that writes will grow buffer deterministically assertEquals("Bad test", out1.getData().length, out2.getData().length); assertArrayEquals(out1.getData(), out2.getData()); } static void eqSeedTest(GridmixRecord x, GridmixRecord y, int max) throws Exception { final Random r = new Random(); final long s = r.nextLong(); r.setSeed(s); LOG.info("eqSeed: " + s); assertEquals(x.fixedBytes(), y.fixedBytes()); final int min = x.fixedBytes() + 1; final DataOutputBuffer out1 = new DataOutputBuffer(); final DataOutputBuffer out2 = new DataOutputBuffer(); for (int i = min; i < max; ++i) { final long seed = r.nextLong(); setSerialize(x, seed, i, out1); setSerialize(y, seed, i, out2); assertEquals(x, y); assertEquals(x.hashCode(), y.hashCode()); // verify written contents match assertEquals(out1.getLength(), out2.getLength()); // assumes that writes will grow buffer deterministically assertEquals("Bad test", out1.getData().length, out2.getData().length); assertArrayEquals(out1.getData(), out2.getData()); } } static void binSortTest(GridmixRecord x, GridmixRecord y, int min, int max, WritableComparator cmp) throws Exception { final Random r = new Random(); final long s = r.nextLong(); r.setSeed(s); LOG.info("sort: " + s); final DataOutputBuffer out1 = new DataOutputBuffer(); final DataOutputBuffer out2 = new DataOutputBuffer(); for (int i = min; i < max; ++i) { final long seed1 = r.nextLong(); setSerialize(x, seed1, i, out1); assertEquals(0, x.compareSeed(seed1, Math.max(0, i - x.fixedBytes()))); final long seed2 = r.nextLong(); setSerialize(y, seed2, i, out2); assertEquals(0, y.compareSeed(seed2, Math.max(0, i - x.fixedBytes()))); // for eq sized records, ensure byte cmp where req final int chk = WritableComparator.compareBytes(out1.getData(), 0, out1.getLength(), out2.getData(), 0, out2.getLength()); assertEquals(chk, x.compareTo(y)); assertEquals(chk, cmp.compare(out1.getData(), 0, out1.getLength(), out2.getData(), 0, out2.getLength())); // write second copy, compare eq final int s1 = out1.getLength(); x.write(out1); assertEquals(0, cmp.compare(out1.getData(), 0, s1, out1.getData(), s1, out1.getLength() - s1)); final int s2 = out2.getLength(); y.write(out2); assertEquals(0, cmp.compare(out2.getData(), 0, s2, out2.getData(), s2, out2.getLength() - s2)); assertEquals(chk, cmp.compare(out1.getData(), 0, s1, out2.getData(), s2, out2.getLength() - s2)); } } static void checkSpec(GridmixKey a, GridmixKey b) throws Exception { final Random r = new Random(); final long s = r.nextLong(); r.setSeed(s); LOG.info("spec: " + s); final DataInputBuffer in = new DataInputBuffer(); final DataOutputBuffer out = new DataOutputBuffer(); a.setType(GridmixKey.REDUCE_SPEC); b.setType(GridmixKey.REDUCE_SPEC); for (int i = 0; i < 100; ++i) { final int in_rec = r.nextInt(Integer.MAX_VALUE); a.setReduceInputRecords(in_rec); final int out_rec = r.nextInt(Integer.MAX_VALUE); a.setReduceOutputRecords(out_rec); final int out_bytes = r.nextInt(Integer.MAX_VALUE); a.setReduceOutputBytes(out_bytes); final int min = WritableUtils.getVIntSize(in_rec) + WritableUtils.getVIntSize(out_rec) + WritableUtils.getVIntSize(out_bytes); assertEquals(min + 2, a.fixedBytes()); // meta + vint min final int size = r.nextInt(1024) + a.fixedBytes() + 1; setSerialize(a, r.nextLong(), size, out); assertEquals(size, out.getLength()); assertTrue(a.equals(a)); assertEquals(0, a.compareTo(a)); in.reset(out.getData(), 0, out.getLength()); b.readFields(in); assertEquals(size, b.getSize()); assertEquals(in_rec, b.getReduceInputRecords()); assertEquals(out_rec, b.getReduceOutputRecords()); assertEquals(out_bytes, b.getReduceOutputBytes()); assertTrue(a.equals(b)); assertEquals(0, a.compareTo(b)); assertEquals(a.hashCode(), b.hashCode()); } } static void setSerialize(GridmixRecord x, long seed, int size, DataOutputBuffer out) throws IOException { x.setSeed(seed); x.setSize(size); out.reset(); x.write(out); } @Test public void testKeySpec() throws Exception { final int min = 5; final int max = 300; final GridmixKey a = new GridmixKey(GridmixKey.REDUCE_SPEC, 1, 0L); final GridmixKey b = new GridmixKey(GridmixKey.REDUCE_SPEC, 1, 0L); lengthTest(a, b, min, max); randomReplayTest(a, b, min, max); binSortTest(a, b, min, max, new GridmixKey.Comparator()); // 2 fixed GR bytes, 1 type, 3 spec eqSeedTest(a, b, max); checkSpec(a, b); } @Test public void testKeyData() throws Exception { final int min = 2; final int max = 300; final GridmixKey a = new GridmixKey(GridmixKey.DATA, 1, 0L); final GridmixKey b = new GridmixKey(GridmixKey.DATA, 1, 0L); lengthTest(a, b, min, max); randomReplayTest(a, b, min, max); binSortTest(a, b, min, max, new GridmixKey.Comparator()); // 2 fixed GR bytes, 1 type eqSeedTest(a, b, 300); } @Test public void testBaseRecord() throws Exception { final int min = 1; final int max = 300; final GridmixRecord a = new GridmixRecord(); final GridmixRecord b = new GridmixRecord(); lengthTest(a, b, min, max); randomReplayTest(a, b, min, max); binSortTest(a, b, min, max, new GridmixRecord.Comparator()); // 2 fixed GR bytes eqSeedTest(a, b, 300); } public static void main(String[] argv) throws Exception { boolean fail = false; final TestGridmixRecord test = new TestGridmixRecord(); try { test.testKeySpec(); } catch (Exception e) { fail = true; e.printStackTrace(); } try { test.testKeyData(); } catch (Exception e) { fail = true; e.printStackTrace(); } try { test.testBaseRecord(); } catch (Exception e) { fail = true; e.printStackTrace(); } System.exit(fail ? -1 : 0); } static void printDebug(GridmixRecord a, GridmixRecord b) throws IOException { DataOutputBuffer out = new DataOutputBuffer(); a.write(out); System.out.println("A " + Arrays.toString(Arrays.copyOf(out.getData(), out.getLength()))); out.reset(); b.write(out); System.out.println("B " + Arrays.toString(Arrays.copyOf(out.getData(), out.getLength()))); } }