Java tutorial
/** * */ package com.linkedin.databus.core; /* * * Copyright 2013 LinkedIn Corp. All rights reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * */ import static org.testng.AssertJUnit.assertEquals; import java.io.File; import java.util.Arrays; import java.util.Vector; import junit.framework.Assert; import org.apache.commons.io.FileUtils; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import com.linkedin.databus.core.DbusEventBuffer.AllocationPolicy; import com.linkedin.databus.core.DbusEventBuffer.QueuePolicy; import com.linkedin.databus.core.test.DbusEventAppender; import com.linkedin.databus.core.test.DbusEventBufferReflector; import com.linkedin.databus.core.test.DbusEventGenerator; import com.linkedin.databus.core.util.BufferPositionParser; import com.linkedin.databus.core.util.InvalidConfigException; import com.linkedin.databus2.core.AssertLevel; import com.linkedin.databus2.test.TestUtil; /** * Seperate some tests from TestDbusEventBuffer since it has grown a lot. */ public class TestDbusEventBufferAppendEvents { @BeforeClass public void setupClass() { TestUtil.setupLogging(true, "TestDbusEventBufferAppendEvents" + System.currentTimeMillis() + ".log", Level.INFO); } @Test /** * Test the case where we have a big event that overlaps both the DbusEventBuffer head and the * limit of the current buffer. */ public void testAppendBigEventsHeadScnOverlap() throws Exception { final Logger log = Logger.getLogger("TestDbusEventBuffer.testAppendBigEventsHeadScnOverlap"); log.setLevel(Level.INFO); log.info("starting"); final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(1200, 500, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL)); BufferPositionParser parser = dbusBuf.getBufferPositionParser(); log.info("append initial events"); DbusEventGenerator generator = new DbusEventGenerator(); Vector<DbusEvent> events = new Vector<DbusEvent>(); generator.generateEvents(7, 1, 120, 39, events); // Add events to the EventBuffer. Now the buffer is full DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null); appender.run(); // running in the same thread log.info("Head:" + parser.toString(dbusBuf.getHead()) + ", Tail:" + parser.toString(dbusBuf.getTail())); log.info("Num buffers: " + dbusBuf.getBuffer().length); log.info("Buffer: " + Arrays.toString(dbusBuf.getBuffer())); long headPos = dbusBuf.getHead(); long tailPos = dbusBuf.getTail(); long headGenId = parser.bufferGenId(headPos); long headIndexId = parser.bufferIndex(headPos); long headOffset = parser.bufferOffset(headPos); long tailGenId = parser.bufferGenId(tailPos); long tailIndexId = parser.bufferIndex(tailPos); long tailOffset = parser.bufferOffset(tailPos); assertEquals(0, headGenId); assertEquals(0, headIndexId); assertEquals(222, headOffset); assertEquals(1, tailGenId); assertEquals(0, tailIndexId); assertEquals(61, tailOffset); log.info("append windows with one small and one big event"); generator = new DbusEventGenerator(100); events = new Vector<DbusEvent>(); generator.generateEvents(1, 1, 280, 139, events); generator.generateEvents(1, 1, 480, 339, events); // Add events to the EventBuffer. Now the buffer is full appender = new DbusEventAppender(events, dbusBuf, null); appender.run(); // running in the same thread log.info("Head:" + parser.toString(dbusBuf.getHead()) + ", Tail:" + parser.toString(dbusBuf.getTail())); log.info("Num buffers: " + dbusBuf.getBuffer().length); log.info("Buffer: " + Arrays.toString(dbusBuf.getBuffer())); headPos = dbusBuf.getHead(); tailPos = dbusBuf.getTail(); headGenId = parser.bufferGenId(headPos); headIndexId = parser.bufferIndex(headPos); headOffset = parser.bufferOffset(headPos); tailGenId = parser.bufferGenId(tailPos); tailIndexId = parser.bufferIndex(tailPos); tailOffset = parser.bufferOffset(tailPos); assertEquals(0, headGenId); assertEquals(2, headIndexId); assertEquals(61, headOffset); assertEquals(1, tailIndexId); assertEquals(461, tailOffset); assertEquals(322, dbusBuf.getBuffer()[0].limit()); log.info("finished"); } @Test /** * Test the following case: * A buffer (one of 3) has CWP = 222 and head at 383. Limit is 483 (capacity 500) * we create one event of size 211 (61 + 150 payload). * we call appender.run() which will add one EOP event (61) and the newly created event 211. * The end of event offset will be 222 + 211 + 61 = 494. It should update limit to this value. * because of the bug (DDSDBUS-1515) it used to fail when end of event goes beyond current limit * but less than the capacity * Now it should pass */ public void testAppendEventWhenLimitLessThanCapacity() throws Exception { final Logger log = Logger.getLogger("TestDbusEventBuffer.testAppendEventWhenLimitLessThanCapacity"); log.setLevel(Level.INFO); log.info("starting"); final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(1200, 500, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL)); BufferPositionParser parser = dbusBuf.getBufferPositionParser(); log.info("append initial events"); DbusEventGenerator generator = new DbusEventGenerator(); Vector<DbusEvent> events = new Vector<DbusEvent>(); generator.generateEvents(11, 1, 120, 39, events); // Add events to the EventBuffer. Now the buffer is full DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null); appender.run(); // running in the same thread log.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail())); log.info("Num buffers :" + dbusBuf.getBuffer().length); log.info("Buffer :" + Arrays.toString(dbusBuf.getBuffer())); long headPos = dbusBuf.getHead(); long tailPos = dbusBuf.getTail(); long headGenId = parser.bufferGenId(headPos); long headIndexId = parser.bufferIndex(headPos); long headOffset = parser.bufferOffset(headPos); long tailGenId = parser.bufferGenId(tailPos); long tailIndexId = parser.bufferIndex(tailPos); long tailOffset = parser.bufferOffset(tailPos); // current writing position should be 222 (id=1) assertEquals(0, headGenId); assertEquals(1, headIndexId); assertEquals(383, headOffset); assertEquals(1, tailGenId); assertEquals(1, tailIndexId); assertEquals(222, tailOffset); log.info("append event to stretch beyond limit but less than capacity"); generator = new DbusEventGenerator(100); events = new Vector<DbusEvent>(); generator.generateEvents(1, 1, 400, 150, events); // will add two event 61 + 150 // Add events. this will cause limit increased appender = new DbusEventAppender(events, dbusBuf, null); appender.run(); log.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail())); log.info("Num buffers :" + dbusBuf.getBuffer().length); log.info("Buffer :" + Arrays.toString(dbusBuf.getBuffer())); headPos = dbusBuf.getHead(); tailPos = dbusBuf.getTail(); headGenId = parser.bufferGenId(headPos); headIndexId = parser.bufferIndex(headPos); headOffset = parser.bufferOffset(headPos); tailGenId = parser.bufferGenId(tailPos); tailIndexId = parser.bufferIndex(tailPos); tailOffset = parser.bufferOffset(tailPos); assertEquals(1, headGenId); assertEquals(0, headIndexId); assertEquals(61, headOffset); assertEquals(2, tailIndexId); assertEquals(61, tailOffset); assertEquals(494, dbusBuf.getBuffer()[1].limit()); log.info("finished"); } @Test /** * verify that changes to the buffer (appendEvent, endEvent, clear) after the buffer is closed * are ignored (and rolled back) * we do this by creating buffer (or loading from mmap files from the previous run), adding one window, * then doing append, close , append/endEvents (which should throw an exception) * then loading again and verifying that it has our events */ public void testAppendEventsWithCloseInMiddle() throws Exception { final Logger log = Logger.getLogger("TestDbusEventBuffer.testAppendEventsWithCloseInMiddle"); log.setLevel(Level.DEBUG); log.info("starting"); // mmap dir File mmapDir = initMmapDir("/tmp/tmp_mmapDir"); DbusEventBuffer.StaticConfig config = getConfig(1200, 500, 100, 500, AllocationPolicy.MMAPPED_MEMORY, mmapDir.getAbsolutePath(), true); DbusEventBuffer dbusBuf = new DbusEventBuffer(config); DbusEventBufferReflector bufferReflector = new DbusEventBufferReflector(dbusBuf); // BufferPositionParser parser = dbusBuf.getBufferPositionParser(); log.info("append initial events"); DbusEventGenerator generator = new DbusEventGenerator(); Vector<DbusEvent> events = new Vector<DbusEvent>(); generator.generateEvents(12, 2, 120, 39, events); // add first window with 2 events dbusBuf.start(-1); int i = 0; addOneEvent(events.get(i++), dbusBuf, EventType.START); addOneEvent(events.get(i++), dbusBuf, EventType.END); checkEventsInBuffer(bufferReflector, 3); // 2 + 1(EOW) // add first event of the second window addOneEvent(events.get(i++), dbusBuf, EventType.START); checkEventsInBuffer(bufferReflector, 3); // 2 + 1(EOW) // now close the buffer dbusBuf.closeBuffer(true); checkEventsInBuffer(bufferReflector, 3); // 2 + 1(EOW) // now add the end try { addOneEvent(events.get(i), dbusBuf, EventType.END); } catch (Throwable ex) { log.info("Got e: ", ex); } checkEventsInBuffer(bufferReflector, 3); // 2 + 1(EOW) // create new buffer dbusBuf = new DbusEventBuffer(config); // should load from mmap bufferReflector = new DbusEventBufferReflector(dbusBuf); checkEventsInBuffer(bufferReflector, 3); // 2 + 1(EOW) addOneEvent(events.get(i++), dbusBuf, EventType.START); addOneEvent(events.get(i++), dbusBuf, EventType.END); checkEventsInBuffer(bufferReflector, 6); // 4 + 2(EOW) // add two more events but don't do endEvents addOneEvent(events.get(i++), dbusBuf, EventType.START); addOneEvent(events.get(i++), dbusBuf, EventType.REG); // no endEvents() // now close the buffer dbusBuf.closeBuffer(true); dbusBuf.closeBuffer(true); // should be ok (WARN in the logs) checkEventsInBuffer(bufferReflector, 6); // 4 + 2(EOW) // call endEvents(on a closed buffer); try { dbusBuf.endEvents(events.get(i - 1).sequence()); } catch (Throwable ex) { log.info("Got e2: ", ex); } checkEventsInBuffer(bufferReflector, 6); // 4 + 2(EOW) // create new buffer dbusBuf = new DbusEventBuffer(config); // should load from mmap bufferReflector = new DbusEventBufferReflector(dbusBuf); checkEventsInBuffer(bufferReflector, 6); // 4 + 2(EOW) addOneEvent(events.get(i++), dbusBuf, EventType.START); addOneEvent(events.get(i++), dbusBuf, EventType.END); checkEventsInBuffer(bufferReflector, 9); // 6 + 3(EOW) // add two more events but don't do endEvents addOneEvent(events.get(i++), dbusBuf, EventType.START); addOneEvent(events.get(i++), dbusBuf, EventType.REG); // no endEvents() // now close the buffer dbusBuf.closeBuffer(true); checkEventsInBuffer(bufferReflector, 9); // 6 + 3(EOW) // call endEvents(on a closed buffer); try { dbusBuf.clear(); //should fail } catch (Throwable ex) { log.info("Got e3: ", ex); } checkEventsInBuffer(bufferReflector, 9); // 6 + 3(EOW) // make sure it is still valid dbusBuf = new DbusEventBuffer(config); // should load from mmap bufferReflector = new DbusEventBufferReflector(dbusBuf); checkEventsInBuffer(bufferReflector, 9); // 6 + 3(EOW) } DbusEventBuffer.StaticConfig getConfig(long maxEventBufferSize, int maxIndividualBufferSize, int maxIndexSize, int maxReadBufferSize, AllocationPolicy allocationPolicy, String mmapDirectory, boolean restoreMMappedBuffers) throws InvalidConfigException { DbusEventBuffer.Config config = new DbusEventBuffer.Config(); config.setMaxSize(maxEventBufferSize); config.setMaxIndividualBufferSize(maxIndividualBufferSize); config.setScnIndexSize(maxIndexSize); config.setAverageEventSize(maxReadBufferSize); config.setAllocationPolicy(allocationPolicy.name()); config.setRestoreMMappedBuffers(restoreMMappedBuffers); config.setMmapDirectory(mmapDirectory); //config.setQueuePolicy(policy.toString()); //config.setAssertLevel(null != assertLevel ? assertLevel.toString(): AssertLevel.NONE.toString()); config.setAssertLevel(AssertLevel.NONE.toString()); return config.build(); } enum EventType { START, END, REG }; private void checkEventsInBuffer(DbusEventBufferReflector bufferReflector, int expCount) { Assert.assertTrue(bufferReflector.validateBuffer()); DbusEventBuffer buf = bufferReflector.getDbusEventBuffer(); DbusEventBuffer.DbusEventIterator iter = buf.acquireIterator("myIter1"); Assert.assertNotNull(iter); int count = 0; while (iter.hasNext()) { DbusEvent e = iter.next(); Assert.assertNotNull(e); System.out.println("seq=" + e.sequence() + "sys:" + e.isEndOfPeriodMarker()); count++; } iter.close(); Assert.assertEquals("doesn't match expected number of events in the buffer", expCount, count); } private void addOneEvent(DbusEvent ev, DbusEventBuffer buf, EventType eventType) { if (eventType.equals(EventType.START)) buf.startEvents(); byte[] payload = new byte[((DbusEventInternalReadable) ev).payloadLength()]; ev.value().get(payload); buf.appendEvent(new DbusEventKey(ev.key()), ev.physicalPartitionId(), ev.logicalPartitionId(), ev.timestampInNanos(), ev.srcId(), ev.schemaId(), payload, false, null); if (eventType.equals(EventType.END)) buf.endEvents(ev.sequence()); } private File initMmapDir(String path) throws Exception { File d = new File(path); if (d.exists()) { FileUtils.cleanDirectory(d); } else { d.mkdir(); } d.deleteOnExit(); return d; } }