Java tutorial
package com.linkedin.databus.core; /* * * Copyright 2013 LinkedIn Corp. All rights reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * */ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.commons.io.IOUtils; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.codehaus.jackson.JsonParseException; import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; import com.linkedin.databus.core.DbusEventBufferMult.PhysicalPartitionKey; import com.linkedin.databus.core.data_model.DatabusSubscription; import com.linkedin.databus.core.data_model.LogicalPartition; import com.linkedin.databus.core.data_model.LogicalSource; import com.linkedin.databus.core.data_model.LogicalSourceId; import com.linkedin.databus.core.data_model.PhysicalPartition; import com.linkedin.databus.core.data_model.PhysicalSource; import com.linkedin.databus.core.monitoring.mbean.AggregatedDbusEventsStatisticsCollector; import com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector; import com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats; import com.linkedin.databus.core.monitoring.mbean.StatsCollectors; import com.linkedin.databus.core.util.InvalidConfigException; import com.linkedin.databus2.core.BufferNotFoundException; import com.linkedin.databus2.core.DatabusException; import com.linkedin.databus2.core.filter.AllowAllDbusFilter; import com.linkedin.databus2.core.filter.ConjunctionDbusFilter; import com.linkedin.databus2.core.filter.DbusFilter; import com.linkedin.databus2.core.filter.LogicalSourceAndPartitionDbusFilter; import com.linkedin.databus2.core.filter.PhysicalPartitionDbusFilter; import com.linkedin.databus2.core.filter.SourceDbusFilter; import com.linkedin.databus2.relay.config.PhysicalSourceConfig; import com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig; import com.linkedin.databus2.test.TestUtil; public class TestDbusEventBufferMult { public static final Logger LOG = Logger.getLogger(TestDbusEventBufferMult.class); /** Encapsulate test set so that multiple threads can run in parallel */ static class TestSetup { PhysicalSourceStaticConfig[] _physConfigs; DbusEventBufferMult _eventBuffer; TestSetup() throws JsonParseException, JsonMappingException, IOException, InvalidConfigException { ObjectMapper mapper = new ObjectMapper(); InputStreamReader isr = new InputStreamReader(IOUtils.toInputStream(_configSource1)); PhysicalSourceConfig pConfig1 = mapper.readValue(isr, PhysicalSourceConfig.class); isr.close(); isr = new InputStreamReader(IOUtils.toInputStream(_configSource2)); PhysicalSourceConfig pConfig2 = mapper.readValue(isr, PhysicalSourceConfig.class); PhysicalSourceStaticConfig pStatConf1 = pConfig1.build(); PhysicalSourceStaticConfig pStatConf2 = pConfig2.build(); _physConfigs = new PhysicalSourceStaticConfig[] { pStatConf1, pStatConf2 }; DbusEventBuffer.Config conf = new DbusEventBuffer.Config(); conf.setAllocationPolicy("MMAPPED_MEMORY"); DbusEventBuffer.StaticConfig config = conf.build(); _eventBuffer = new DbusEventBufferMult(_physConfigs, config, new DbusEventV2Factory()); for (DbusEventBuffer b : _eventBuffer.bufIterable()) { b.start(1); } } } DbusEventBuffer.StaticConfig _config; PhysicalSourceStaticConfig[] _pConfigs; DbusEventBufferMult _eventBufferMult; TestDbusEvent[] _events; PhysicalSource _phSource; PhysicalPartition _phPartition; ObjectMapper _mapper; static final String _configSource1 = "{\n" + " \"name\" : \"multBufferTest1\",\n" + " \"id\" : 100,\n" + " \"uri\" : \"uri1\",\n" + " \"slowSourceQueryThreshold\" : 2000,\n" + " \"sources\" :\n" + " [\n" + " {\"id\" : 1, \n" + " \"name\" : \"srcName1\",\n" + " \"uri\": \"member2.member_profile\", \n" + " \"partitionFunction\" : \"constant:1\", \n" + " \"partition\" : 0 \n" + " },\n" + " {\"id\" : 2, \n" + " \"name\" : \"srcName2\",\n" + " \"uri\" : \"member2.member_account\", \n" + " \"partitionFunction\" : \"constant:1\", \n" + " \"partition\" : 0 \n" + " },\n" + " {\"id\" : 2, \n" + " \"name\" : \"srcName2\",\n" + " \"uri\" : \"member2.member_account\", \n" + " \"partitionFunction\" : \"constant:1\", \n" + " \"partition\" : 1 \n" + " }\n" + " ]\n" + "}"; static final String _configSource2 = "{\n" + " \"name\" : \"multBufferTest2\",\n" + " \"id\" : 101,\n" + " \"uri\" : \"uri2\",\n" + " \"slowSourceQueryThreshold\" : 2000,\n" + " \"sources\" :\n" + " [\n" + " {\"id\" : 11, \n" + " \"name\" : \"srcName11\",\n" + " \"uri\": \"member2.member_profile\", \n" + " \"partitionFunction\" : \"constant:1\",\n" + " \"partition\" : 0 \n" + " },\n" + " {\"id\" : 12, \n" + " \"name\" : \"srcName12\",\n" + " \"uri\" : \"member2.member_account\", \n" + " \"partitionFunction\" : \"constant:1\",\n" + " \"partition\" : 0 \n" + " },\n" + " {\"id\" : 12, \n" + " \"name\" : \"srcName12\",\n" + " \"uri\" : \"member2.member_account\", \n" + " \"partitionFunction\" : \"constant:1\"\n" + " },\n" + " {\"id\" : 2, \n" + " \"name\" : \"srcName2\",\n" + " \"uri\" : \"member2.member_account\", \n" + " \"partitionFunction\" : \"constant:1\", \n" + " \"partition\" : 2 \n" + " }\n" + " ]\n" + "}"; static final String _configSource3 = "{\n" + " \"name\" : \"multBufferTest3\",\n" + " \"id\" : 102,\n" + " \"uri\" : \"uri3\",\n" + " \"slowSourceQueryThreshold\" : 2000,\n" + " \"sources\" :\n" + " [\n" + " {\"id\" : 21, \n" + " \"name\" : \"srcName21\",\n" + " \"uri\": \"member2.member_profile\", \n" + " \"partitionFunction\" : \"constant:1\", \n" + " \"partition\" : 0 \n" + " },\n" + " {\"id\" : 22, \n" + " \"name\" : \"srcName22\",\n" + " \"uri\" : \"member2.member_account\", \n" + " \"partitionFunction\" : \"constant:1\", \n" + " \"partition\" : 0 \n" + " },\n" + " {\"id\" : 23, \n" + " \"name\" : \"srcName23\",\n" + " \"uri\" : \"member2.member_account\", \n" + " \"partitionFunction\" : \"constant:1\", \n" + " \"partition\" : 0 \n" + " }\n" + " ]\n" + "}"; public PhysicalSourceConfig convertToPhysicalSourceConfig(String str) { _mapper = new ObjectMapper(); InputStreamReader isr = new InputStreamReader(IOUtils.toInputStream(str)); PhysicalSourceConfig pConfig = null; try { pConfig = _mapper.readValue(isr, PhysicalSourceConfig.class); } catch (JsonParseException e) { fail("Failed parsing", e); } catch (JsonMappingException e) { fail("Failed parsing", e); } catch (IOException e) { fail("Failed parsing", e); } try { isr.close(); } catch (IOException e) { fail("Failed", e); } return pConfig; } @BeforeTest public void setUpTest() throws IOException, InvalidConfigException { LOG.info("Setting up Test"); PhysicalSourceStaticConfig pStatConf1 = convertToPhysicalSourceConfig(_configSource1).build(); PhysicalSourceStaticConfig pStatConf2 = convertToPhysicalSourceConfig(_configSource2).build(); PhysicalSourceStaticConfig pStatConf3 = convertToPhysicalSourceConfig(_configSource3).build(); _pConfigs = new PhysicalSourceStaticConfig[] { pStatConf1, pStatConf2, pStatConf3 }; // generate testData int scn = 100; String srcName = "srcName"; int srcId = 1; PhysicalSource pS = pStatConf1.getPhysicalSource(); PhysicalPartition pP = pStatConf1.getPhysicalPartition(); _events = new TestDbusEvent[20]; LogicalPartition lP = new LogicalPartition((short) 0); for (int i = 0; i < _events.length; i++) { _events[i] = new TestDbusEvent(i, scn, new LogicalSource(srcId, srcName + srcId), pS, pP, lP); switch (i) { case 4: srcId = 2; break; case 9: srcId = 11; pS = pStatConf2.getPhysicalSource(); pP = pStatConf2.getPhysicalPartition(); break; case 14: srcId = 12; break; } if ((i & 1) == 1) scn++; } ; } private static class TestDbusEvent { public TestDbusEvent(int k, int scn, LogicalSource lS, PhysicalSource pS, PhysicalPartition pP, LogicalPartition lP) { _key = k; _scn = scn; _lSource = lS; _lPartition = lP; _phPartition = pP; _phSource = pS; } public int _key; public int _scn; public LogicalSource _lSource; public PhysicalPartition _phPartition; public PhysicalSource _phSource; public LogicalPartition _lPartition; } // create the bufs - we want a new set for every test private void createBufMult() throws InvalidConfigException { if (_config == null) { try { DbusEventBuffer.Config cfgBuilder = new DbusEventBuffer.Config(); LOG.info("max size = " + cfgBuilder.getMaxSize()); LOG.info("scnIndex size = " + cfgBuilder.getScnIndexSize()); cfgBuilder.setMaxSize(10 * 1024 * 1024); cfgBuilder.setScnIndexSize(2 * 1024 * 1024); cfgBuilder.setAllocationPolicy("MMAPPED_MEMORY"); _config = cfgBuilder.build(); //_config = new DbusEventBuffer.Config().build(); } catch (InvalidConfigException e) { fail("invalid configuration", e); } } _eventBufferMult = new DbusEventBufferMult(_pConfigs, _config, new DbusEventV2Factory()); for (DbusEventBuffer b : _eventBufferMult.bufIterable()) { b.start(1); } } private void addEvents() { byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset()); for (int i = 0; i < _events.length; i++) { TestDbusEvent e = _events[i]; DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(e._phPartition); //System.out.println("i= " + i + "; phS = " + e._phSource + "; phP = " + e._phPartition + // "; buf=" + buf.hashCode()); // two events per scn if (i % 2 == 0) buf.startEvents(); DbusEventKey key = new DbusEventKey(e._key); //short lPartitionId = (short) (key.getLongKey() % Short.MAX_VALUE); String value = "" + i; short srcId = e._lSource.getId().shortValue(); LogicalPartition lPart = e._lPartition; //System.out.print("appending events for i=" + i + "; lSource=" + srcId); short pPartitionId = _eventBufferMult.getPhysicalPartition(srcId, lPart).getId().shortValue(); //System.out.println(";partitionid=" + pPartitionId); assertTrue(buf.appendEvent(key, pPartitionId, lPart.getId(), System.currentTimeMillis(), srcId, schema, value.getBytes(Charset.defaultCharset()), false, null)); if (i % 2 != 0) buf.endEvents(e._scn, null); } } @Test public void verifyMultBuffer() throws InvalidConfigException { createBufMult(); assertEquals(_eventBufferMult.bufsNum(), 3); } @Test // reading from the source that belong to the same physical buffer public void verifyReadingSingleBuf() throws IOException, ScnNotFoundException, DatabusException, OffsetNotFoundException { createBufMult(); addEvents(); Set<Integer> srcIds = new HashSet<Integer>(2); srcIds.add(1); srcIds.add(2); // total expected events 15 = 10 original (from first buffer (src 1 and 2) + 5 control events batchReading(srcIds, 10); } @Test //reading from the source that belong to two different physical buffers public void verifyReadingMultBuf() throws IOException, ScnNotFoundException, InvalidConfigException, DatabusException, OffsetNotFoundException { verifyReadingMultBuf(2000); verifyReadingMultBuf(100); // Pretty much only one event can fit at a time. } private void verifyReadingMultBuf(final int batchFetchSize) throws IOException, ScnNotFoundException, InvalidConfigException, DatabusException, OffsetNotFoundException { createBufMult(); addEvents(); Set<Integer> srcIds = new HashSet<Integer>(2); srcIds.add(1); srcIds.add(12); // total expected events 20 = 20 original events - 10 filtered out (srcs 2 and 11) batchReading(srcIds, 10, batchFetchSize); } @Test public void verifyReadingLogicalPartitionWildcard() throws IOException, ScnNotFoundException, InvalidConfigException, DatabusException, OffsetNotFoundException { createBufMult(); PhysicalSourceStaticConfig pStatConf1 = convertToPhysicalSourceConfig(_configSource1).build(); PhysicalSourceStaticConfig pStatConf2 = convertToPhysicalSourceConfig(_configSource2).build(); PhysicalPartition pP = pStatConf1.getPhysicalPartition(); DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(pP); buf.startEvents(); byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset()); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 1, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); buf.endEvents(100, null); pP = pStatConf2.getPhysicalPartition(); buf = _eventBufferMult.getDbusEventBufferAppendable(pP); buf.startEvents(); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); buf.endEvents(200, null); Set<Integer> srcIds = new HashSet<Integer>(1); srcIds.add(2); // total expected events 3 - for pp100:lp=0,pp100:lp=1;pp101:lp=2 batchReading(srcIds, 3); } private StatsCollectors<DbusEventsStatisticsCollector> createStats(String[] pNames) { DbusEventsStatisticsCollector stats = new AggregatedDbusEventsStatisticsCollector(1, "test", true, true, null); StatsCollectors<DbusEventsStatisticsCollector> statsColl = new StatsCollectors<DbusEventsStatisticsCollector>( stats); int count = 1; for (String pname : pNames) { DbusEventsStatisticsCollector s = new DbusEventsStatisticsCollector(1, "test" + count, true, true, null); statsColl.addStatsCollector(pname, s); ++count; } return statsColl; } @Test public void testMultiPPartionStreamStats() throws Exception { createBufMult(); PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition(), _pConfigs[1].getPhysicalPartition(), _pConfigs[2].getPhysicalPartition() }; //generate a bunch of windows for 3 partitions int windowsNum = 10; for (int i = 1; i <= windowsNum; ++i) { DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]); buf.startEvents(); byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset()); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); buf.endEvents(100 * i, null); buf = _eventBufferMult.getDbusEventBufferAppendable(p[1]); buf.startEvents(); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); buf.endEvents(100 * i + 1, null); buf = _eventBufferMult.getDbusEventBufferAppendable(p[2]); buf.startEvents(); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); assertTrue(buf.appendEvent(new DbusEventKey(3), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); buf.endEvents(100 * i + 2, null); } String[] pnames = new String[p.length]; int count = 0; for (PhysicalPartition ip : p) { pnames[count++] = ip.toSimpleString(); } StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames); PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]), new PhysicalPartitionKey(p[1]), new PhysicalPartitionKey(p[2]) }; CheckpointMult cpMult = new CheckpointMult(); for (int i = 0; i < 3; ++i) { Checkpoint cp = new Checkpoint(); cp.setFlexible(); cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION); cpMult.addCheckpoint(p[i], cp); } DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl); ByteArrayOutputStream baos = new ByteArrayOutputStream(); WritableByteChannel writeChannel = Channels.newChannel(baos); reader.streamEvents(false, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter()); writeChannel.close(); baos.close(); //make sure we got the physical partition names right List<String> ppartNames = statsColl.getStatsCollectorKeys(); assertEquals(ppartNames.size(), 3); HashSet<String> expectedPPartNames = new HashSet<String>( Arrays.asList(p[0].toSimpleString(), p[1].toSimpleString(), p[2].toSimpleString())); for (String ppartName : ppartNames) { assertTrue(expectedPPartNames.contains(ppartName)); } //verify event counts per partition DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[1].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[2].toSimpleString()).getTotalStats() }; assertEquals(ppartStats[0].getNumDataEvents(), windowsNum); assertEquals(ppartStats[1].getNumDataEvents(), windowsNum * 2); assertEquals(ppartStats[2].getNumDataEvents(), windowsNum * 3); assertEquals(ppartStats[0].getNumSysEvents(), windowsNum); assertEquals(ppartStats[1].getNumSysEvents(), windowsNum); assertEquals(ppartStats[2].getNumSysEvents(), windowsNum); assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), windowsNum * (1 + 2 + 3)); assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), windowsNum * 3); assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), Math .max(ppartStats[0].getTimeLag(), Math.max(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag()))); assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), Math .min(ppartStats[0].getTimeLag(), Math.min(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag()))); } @Test public void testSinglePPartionStreamFromLatest() throws Exception { createBufMult(); PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition() }; //generate a bunch of windows for 3 partitions int windowsNum = 10; for (int i = 1; i <= windowsNum; ++i) { DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]); buf.startEvents(); byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset()); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); buf.endEvents(100 * i, null); } String[] pnames = new String[p.length]; int count = 0; for (PhysicalPartition ip : p) { pnames[count++] = ip.toSimpleString(); } StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames); PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]) }; CheckpointMult cpMult = new CheckpointMult(); Checkpoint cp = new Checkpoint(); cp.setFlexible(); cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION); cpMult.addCheckpoint(p[0], cp); DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl); ByteArrayOutputStream baos = new ByteArrayOutputStream(); WritableByteChannel writeChannel = Channels.newChannel(baos); // Set streamFromLatestScn == true reader.streamEvents(true, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter()); writeChannel.close(); baos.close(); //make sure we got the physical partition names right List<String> ppartNames = statsColl.getStatsCollectorKeys(); assertEquals(ppartNames.size(), 1); HashSet<String> expectedPPartNames = new HashSet<String>(Arrays.asList(p[0].toSimpleString())); for (String ppartName : ppartNames) { assertTrue(expectedPPartNames.contains(ppartName)); } //verify event counts per partition DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats() }; // Only the last window is returned in each of the partitions assertEquals(ppartStats[0].getNumDataEvents(), 1); assertEquals(ppartStats[0].getNumSysEvents(), 1); assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), (1)); assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), (1)); assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), ppartStats[0].getTimeLag()); assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), ppartStats[0].getTimeLag()); } @Test public void testMultiPPartionStreamFromLatest() throws Exception { createBufMult(); PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition(), _pConfigs[1].getPhysicalPartition(), _pConfigs[2].getPhysicalPartition() }; //generate a bunch of windows for 3 partitions int windowsNum = 10; for (int i = 1; i <= windowsNum; ++i) { DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]); buf.startEvents(); byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset()); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); buf.endEvents(100 * i, null); buf = _eventBufferMult.getDbusEventBufferAppendable(p[1]); buf.startEvents(); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); buf.endEvents(100 * i + 1, null); buf = _eventBufferMult.getDbusEventBufferAppendable(p[2]); buf.startEvents(); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); assertTrue(buf.appendEvent(new DbusEventKey(3), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); buf.endEvents(100 * i + 2, null); } String[] pnames = new String[p.length]; int count = 0; for (PhysicalPartition ip : p) { pnames[count++] = ip.toSimpleString(); } StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames); PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]), new PhysicalPartitionKey(p[1]), new PhysicalPartitionKey(p[2]) }; CheckpointMult cpMult = new CheckpointMult(); for (int i = 0; i < 3; ++i) { Checkpoint cp = new Checkpoint(); cp.setFlexible(); cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION); cpMult.addCheckpoint(p[i], cp); } DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl); ByteArrayOutputStream baos = new ByteArrayOutputStream(); WritableByteChannel writeChannel = Channels.newChannel(baos); // Set streamFromLatestScn == true reader.streamEvents(true, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter()); writeChannel.close(); baos.close(); //make sure we got the physical partition names right List<String> ppartNames = statsColl.getStatsCollectorKeys(); assertEquals(ppartNames.size(), 3); HashSet<String> expectedPPartNames = new HashSet<String>( Arrays.asList(p[0].toSimpleString(), p[1].toSimpleString(), p[2].toSimpleString())); for (String ppartName : ppartNames) { assertTrue(expectedPPartNames.contains(ppartName)); } //verify event counts per partition DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[1].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[2].toSimpleString()).getTotalStats() }; // Only the last window is returned in each of the partitions assertEquals(ppartStats[0].getNumDataEvents(), 1); assertEquals(ppartStats[1].getNumDataEvents(), 2); assertEquals(ppartStats[2].getNumDataEvents(), 3); assertEquals(ppartStats[0].getNumSysEvents(), 1); assertEquals(ppartStats[1].getNumSysEvents(), 1); assertEquals(ppartStats[2].getNumSysEvents(), 1); assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), (1 + 2 + 3)); assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), (1 + 1 + 1)); assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), Math .max(ppartStats[0].getTimeLag(), Math.max(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag()))); assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), Math .min(ppartStats[0].getTimeLag(), Math.min(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag()))); } private void batchReading(Set<Integer> srcIds, final int expectEvents) throws IOException, ScnNotFoundException, DatabusException, OffsetNotFoundException { batchReading(srcIds, expectEvents, 2000); } private void batchReading(Set<Integer> srcIds, final int expectEvents, final int batchFetchSize) throws IOException, ScnNotFoundException, DatabusException, OffsetNotFoundException { // for debug generate this line String inputSrcIds = Arrays.toString(srcIds.toArray()); LOG.info("Reading events from " + inputSrcIds); // create checkpoints CheckpointMult cpMult = new CheckpointMult(); for (PhysicalSourceStaticConfig pConf : _pConfigs) { PhysicalPartition pPart = pConf.getPhysicalPartition(); Checkpoint cp; if (cpMult.getCheckpoint(pPart) == null) { // needs a new checkpoint cp = new Checkpoint(); cp.setFlexible(); cpMult.addCheckpoint(pPart, cp); } } DbusEventBufferBatchReadable read = _eventBufferMult.getDbusEventBufferBatchReadable(srcIds, cpMult, null); int totalRead = 0; int numEventsRead = Integer.MAX_VALUE; int numNonControlEventsRead = 0; int maxIterNum = 100, iterCount = 0; String prevEvent = ""; while (numEventsRead > 0) { if (iterCount++ > maxIterNum) { fail("Checkpoint doesn't work - it is a never-ending loop"); } ByteArrayOutputStream jsonOut = new ByteArrayOutputStream(); WritableByteChannel jsonOutChannel = Channels.newChannel(jsonOut); numEventsRead = read.streamEvents(false, batchFetchSize, jsonOutChannel, Encoding.JSON_PLAIN_VALUE, new SourceDbusFilter(srcIds)).getNumEventsStreamed(); totalRead += numEventsRead; LOG.info("read for " + inputSrcIds + ": " + numEventsRead + " events"); byte[] jsonBytes = jsonOut.toByteArray(); if (jsonBytes.length == 0) break; // nothing more to read String jsonString = new String(jsonBytes); String[] jsonStrings = jsonString.split("\n"); assertEquals(jsonStrings.length, numEventsRead); ObjectMapper mapper = new ObjectMapper(); for (int i = 0; i < jsonStrings.length; i++) { // verify what was written String evtStr = jsonStrings[i]; if (evtStr.equals(prevEvent)) { // It may so happen that we receive the same event twice, especially when the // offered buffer is small. This check gets around the issue. continue; } prevEvent = evtStr; Map<String, Object> jsonMap = mapper.readValue(evtStr, new TypeReference<Map<String, Object>>() { }); //assertEquals(jsonMap.size(), 10); Integer srcId = (Integer) jsonMap.get("srcId"); if (!DbusEventUtils.isControlSrcId(srcId)) { // not a control message numNonControlEventsRead++; Integer physicalPartitionId = (Integer) jsonMap.get("physicalPartitionId"); Integer logicalPartitionId = (Integer) jsonMap.get("logicalPartitionId"); PhysicalPartition pPartition = _eventBufferMult.getPhysicalPartition(srcId, new LogicalPartition(logicalPartitionId.shortValue())); LOG.info("EVENT: " + jsonMap.toString()); assertTrue(srcIds.contains(srcId), "src id " + srcId + " doesn't match to " + inputSrcIds); assertEquals(physicalPartitionId, pPartition.getId(), "physical partition id didn't match"); } else { LOG.info("Control event: " + jsonMap.toString()); } } } assertTrue(totalRead > numNonControlEventsRead); assertEquals(numNonControlEventsRead, expectEvents); } @BeforeClass public void setupClass() { TestUtil.setupLoggingWithTimestampedFile(true, "/tmp/TestDbusEventBufferMult_", ".log", Level.INFO); } @Test public void testGetBuffer() throws InvalidConfigException { // get buffer by physical partition createBufMult(); addEvents(); Integer pPartitionId = 100; // get buffer by partition id PhysicalPartition pPartition = new PhysicalPartition(pPartitionId, "multBufferTest1"); DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(pPartition); assertNotNull(buf, "cannot get by pPartition" + pPartition); // get buffer by physical partition buf = _eventBufferMult.getDbusEventBufferAppendable(pPartition); assertNotNull(buf, "cannot get by pPartition" + pPartition); // get buffer by logical source id int lSrcId = 1; buf = _eventBufferMult.getDbusEventBufferAppendable(lSrcId); assertNotNull(buf, "cannot get by lSrcId" + lSrcId); // logical source 2 , default partition - SAME BUFFER lSrcId = 2; LogicalSource lSource = new LogicalSource(lSrcId, "srcName2"); // correct source name, default partition 0 DbusEventBufferAppendable buf1 = _eventBufferMult.getDbusEventBufferAppendable(lSource); assertNotNull(buf1, "cannot get buffer by lSource " + lSource); assertTrue(buf1 == buf, "buf and buf1 should be the same buffer"); // logical source, different logical partition 1 - SAME BUFFER LogicalPartition lPartition = new LogicalPartition((short) 1); buf1 = _eventBufferMult.getDbusEventBufferAppendable(lSource, lPartition); // should be the same buffer assertNotNull(buf1, "cannot get buffer by lSource " + lSource + ";lPartition =" + lPartition); assertTrue(buf1 == buf, "buf and buf1 should be the same buffer"); // logical source, different logical partition 2 - DIFFERENT BUFFER lPartition = new LogicalPartition((short) 2); buf1 = _eventBufferMult.getDbusEventBufferAppendable(lSource, lPartition); assertNotNull(buf1, "cannot get buffer by lSource " + lSource + ";lPartition =" + lPartition); assertTrue(buf1 != buf, "buf and buf1 should not be the same buffer"); // logical source, different logical partition 12 - DIFFERENT BUFFER (same as for lsr=2,lp=2) DbusEventBufferAppendable buf2 = _eventBufferMult.getDbusEventBufferAppendable(12); assertNotNull(buf2, "cannot get buffer by lSourceid " + 12); assertTrue(buf2 != buf, "buf and buf2 should not be the same buffer"); assertTrue(buf1 == buf2, "buf2 and buf1 should be the same buffer"); } @Test public void testReset() throws Exception { createBufMult(); addEvents(); Integer pPartitionId = 100; PhysicalPartition pPartition = new PhysicalPartition(pPartitionId, "multBufferTest1"); _eventBufferMult.resetBuffer(pPartition, 108L); DbusEventBufferAppendable eventBuffer = _eventBufferMult.getDbusEventBufferAppendable(pPartition); Assert.assertTrue(eventBuffer.empty()); pPartition = new PhysicalPartition(pPartitionId, "unknownBuffer"); boolean caughtException = false; try { _eventBufferMult.resetBuffer(pPartition, -1L); } catch (BufferNotFoundException e) { caughtException = true; } assertTrue(caughtException); } @Test public void testConstructFilters() throws Exception { TestSetup t = new TestSetup(); //test single Physical Partition subscription DatabusSubscription sub1 = DatabusSubscription .createPhysicalPartitionReplicationSubscription(new PhysicalPartition(100, "multBufferTest1")); DbusFilter filter1 = t._eventBuffer.constructFilters(Arrays.asList(sub1)); assertNotNull(filter1); assertTrue(filter1 instanceof PhysicalPartitionDbusFilter); PhysicalPartitionDbusFilter ppfilter1 = (PhysicalPartitionDbusFilter) filter1; assertEquals(ppfilter1.getPhysicalPartition(), new PhysicalPartition(100, "multBufferTest1")); assertNull(ppfilter1.getNestedFilter()); DatabusSubscription sub2 = DatabusSubscription .createPhysicalPartitionReplicationSubscription(new PhysicalPartition(101, "multBufferTest2")); //test two Physical Partition subscriptions DbusFilter filter2 = t._eventBuffer.constructFilters(Arrays.asList(sub1, sub2)); assertNotNull(filter2); assertTrue(filter2 instanceof ConjunctionDbusFilter); ConjunctionDbusFilter conjFilter2 = (ConjunctionDbusFilter) filter2; boolean hasPP100 = false; boolean hasPP101 = false; assertEquals(conjFilter2.getFilterList().size(), 2); for (DbusFilter f : conjFilter2.getFilterList()) { assertTrue(f instanceof PhysicalPartitionDbusFilter); PhysicalPartitionDbusFilter ppf = (PhysicalPartitionDbusFilter) f; if (ppf.getPhysicalPartition().getId() == 100) hasPP100 = true; else if (ppf.getPhysicalPartition().getId() == 101) hasPP101 = true; else fail("unknown physical partition filter:" + ppf.getPhysicalPartition()); } assertTrue(hasPP100); assertTrue(hasPP101); //test a subcription with a logical source DatabusSubscription sub3 = DatabusSubscription .createSimpleSourceSubscription(new LogicalSource(2, "srcName2")); DbusFilter filter3 = t._eventBuffer.constructFilters(Arrays.asList(sub3)); assertNotNull(filter3); assertTrue(filter3 instanceof PhysicalPartitionDbusFilter); PhysicalPartitionDbusFilter ppfilter3 = (PhysicalPartitionDbusFilter) filter3; assertEquals(ppfilter3.getPhysicalPartition(), PhysicalPartition.ANY_PHYSICAL_PARTITION); DbusFilter ppfilter3_child = ppfilter3.getNestedFilter(); assertNotNull(ppfilter3_child); assertTrue(ppfilter3_child instanceof LogicalSourceAndPartitionDbusFilter); LogicalSourceAndPartitionDbusFilter lsourceFilter3 = (LogicalSourceAndPartitionDbusFilter) ppfilter3_child; LogicalSourceAndPartitionDbusFilter.LogicalPartitionDbusFilter lpartFilter3_1 = lsourceFilter3 .getSourceFilter(2); assertNotNull(lpartFilter3_1); assertTrue(lpartFilter3_1.isAllPartitionsWildcard()); //test a subcription with a physical and logical partition DatabusSubscription sub4 = new DatabusSubscription(PhysicalSource.MASTER_PHISYCAL_SOURCE, new PhysicalPartition(101, "multBufferTest2"), new LogicalSourceId(new LogicalSource(2, "srcName2"), (short) 2)); DbusFilter filter4 = t._eventBuffer.constructFilters(Arrays.asList(sub4)); assertNotNull(filter4); assertTrue(filter4 instanceof PhysicalPartitionDbusFilter); PhysicalPartitionDbusFilter ppfilter4 = (PhysicalPartitionDbusFilter) filter4; assertEquals(ppfilter4.getPhysicalPartition(), new PhysicalPartition(101, "multBufferTest2")); DbusFilter ppfilter4_child = ppfilter4.getNestedFilter(); assertNotNull(ppfilter4_child); assertTrue(ppfilter4_child instanceof LogicalSourceAndPartitionDbusFilter); LogicalSourceAndPartitionDbusFilter lsourceFilter4 = (LogicalSourceAndPartitionDbusFilter) ppfilter4_child; LogicalSourceAndPartitionDbusFilter.LogicalPartitionDbusFilter lpartFilter4_1 = lsourceFilter4 .getSourceFilter(2); assertNotNull(lpartFilter4_1); assertTrue(lpartFilter4_1.getPartitionsMask().contains(2)); } @Test public void testSubscriptionStream() throws Exception { final Logger log = Logger.getLogger("TestDbusEventBufferMult.testSubscriptionStream"); log.info("start"); TestSetup t = new TestSetup(); PhysicalPartition pp100 = new PhysicalPartition(100, "multBufferTest1"); PhysicalPartitionKey pk1 = new PhysicalPartitionKey(pp100); PhysicalPartition pp101 = new PhysicalPartition(101, "multBufferTest2"); PhysicalPartitionKey pk2 = new PhysicalPartitionKey(pp101); //generate events in pp100 byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset()); DbusEventBufferAppendable buf100 = t._eventBuffer.getDbusEventBufferAppendable(pp100); buf100.startEvents(); assertTrue(buf100.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 1, schema, new byte[100], false, null)); assertTrue(buf100.appendEvent(new DbusEventKey(10), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 1, schema, new byte[100], false, null)); assertTrue(buf100.appendEvent(new DbusEventKey(11), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 1, schema, new byte[100], false, null)); assertTrue(buf100.appendEvent(new DbusEventKey(2), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); buf100.endEvents(100, null); buf100.startEvents(); assertTrue(buf100.appendEvent(new DbusEventKey(3), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); assertTrue(buf100.appendEvent(new DbusEventKey(4), (short) 100, (short) 1, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); buf100.endEvents(200, null); //generate events in pp100 DbusEventBufferAppendable buf101 = t._eventBuffer.getDbusEventBufferAppendable(pp101); buf101.startEvents(); assertTrue(buf101.appendEvent(new DbusEventKey(51), (short) 101, (short) 0, System.currentTimeMillis() * 1000000, (short) 11, schema, new byte[100], false, null)); assertTrue(buf101.appendEvent(new DbusEventKey(52), (short) 101, (short) 0, System.currentTimeMillis() * 1000000, (short) 12, schema, new byte[100], false, null)); assertTrue(buf101.appendEvent(new DbusEventKey(53), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); buf101.endEvents(120, null); buf101.startEvents(); assertTrue(buf101.appendEvent(new DbusEventKey(54), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); assertTrue(buf101.appendEvent(new DbusEventKey(55), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); assertTrue(buf101.appendEvent(new DbusEventKey(56), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); buf101.endEvents(200, null); //initialization DatabusSubscription sub1 = DatabusSubscription .createPhysicalPartitionReplicationSubscription(new PhysicalPartition(100, "multBufferTest1")); DbusFilter filter1 = t._eventBuffer.constructFilters(Arrays.asList(sub1)); assertNotNull(filter1); CheckpointMult cpMult1 = new CheckpointMult(); Checkpoint cp100 = new Checkpoint(); cp100.init(); cp100.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION); cp100.setWindowScn(10L); cp100.setWindowOffset(-1); cpMult1.addCheckpoint(pp100, cp100); String[] pnames = { "multBufferTest1:100", "multBufferTest2:101" }; StatsCollectors<DbusEventsStatisticsCollector> statsColls1 = createStats(pnames); DbusEventsStatisticsCollector statsCol1 = statsColls1.getStatsCollector("multBufferTest1:100"); DbusEventsStatisticsCollector statsCol2 = statsColls1.getStatsCollector("multBufferTest2:101"); //read an entire buffer DbusEventBufferBatchReadable reader1 = t._eventBuffer.getDbusEventBufferBatchReadable(cpMult1, Arrays.asList(pk1), statsColls1); ByteArrayOutputStream baos = new ByteArrayOutputStream(); // Try a call with 20 bytes of fetch size, we should see the event size in the first return with 0 events read. StreamEventsResult result = reader1.streamEvents(false, 20, Channels.newChannel(baos), Encoding.BINARY, filter1); assertEquals(0, result.getNumEventsStreamed()); assertEquals(161, result.getSizeOfPendingEvent()); result = reader1.streamEvents(false, 1000000, Channels.newChannel(baos), Encoding.BINARY, filter1); int eventsRead = result.getNumEventsStreamed(); assertEquals(eventsRead, 8); //4 events + 1 eop + 2 events + 1 eop assertEquals(statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumSysEvents(), 2); assertEquals(statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumDataEvents(), 6); assertEquals(result.getSizeOfPendingEvent(), 0, "Size of pending event not zero"); // Now that we have read all the events, we should not see a pending event even if we offer a small fetch size. result = reader1.streamEvents(false, 20, Channels.newChannel(baos), Encoding.BINARY, filter1); assertEquals(0, result.getNumEventsStreamed(), "There should be no more events in the buffer now"); assertEquals(0, result.getSizeOfPendingEvent(), "We should not see pending event size since there are no events in buffer"); baos.reset(); statsCol1.reset(); statsCol2.reset(); //read from two buffers, filtering out one cpMult1 = new CheckpointMult(); cp100.init(); cp100.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION); cp100.setWindowScn(10L); cp100.setWindowOffset(-1); cpMult1.addCheckpoint(pp100, cp100); reader1 = t._eventBuffer.getDbusEventBufferBatchReadable(cpMult1, Arrays.asList(pk1, pk2), statsColls1); eventsRead = reader1.streamEvents(false, 1000000, Channels.newChannel(baos), Encoding.BINARY, filter1) .getNumEventsStreamed(); assertEquals(eventsRead, 10); //4 events + 1 eop + 1 eop from the other buffer + 2 events + //1 eop + 1 eop from the other buffer assertEquals(statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumSysEvents(), 2); assertEquals(statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumDataEvents(), 6); baos.reset(); statsCol1.reset(); //read from one buffer and one source partition DatabusSubscription sub2 = new DatabusSubscription(PhysicalSource.MASTER_PHISYCAL_SOURCE, new PhysicalPartition(101, "multBufferTest2"), new LogicalSourceId(new LogicalSource(2, "srcName2"), (short) 2)); DbusFilter filter2 = t._eventBuffer.constructFilters(Arrays.asList(sub2)); assertNotNull(filter2); CheckpointMult cpMult2 = new CheckpointMult(); Checkpoint cp101 = new Checkpoint(); cp101.init(); cp101.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION); cp101.setWindowScn(10L); cp101.setWindowOffset(-1); cpMult2.addCheckpoint(pp101, cp101); DbusEventBufferBatchReadable reader2 = t._eventBuffer.getDbusEventBufferBatchReadable(cpMult2, Arrays.asList(pk2), statsColls1); eventsRead = reader2.streamEvents(false, 1000000, Channels.newChannel(baos), Encoding.BINARY, filter2) .getNumEventsStreamed(); assertEquals(eventsRead, 6); //1 events + 1 eop + 3events + 1 eop baos.reset(); statsCol1.reset(); statsCol2.reset(); //read all partitions for a source DatabusSubscription sub3 = new DatabusSubscription(PhysicalSource.MASTER_PHISYCAL_SOURCE, PhysicalPartition.ANY_PHYSICAL_PARTITION, LogicalSourceId.createAllPartitionsWildcard(new LogicalSource(2, "srcName2"))); DbusFilter filter3 = t._eventBuffer.constructFilters(Arrays.asList(sub3)); assertNotNull(filter3); CheckpointMult cpMult3 = new CheckpointMult(); cp100.init(); cp100.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION); cp100.setWindowScn(10L); cp100.setWindowOffset(-1); cpMult1.addCheckpoint(pp100, cp100); cp101.init(); cp101.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION); cp101.setWindowScn(10L); cp101.setWindowOffset(-1); cpMult2.addCheckpoint(pp101, cp101); DbusEventBufferBatchReadable reader3 = t._eventBuffer.getDbusEventBufferBatchReadable(cpMult3, Arrays.asList(pk1, pk2), statsColls1); eventsRead = reader3.streamEvents(false, 1000000, Channels.newChannel(baos), Encoding.BINARY, filter3) .getNumEventsStreamed(); assertEquals(eventsRead, 11); //1 events + 1 eop + 1 events + 1 eop + 2 events + 1 eop + 3 events + 1 eop assertEquals(statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumSysEvents(), 2); assertEquals(statsColls1.getStatsCollector("multBufferTest2:101").getTotalStats().getNumSysEvents(), 2); assertEquals( statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumDataEventsFiltered(), 3); assertEquals( statsColls1.getStatsCollector("multBufferTest2:101").getTotalStats().getNumDataEventsFiltered(), 4); baos.reset(); statsCol1.reset(); statsCol2.reset(); log.info("end"); } @Test public void addRemoveBuffers() throws IOException, ScnNotFoundException, InvalidConfigException, DatabusException, OffsetNotFoundException { createBufMult(); addEvents(); Set<Integer> srcIds = new HashSet<Integer>(2); srcIds.add(1); // total expected events from source 1 is 5 batchReading(srcIds, 5); PhysicalSourceStaticConfig pConfig = convertToPhysicalSourceConfig(_configSource3).build(); LOG.info("one more buffer for " + pConfig); DbusEventBuffer buf = _eventBufferMult.addNewBuffer(pConfig, _config); buf.start(100); // add events to the new buffer byte[] schema = "ABCDEFGHIJKLMNOP".getBytes(Charset.defaultCharset()); for (int i = 100; i < 110; i++) { if (i % 2 == 0) buf.startEvents(); assertTrue(buf.appendEvent(new DbusEventKey(i), pConfig.getPhysicalPartition().getId().shortValue(), (short) 0, // logical source id System.currentTimeMillis(), (short) (i < 105 ? 21 : 22), schema, ("" + i).getBytes(Charset.defaultCharset()), false, null)); if ((i & 1) == 1) buf.endEvents(i); } // read from one old and one new buffer srcIds.clear(); srcIds.add(12); srcIds.add(22); // total expected events 10 - 5 from source 12, and 5 from source 22 batchReading(srcIds, 10); // remove a buf pConfig = convertToPhysicalSourceConfig(_configSource2).build(); _eventBufferMult.removeBuffer(pConfig); _eventBufferMult.deallocateRemovedBuffers(true); srcIds.clear(); srcIds.add(11); boolean failed = false; try { batchReading(srcIds, 5); failed = false; // didn't fail } catch (java.lang.AssertionError ae) { // expected failed = true; } Assert.assertTrue(failed, "should've failed for src 11, since the buffer was removed" + pConfig); } @Test public void addRemoveBufferMapping() throws InvalidConfigException, DatabusException { createBufMult(); PhysicalSourceStaticConfig pConfig = convertToPhysicalSourceConfig(_configSource2).build(); PhysicalPartition pPart = _eventBufferMult.getPhysicalPartition(11); Assert.assertNotNull(pPart); DbusEventBuffer buf = _eventBufferMult.getDbusEventBuffer(pConfig.getSources()[0].getLogicalSource()); Assert.assertNotNull(buf); DbusEventBuffer buf1 = _eventBufferMult.getOneBuffer(pPart); Assert.assertNotNull(buf1); Assert.assertTrue(buf == buf1, "same references to the buffer"); // now remove the buffer and the mapping _eventBufferMult.removeBuffer(pConfig); _eventBufferMult.deallocateRemovedBuffers(true); // same should fail pPart = _eventBufferMult.getPhysicalPartition(11); Assert.assertNull(pPart); buf = _eventBufferMult.getDbusEventBuffer(pConfig.getSources()[0].getLogicalSource()); Assert.assertNull(buf); buf1 = _eventBufferMult.getOneBuffer(pPart); Assert.assertNull(buf1); } @Test public void testEnableStreamFromLatest() throws DatabusException, IOException { CheckpointMult cp = null; Collection<PhysicalPartitionKey> phyPartitions = new ArrayList<PhysicalPartitionKey>(); DbusEventBufferMult db = new DbusEventBufferMult(); DbusEventBufferMult.DbusEventBufferBatchReader dbr = db.new DbusEventBufferBatchReader(cp, phyPartitions, null); // Return true only once if streamFromLatest == true PhysicalPartitionKey pk = new PhysicalPartitionKey(); Set<PhysicalPartitionKey> sls = new HashSet<PhysicalPartitionKey>(); boolean streamFromLatestScn = true; // The set is initially empty - meaning none of the partitions have been served Assert.assertTrue(dbr.computeStreamFromLatestScnForPartition(pk, sls, streamFromLatestScn)); Assert.assertEquals(sls.size(), 1); Assert.assertFalse(dbr.computeStreamFromLatestScnForPartition(pk, sls, streamFromLatestScn)); sls.clear(); // Check null input Assert.assertFalse(dbr.computeStreamFromLatestScnForPartition(null, sls, streamFromLatestScn)); streamFromLatestScn = false; // The set is initially empty - meaning none of the partitions have been served Assert.assertFalse(dbr.computeStreamFromLatestScnForPartition(pk, sls, streamFromLatestScn)); Assert.assertEquals(sls.size(), 0); Assert.assertFalse(dbr.computeStreamFromLatestScnForPartition(pk, sls, streamFromLatestScn)); Assert.assertEquals(sls.size(), 0); // Check null input Assert.assertFalse(dbr.computeStreamFromLatestScnForPartition(null, sls, streamFromLatestScn)); return; } }