Java tutorial
package com.linkedin.databus.core; /* * * Copyright 2013 LinkedIn Corp. All rights reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * */ import static org.testng.AssertJUnit.assertTrue; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; import java.nio.channels.WritableByteChannel; import java.util.HashMap; import java.util.HashSet; import java.util.Vector; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.log4j.ConsoleAppender; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.log4j.PatternLayout; import org.codehaus.jackson.map.ObjectMapper; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import com.linkedin.databus.core.DbusEventBuffer.AllocationPolicy; import com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator; import com.linkedin.databus.core.DbusEventBufferMetaInfo.DbusEventBufferMetaInfoException; import com.linkedin.databus.core.test.DbusEventCorrupter; import com.linkedin.databus.core.test.DbusEventGenerator; import com.linkedin.databus.core.util.InvalidConfigException; import com.linkedin.databus2.core.AssertLevel; import com.linkedin.databus2.relay.config.PhysicalSourceConfig; import com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig; public class TestDbusEventBufferPersistence { public static final Logger LOG = Logger.getLogger(TestDbusEventBufferPersistence.class); static { PatternLayout defaultLayout = new PatternLayout("%d{ISO8601} +%r [%t] (%p) {%c} %m%n"); ConsoleAppender defaultAppender = new ConsoleAppender(defaultLayout); Logger.getRootLogger().removeAllAppenders(); Logger.getRootLogger().addAppender(defaultAppender); Logger.getRootLogger().setLevel(Level.OFF); //Logger.getRootLogger().setLevel(Level.ERROR); //Logger.getRootLogger().setLevel(Level.INFO); //Logger.getRootLogger().setLevel(Level.DEBUG); } String _mmapDirStr; File _mmapDir; String _mmapBakDirStr; File _mmapBakDir; DbusEventBuffer.StaticConfig getConfig(long maxEventBufferSize, int maxIndividualBufferSize, int maxIndexSize, int maxReadBufferSize, AllocationPolicy allocationPolicy, String mmapDirectory, boolean restoreMMappedBuffers) throws InvalidConfigException { DbusEventBuffer.Config config = new DbusEventBuffer.Config(); config.setMaxSize(maxEventBufferSize); config.setMaxIndividualBufferSize(maxIndividualBufferSize); config.setScnIndexSize(maxIndexSize); config.setAverageEventSize(maxReadBufferSize); config.setAllocationPolicy(allocationPolicy.name()); config.setRestoreMMappedBuffers(restoreMMappedBuffers); config.setMmapDirectory(mmapDirectory); //config.setQueuePolicy(policy.toString()); //config.setAssertLevel(null != assertLevel ? assertLevel.toString(): AssertLevel.NONE.toString()); config.setAssertLevel(AssertLevel.NONE.toString()); return config.build(); } private File initDir(String path) throws Exception { File d = new File(path); if (d.exists()) { FileUtils.deleteDirectory(d); } d.mkdir(); d.deleteOnExit(); return d; } @BeforeMethod public void setup() throws Exception { _mmapDirStr = "/tmp/test_mmap"; _mmapBakDirStr = _mmapDirStr + DbusEventBufferMult.BAK_DIRNAME_SUFFIX; _mmapDir = initDir(_mmapDirStr); _mmapBakDir = initDir(_mmapBakDirStr); } @AfterMethod public void tearDown() throws IOException { if (_mmapDir != null && _mmapDir.exists()) { FileUtils.deleteDirectory(_mmapDir); } if (_mmapBakDir != null && _mmapBakDir.exists()) { FileUtils.deleteDirectory(_mmapBakDir); } } @Test public void testMetaFileCreationMult() throws InvalidConfigException, IOException { int maxEventBufferSize = 1144; int maxIndividualBufferSize = 500; int bufNum = maxEventBufferSize / maxIndividualBufferSize; if (maxEventBufferSize % maxIndividualBufferSize > 0) bufNum++; DbusEventBuffer.StaticConfig config = getConfig(maxEventBufferSize, maxIndividualBufferSize, 100, 500, AllocationPolicy.MMAPPED_MEMORY, _mmapDirStr, true); // create buffer mult DbusEventBufferMult bufMult = createBufferMult(config); // go over all the buffers for (DbusEventBuffer dbusBuf : bufMult.bufIterable()) { File metaFile = new File(_mmapDir, dbusBuf.metaFileName()); // check that we don't have the files Assert.assertFalse(metaFile.exists()); } bufMult.saveBufferMetaInfo(false); // now the files should be there for (DbusEventBuffer dbusBuf : bufMult.bufIterable()) { File metaFile = new File(_mmapDir, dbusBuf.metaFileName()); // check that we don't have the files Assert.assertTrue(metaFile.exists()); validateFiles(metaFile, bufNum); } } @Test public void testMetaFileCloseMult() throws Exception { int maxEventBufferSize = 1144; int maxIndividualBufferSize = 500; int bufNum = maxEventBufferSize / maxIndividualBufferSize; if (maxEventBufferSize % maxIndividualBufferSize > 0) bufNum++; DbusEventBuffer.StaticConfig config = getConfig(maxEventBufferSize, maxIndividualBufferSize, 100, 500, AllocationPolicy.MMAPPED_MEMORY, _mmapDirStr, true); // create buffer mult DbusEventBufferMult bufMult = createBufferMult(config); // Save all the files and validate the meta files. bufMult.close(); for (DbusEventBuffer dbusBuf : bufMult.bufIterable()) { File metaFile = new File(_mmapDir, dbusBuf.metaFileName()); // check that we don't have the files Assert.assertTrue(metaFile.exists()); validateFiles(metaFile, bufNum); } File[] entries = _mmapDir.listFiles(); // When we create a new multi-buffer, we should get renamed files as well as new files. bufMult = createBufferMult(config); entries = _mmapDir.listFiles(); // Has session dirs and renamed meta files. // Create an info file for one buffer. DbusEventBuffer buf = bufMult.bufIterable().iterator().next(); buf.saveBufferMetaInfo(true); File infoFile = new File(_mmapDir, buf.metaFileName() + ".info"); Assert.assertTrue(infoFile.exists()); // Create a session directory that has one file in it. File badSes1 = new File(_mmapDir, DbusEventBuffer.getSessionPrefix() + "m"); badSes1.mkdir(); badSes1.deleteOnExit(); File junkFile = new File(badSes1.getAbsolutePath() + "/junkFile"); junkFile.createNewFile(); junkFile.deleteOnExit(); // Create a directory that is empty File badSes2 = new File(_mmapDir, DbusEventBuffer.getSessionPrefix() + "n"); badSes2.mkdir(); badSes2.deleteOnExit(); // Create a good file under mmap directory that we don't want to see removed. final String goodFile = "GoodFile"; File gf = new File(_mmapDir, goodFile); gf.createNewFile(); // Now close the multibuf, and see that the new files are still there. // We should have deleted the unused sessions and info files. bufMult.close(); HashSet<String> validEntries = new HashSet<String>(bufNum); for (DbusEventBuffer dbusBuf : bufMult.bufIterable()) { File metaFile = new File(_mmapDir, dbusBuf.metaFileName()); // check that we don't have the files Assert.assertTrue(metaFile.exists()); validateFiles(metaFile, bufNum); validEntries.add(metaFile.getName()); DbusEventBufferMetaInfo mi = new DbusEventBufferMetaInfo(metaFile); mi.loadMetaInfo(); validEntries.add(mi.getSessionId()); } validEntries.add(goodFile); // Now we should be left with meta files, and session dirs and nothing else. entries = _mmapDir.listFiles(); for (File f : entries) { Assert.assertTrue(validEntries.contains(f.getName())); validEntries.remove(f.getName()); } Assert.assertTrue(validEntries.isEmpty()); // And everything else should have moved to the .BAK directory entries = _mmapBakDir.listFiles(); HashMap<String, File> fileHashMap = new HashMap<String, File>(entries.length); for (File f : entries) { fileHashMap.put(f.getName(), f); } Assert.assertTrue(fileHashMap.containsKey(badSes1.getName())); Assert.assertTrue(fileHashMap.get(badSes1.getName()).isDirectory()); Assert.assertEquals(fileHashMap.get(badSes1.getName()).listFiles().length, 1); Assert.assertEquals(fileHashMap.get(badSes1.getName()).listFiles()[0].getName(), junkFile.getName()); fileHashMap.remove(badSes1.getName()); Assert.assertTrue(fileHashMap.containsKey(badSes2.getName())); Assert.assertTrue(fileHashMap.get(badSes2.getName()).isDirectory()); Assert.assertEquals(fileHashMap.get(badSes2.getName()).listFiles().length, 0); fileHashMap.remove(badSes2.getName()); // We should have the renamed meta files in the hash now. for (File f : entries) { if (f.getName().startsWith(DbusEventBuffer.getMmapMetaInfoFileNamePrefix())) { Assert.assertTrue(fileHashMap.containsKey(f.getName())); Assert.assertTrue(f.isFile()); fileHashMap.remove(f.getName()); } } Assert.assertTrue(fileHashMap.isEmpty()); // One more test to make sure we create the BAK directory dynamically if it does not exist. FileUtils.deleteDirectory(_mmapBakDir); bufMult = createBufferMult(config); entries = _mmapDir.listFiles(); // Create an info file for one buffer. buf = bufMult.bufIterable().iterator().next(); buf.saveBufferMetaInfo(true); infoFile = new File(_mmapDir, buf.metaFileName() + ".info"); Assert.assertTrue(infoFile.exists()); bufMult.close(); entries = _mmapBakDir.listFiles(); fileHashMap = new HashMap<String, File>(entries.length); for (File f : entries) { fileHashMap.put(f.getName(), f); } Assert.assertTrue(fileHashMap.containsKey(infoFile.getName())); Assert.assertTrue(fileHashMap.get(infoFile.getName()).isFile()); } @Test public void testMetaFileCreation() throws InvalidConfigException, IOException { int maxEventBufferSize = 1144; int maxIndividualBufferSize = 500; DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(maxEventBufferSize, maxIndividualBufferSize, 100, 500, AllocationPolicy.HEAP_MEMORY, _mmapDirStr, true)); File metaFile = new File(_mmapDir, dbusBuf.metaFileName()); dbusBuf.saveBufferMetaInfo(false); Assert.assertFalse(metaFile.exists()); // because of Allocation policy HEAP dbusBuf = new DbusEventBuffer(getConfig(maxEventBufferSize, maxIndividualBufferSize, 100, 500, AllocationPolicy.MMAPPED_MEMORY, _mmapDirStr, true)); // after buffer is created - meta file should be removed Assert.assertFalse(metaFile.exists()); dbusBuf.saveBufferMetaInfo(false); // new file should be created Assert.assertTrue(metaFile.exists()); int bufNum = maxEventBufferSize / maxIndividualBufferSize; if (maxEventBufferSize % maxIndividualBufferSize > 0) bufNum++; validateFiles(metaFile, bufNum); // now files are there - but if we create a buffer again - the meta file should be moved dbusBuf = new DbusEventBuffer(getConfig(maxEventBufferSize, maxIndividualBufferSize, 100, 500, AllocationPolicy.MMAPPED_MEMORY, _mmapDirStr, true)); Assert.assertFalse(metaFile.exists()); dbusBuf.saveBufferMetaInfo(true); // .info file should be create , but not a regular meta file Assert.assertFalse(metaFile.exists()); Assert.assertTrue(new File(metaFile.getAbsolutePath() + ".info").exists()); } @Test public void testMetaFileCleanup() throws InvalidConfigException, IOException { testMetaFileCleanup(true); //do not remove sessionid and meta file testMetaFileCleanup(false); } public void testMetaFileCleanup(boolean persistBuffer) throws InvalidConfigException, IOException { int maxEventBufferSize = 1144; int maxIndividualBufferSize = 500; DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(maxEventBufferSize, maxIndividualBufferSize, 100, 500, AllocationPolicy.MMAPPED_MEMORY, _mmapDirStr, true)); File metaFile = new File(_mmapDir, dbusBuf.metaFileName()); // after buffer is created - meta file should be removed Assert.assertFalse(metaFile.exists()); dbusBuf.saveBufferMetaInfo(false); // new file should be created Assert.assertTrue(metaFile.exists()); DbusEventBufferMetaInfo mi = new DbusEventBufferMetaInfo(metaFile); mi.loadMetaInfo(); Assert.assertTrue(mi.isValid()); dbusBuf.closeBuffer(persistBuffer); // this should close buffer and removed the files File sessionDir = new File(metaFile.getParent(), mi.getSessionId()); if (persistBuffer) { // dir and meta data should be still around Assert.assertTrue(sessionDir.exists() && sessionDir.isDirectory(), "mmap session dir still exists"); // now let's touch one of the mmap files File writeBufferFile = new File(sessionDir, "writeBuffer_0"); Assert.assertTrue(writeBufferFile.exists() && writeBufferFile.isFile(), "writeBufferFile still exist"); mi = new DbusEventBufferMetaInfo(metaFile); mi.loadMetaInfo(); Assert.assertTrue(mi.isValid(), "MetaInfo file is valid"); } else { // dir and meta files both should be gone Assert.assertFalse(sessionDir.exists(), "mmap session dir doesn't exist"); Assert.assertFalse(metaFile.exists(), "meta file session dir doesn't exist"); } } @Test public void testMetaFileTimeStamp() throws InvalidConfigException, IOException { int maxEventBufferSize = 1144; int maxIndividualBufferSize = 500; DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(maxEventBufferSize, maxIndividualBufferSize, 100, 500, AllocationPolicy.MMAPPED_MEMORY, _mmapDirStr, true)); File metaFile = new File(_mmapDir, dbusBuf.metaFileName()); // after buffer is created - meta file should be removed Assert.assertFalse(metaFile.exists()); dbusBuf.saveBufferMetaInfo(false); // new file should be created Assert.assertTrue(metaFile.exists()); DbusEventBufferMetaInfo mi = new DbusEventBufferMetaInfo(metaFile); mi.loadMetaInfo(); Assert.assertTrue(mi.isValid()); // now let's touch one of the mmap files File writeBufferFile = new File(new File(metaFile.getParent(), mi.getSessionId()), "writeBuffer_0"); long modTime = System.currentTimeMillis() + 1000; //Precision of the mod time is 1 sec writeBufferFile.setLastModified(modTime); LOG.debug("setting mod time for " + writeBufferFile + " to " + modTime + " now val = " + writeBufferFile.lastModified()); mi = new DbusEventBufferMetaInfo(metaFile); mi.loadMetaInfo(); Assert.assertTrue(mi.isValid()); //we don't invalidate the meta file based on mod time } @Test /** * test: * 1. create buffer, push events , validate events are there * 2. save meta info * 3. create a new buffer - it should pick up all the events.Validate it * 4. create another buffer (without calling save metainfo again) - this one should be empty. * * @throws InvalidConfigException * @throws IOException */ public void testRestoringBufferWithEvents() throws InvalidConfigException, IOException { int maxEventBufferSize = 1144; int maxIndividualBufferSize = 500; int numEvents = 10; // must be even DbusEventBuffer.StaticConfig conf = getConfig(maxEventBufferSize, maxIndividualBufferSize, 100, 500, AllocationPolicy.MMAPPED_MEMORY, _mmapDirStr, true); DbusEventBuffer dbusBuf = new DbusEventBuffer(conf); pushEventsToBuffer(dbusBuf, numEvents); // verify events are in the buffer DbusEventIterator it = dbusBuf.acquireIterator("allevents"); int count = -1; // first event is "prev" event while (it.hasNext()) { DbusEvent e = it.next(); Assert.assertTrue(e.isValid()); count++; } dbusBuf.releaseIterator(it); Assert.assertEquals(count, numEvents); // save meta data dbusBuf.saveBufferMetaInfo(false); // create another similar buffer and see if has the events dbusBuf = new DbusEventBuffer(conf); dbusBuf.validateEventsInBuffer(); it = dbusBuf.acquireIterator("alleventsNew"); count = -1; // first event is "prev" event while (it.hasNext()) { DbusEvent e = it.next(); Assert.assertTrue(e.isValid()); count++; } Assert.assertEquals(count, numEvents); dbusBuf.releaseIterator(it); // meta file should be gone by now File metaFile = new File(_mmapDir, dbusBuf.metaFileName()); Assert.assertFalse(metaFile.exists()); // we can start a buffer and would be empty dbusBuf = new DbusEventBuffer(conf); it = dbusBuf.acquireIterator("alleventsNewEmpty"); Assert.assertFalse(it.hasNext()); dbusBuf.releaseIterator(it); } private void pushEventsToBuffer(DbusEventBuffer dbusBuf, int numEvents) { dbusBuf.start(1); dbusBuf.startEvents(); DbusEventGenerator generator = new DbusEventGenerator(); Vector<DbusEvent> events = new Vector<DbusEvent>(); generator.generateEvents(numEvents, 1, 100, 10, events); // set end of windows for (int i = 0; i < numEvents - 1; ++i) { long scn = events.get(i).sequence(); ++i; DbusEventInternalWritable writableEvent; try { writableEvent = DbusEventCorrupter.makeWritable(events.get(i)); } catch (InvalidEventException ie) { LOG.error("Exception trace is " + ie); Assert.fail(); return; } writableEvent.setSrcId((short) -2); writableEvent.setSequence(scn); writableEvent.applyCrc(); assertTrue("invalid event #" + i, writableEvent.isValid(true)); } // set up the ReadChannel with 2 events ByteArrayOutputStream oStream = new ByteArrayOutputStream(); WritableByteChannel oChannel = Channels.newChannel(oStream); for (int i = 0; i < numEvents; ++i) { ((DbusEventInternalReadable) events.get(i)).writeTo(oChannel, Encoding.BINARY); } byte[] writeBytes = oStream.toByteArray(); ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes); ReadableByteChannel rChannel = Channels.newChannel(iStream); try { dbusBuf.readEvents(rChannel); } catch (InvalidEventException ie) { LOG.error("Exception trace is " + ie); Assert.fail(); return; } } private void validateFiles(File metaFile, int bufNumbers) throws DbusEventBufferMetaInfoException { DbusEventBufferMetaInfo mi = new DbusEventBufferMetaInfo(metaFile); mi.loadMetaInfo(); Assert.assertTrue(mi.isValid()); String sessionId = mi.getSessionId(); // figure out what session directory to use for the content of the buffers File dir = new File(metaFile.getParentFile(), sessionId); Assert.assertTrue(new File(dir, "scnIndexMetaFile").exists()); for (int i = 0; i < bufNumbers; i++) { Assert.assertTrue(new File(dir, "writeBuffer_" + i).exists()); } Assert.assertTrue(new File(dir, "scnIndex").exists()); } private DbusEventBufferMult createBufferMult(DbusEventBuffer.StaticConfig config) throws IOException, InvalidConfigException { ObjectMapper mapper = new ObjectMapper(); InputStreamReader isr = new InputStreamReader( IOUtils.toInputStream(TestDbusEventBufferMult._configSource1)); PhysicalSourceConfig pConfig1 = mapper.readValue(isr, PhysicalSourceConfig.class); isr.close(); isr = new InputStreamReader(IOUtils.toInputStream(TestDbusEventBufferMult._configSource2)); PhysicalSourceConfig pConfig2 = mapper.readValue(isr, PhysicalSourceConfig.class); PhysicalSourceStaticConfig pStatConf1 = pConfig1.build(); PhysicalSourceStaticConfig pStatConf2 = pConfig2.build(); PhysicalSourceStaticConfig[] _physConfigs = new PhysicalSourceStaticConfig[] { pStatConf1, pStatConf2 }; return new DbusEventBufferMult(_physConfigs, config, new DbusEventV2Factory()); } }