Java tutorial
/** * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * */ package org.apache.bookkeeper.bookie; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.conf.TestBKConfiguration; import org.apache.bookkeeper.util.IOUtils; import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.junit.Assert.*; public class EntryLogTest { private final static Logger LOG = LoggerFactory.getLogger(EntryLogTest.class); final List<File> tempDirs = new ArrayList<File>(); File createTempDir(String prefix, String suffix) throws IOException { File dir = IOUtils.createTempDir(prefix, suffix); tempDirs.add(dir); return dir; } @After public void tearDown() throws Exception { for (File dir : tempDirs) { FileUtils.deleteDirectory(dir); } tempDirs.clear(); } @Test(timeout = 60000) public void testCorruptEntryLog() throws Exception { File tmpDir = createTempDir("bkTest", ".dir"); File curDir = Bookie.getCurrentDirectory(tmpDir); Bookie.checkDirectoryStructure(curDir); int gcWaitTime = 1000; ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); conf.setGcWaitTime(gcWaitTime); conf.setLedgerDirNames(new String[] { tmpDir.toString() }); Bookie bookie = new Bookie(conf); // create some entries EntryLogger logger = ((InterleavedLedgerStorage) bookie.ledgerStorage).entryLogger; logger.addEntry(1, generateEntry(1, 1)); logger.addEntry(3, generateEntry(3, 1)); logger.addEntry(2, generateEntry(2, 1)); logger.flush(); // now lets truncate the file to corrupt the last entry, which simulates a partial write File f = new File(curDir, "0.log"); RandomAccessFile raf = new RandomAccessFile(f, "rw"); raf.setLength(raf.length() - 10); raf.close(); // now see which ledgers are in the log logger = new EntryLogger(conf, bookie.getLedgerDirsManager()); EntryLogMetadata meta = logger.getEntryLogMetadata(0L); LOG.info("Extracted Meta From Entry Log {}", meta); assertNotNull(meta.getLedgersMap().get(1L)); assertNull(meta.getLedgersMap().get(2L)); assertNotNull(meta.getLedgersMap().get(3L)); } private ByteBuffer generateEntry(long ledger, long entry) { byte[] data = ("ledger-" + ledger + "-" + entry).getBytes(); ByteBuffer bb = ByteBuffer.wrap(new byte[8 + 8 + data.length]); bb.putLong(ledger); bb.putLong(entry); bb.put(data); bb.flip(); return bb; } @Test(timeout = 60000) public void testMissingLogId() throws Exception { File tmpDir = createTempDir("entryLogTest", ".dir"); File curDir = Bookie.getCurrentDirectory(tmpDir); Bookie.checkDirectoryStructure(curDir); ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); conf.setLedgerDirNames(new String[] { tmpDir.toString() }); Bookie bookie = new Bookie(conf); // create some entries int numLogs = 3; int numEntries = 10; long[][] positions = new long[2 * numLogs][]; for (int i = 0; i < numLogs; i++) { positions[i] = new long[numEntries]; EntryLogger logger = new EntryLogger(conf, bookie.getLedgerDirsManager()); for (int j = 0; j < numEntries; j++) { positions[i][j] = logger.addEntry(i, generateEntry(i, j)); } logger.flush(); } // delete last log id File lastLogId = new File(curDir, "lastId"); lastLogId.delete(); // write another entries for (int i = numLogs; i < 2 * numLogs; i++) { positions[i] = new long[numEntries]; EntryLogger logger = new EntryLogger(conf, bookie.getLedgerDirsManager()); for (int j = 0; j < numEntries; j++) { positions[i][j] = logger.addEntry(i, generateEntry(i, j)); } logger.flush(); } EntryLogger newLogger = new EntryLogger(conf, bookie.getLedgerDirsManager()); for (int i = 0; i < (2 * numLogs + 1); i++) { File logFile = new File(curDir, Long.toHexString(i) + ".log"); assertTrue(logFile.exists()); } for (int i = 0; i < 2 * numLogs; i++) { for (int j = 0; j < numEntries; j++) { String expectedValue = "ledger-" + i + "-" + j; byte[] value = newLogger.readEntry(i, j, positions[i][j]); ByteBuffer buf = ByteBuffer.wrap(value); long ledgerId = buf.getLong(); long entryId = buf.getLong(); byte[] data = new byte[buf.remaining()]; buf.get(data); assertEquals(i, ledgerId); assertEquals(j, entryId); assertEquals(expectedValue, new String(data)); } } } @Test(timeout = 60000) /** Test that EntryLogger Should fail with FNFE, if entry logger directories does not exist*/ public void testEntryLoggerShouldThrowFNFEIfDirectoriesDoesNotExist() throws Exception { File tmpDir = createTempDir("bkTest", ".dir"); ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); conf.setLedgerDirNames(new String[] { tmpDir.toString() }); EntryLogger entryLogger = null; try { entryLogger = new EntryLogger(conf, new LedgerDirsManager(conf, conf.getLedgerDirs())); fail("Expecting FileNotFoundException"); } catch (FileNotFoundException e) { assertEquals("Entry log directory does not exist", e.getLocalizedMessage()); } finally { if (entryLogger != null) { entryLogger.shutdown(); } } } /** * Test to verify the DiskFull during addEntry */ @Test(timeout = 60000) public void testAddEntryFailureOnDiskFull() throws Exception { File ledgerDir1 = createTempDir("bkTest", ".dir"); File ledgerDir2 = createTempDir("bkTest", ".dir"); ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); conf.setLedgerDirNames(new String[] { ledgerDir1.getAbsolutePath(), ledgerDir2.getAbsolutePath() }); Bookie bookie = new Bookie(conf); EntryLogger entryLogger = new EntryLogger(conf, bookie.getLedgerDirsManager()); InterleavedLedgerStorage ledgerStorage = ((InterleavedLedgerStorage) bookie.ledgerStorage); ledgerStorage.entryLogger = entryLogger; // Create ledgers ledgerStorage.setMasterKey(1, "key".getBytes()); ledgerStorage.setMasterKey(2, "key".getBytes()); ledgerStorage.setMasterKey(3, "key".getBytes()); // Add entries ledgerStorage.addEntry(generateEntry(1, 1)); ledgerStorage.addEntry(generateEntry(2, 1)); // Add entry with disk full failure simulation bookie.getLedgerDirsManager().addToFilledDirs(entryLogger.currentDir); ledgerStorage.addEntry(generateEntry(3, 1)); // Verify written entries Assert.assertTrue(0 == generateEntry(1, 1).compareTo(ledgerStorage.getEntry(1, 1))); Assert.assertTrue(0 == generateEntry(2, 1).compareTo(ledgerStorage.getEntry(2, 1))); Assert.assertTrue(0 == generateEntry(3, 1).compareTo(ledgerStorage.getEntry(3, 1))); } /** * Explicitely try to recover using the ledgers map index at the end of the entry log */ @Test(timeout = 60000) public void testRecoverFromLedgersMap() throws Exception { File tmpDir = createTempDir("bkTest", ".dir"); File curDir = Bookie.getCurrentDirectory(tmpDir); Bookie.checkDirectoryStructure(curDir); int gcWaitTime = 1000; ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); conf.setGcWaitTime(gcWaitTime); conf.setLedgerDirNames(new String[] { tmpDir.toString() }); Bookie bookie = new Bookie(conf); // create some entries EntryLogger logger = ((InterleavedLedgerStorage) bookie.ledgerStorage).entryLogger; logger.addEntry(1, generateEntry(1, 1)); logger.addEntry(3, generateEntry(3, 1)); logger.addEntry(2, generateEntry(2, 1)); logger.addEntry(1, generateEntry(1, 2)); logger.rollLog(); logger.flushRotatedLogs(); EntryLogMetadata meta = logger.extractEntryLogMetadataFromIndex(0L); LOG.info("Extracted Meta From Entry Log {}", meta); assertEquals(60, meta.getLedgersMap().get(1L).longValue()); assertEquals(30, meta.getLedgersMap().get(2L).longValue()); assertEquals(30, meta.getLedgersMap().get(3L).longValue()); assertNull(meta.getLedgersMap().get(4L)); assertEquals(120, meta.getTotalSize()); assertEquals(120, meta.getRemainingSize()); } /** * Explicitely try to recover using the ledgers map index at the end of the entry log */ @Test(timeout = 60000) public void testRecoverFromLedgersMapOnV0EntryLog() throws Exception { File tmpDir = createTempDir("bkTest", ".dir"); File curDir = Bookie.getCurrentDirectory(tmpDir); Bookie.checkDirectoryStructure(curDir); int gcWaitTime = 1000; ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); conf.setGcWaitTime(gcWaitTime); conf.setLedgerDirNames(new String[] { tmpDir.toString() }); Bookie bookie = new Bookie(conf); // create some entries EntryLogger logger = ((InterleavedLedgerStorage) bookie.ledgerStorage).entryLogger; logger.addEntry(1, generateEntry(1, 1)); logger.addEntry(3, generateEntry(3, 1)); logger.addEntry(2, generateEntry(2, 1)); logger.addEntry(1, generateEntry(1, 2)); logger.rollLog(); // Rewrite the entry log header to be on V0 format File f = new File(curDir, "0.log"); RandomAccessFile raf = new RandomAccessFile(f, "rw"); raf.seek(EntryLogger.HEADER_VERSION_POSITION); // Write zeros to indicate V0 + no ledgers map info raf.write(new byte[4 + 8]); raf.close(); // now see which ledgers are in the log logger = new EntryLogger(conf, bookie.getLedgerDirsManager()); try { logger.extractEntryLogMetadataFromIndex(0L); fail("Should not be possible to recover from ledgers map index"); } catch (IOException e) { // Ok } // Public method should succeed by falling back to scanning the file EntryLogMetadata meta = logger.getEntryLogMetadata(0L); LOG.info("Extracted Meta From Entry Log {}", meta); assertEquals(60, meta.getLedgersMap().get(1L).longValue()); assertEquals(30, meta.getLedgersMap().get(2L).longValue()); assertEquals(30, meta.getLedgersMap().get(3L).longValue()); assertNull(meta.getLedgersMap().get(4L)); assertEquals(120, meta.getTotalSize()); assertEquals(120, meta.getRemainingSize()); } }