net.timewalker.ffmq4.storage.data.impl.AbstractBlockBasedDataStore.java Source code

Java tutorial

Introduction

Here is the source code for net.timewalker.ffmq4.storage.data.impl.AbstractBlockBasedDataStore.java

Source

/*
 * This file is part of FFMQ.
 *
 * FFMQ is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * FFMQ is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with FFMQ; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
package net.timewalker.ffmq4.storage.data.impl;

import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;

import net.timewalker.ffmq4.management.destination.AbstractDestinationDescriptor;
import net.timewalker.ffmq4.storage.data.DataStoreException;
import net.timewalker.ffmq4.utils.ArrayTools;
import net.timewalker.ffmq4.utils.FastBitSet;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

/**
 * <b>Block-based data store with journal support</b><br>
 * <p>
 * Implements a mix between a block-based file system structure and a doubly-linked list.
 * </p>
 * <p>
 *  Disk layout of the allocation table :
 * </p>
 * <pre>
 *  blockCount               | 4 bytes (int)     | Number of blocks in store
 *  blockSize                | 4 bytes (int)     | Size of a storage block
 *  firstClusterIndex        | 4 bytes (int)     | Index of the first block
 *  allocationTable  (*blockCount allocation entries)
 *      flags                | 1 byte            | Flags indicating the block usage
 *      allocatedSize        | 4 bytes (int)     | Size of the actual data stored in this block
 *      previousClusterIndex | 4 bytes (int)     | Index of the previous block
 *      nextClusterIndex     | 4 bytes (int)     | Index of the next block
 * </pre>
 * <p>
 *  Disk layout of the data store :
 * </p>
 * <pre>
 *  blocks (*blockCount blocks of size blockSize)
 * </pre>
 * <p>
 *  <b>Computing a block offset from its index :</b><br>
 *  blockOffset = blockIndex*blockSize<br>
 * </p>
 */
public abstract class AbstractBlockBasedDataStore extends AbstractDataStore {
    private static final Log log = LogFactory.getLog(AbstractBlockBasedDataStore.class);

    // Constants
    public static final String DATA_FILE_SUFFIX = ".store";
    public static final String ALLOCATION_TABLE_SUFFIX = ".index";

    // Flags
    private static final byte FLAG_START_BLOCK = 1;
    private static final byte FLAG_END_BLOCK = 2;

    // Offsets in the allocation table   
    public static final int AT_HEADER_SIZE = 4 + 4 + 4;
    public static final int AT_BLOCK_SIZE = 1 + 4 + 4 + 4;
    public static final int AT_HEADER_BLOCKCOUNT_OFFSET = 0;
    protected static final int AT_HEADER_FIRSTBLOCK_OFFSET = 4 + 4;

    // Offsets inside an allocation block
    protected static final int AB_FLAGS_OFFSET = 0;
    protected static final int AB_ALLOCSIZE_OFFSET = 1;
    protected static final int AB_PREVBLOCK_OFFSET = 1 + 4;
    protected static final int AB_NEXTBLOCK_OFFSET = 1 + 4 + 4;

    // Setup
    protected AbstractDestinationDescriptor descriptor;

    // In-memory allocation table
    protected int blockSize;
    protected int blockCount;
    private int maxBlockCount;
    private int autoExtendAmount;
    protected byte[] flags;
    protected int[] allocatedSize;
    protected int[] nextBlock;
    protected int[] previousBlock;
    protected int firstBlock;

    // Filesystem 
    protected File allocationTableFile;
    protected File dataFile;
    protected RandomAccessFile allocationTableRandomAccessFile;
    protected RandomAccessFile dataRandomAccessFile;

    // Runtime only
    private int lastEmpty;
    private int size;
    private int blocksInUse;

    /**
     * Constructor
     */
    public AbstractBlockBasedDataStore(AbstractDestinationDescriptor descriptor) {
        this.descriptor = descriptor;
        this.maxBlockCount = descriptor.getMaxBlockCount();
        this.autoExtendAmount = descriptor.getAutoExtendAmount();
    }

    /*
     * (non-Javadoc)
     * @see net.timewalker.ffmq4.storage.data.LinkedDataStore#init()
     */
    @Override
    public final void init() throws DataStoreException {
        initFilesystem();
        loadAllocationTable();
    }

    protected void initFilesystem() throws DataStoreException {
        String baseName = descriptor.getName();
        File dataFolder = descriptor.getDataFolder();

        log.debug("[" + baseName + "] Initializing store '" + baseName + "' filesystem");
        try {
            allocationTableFile = new File(dataFolder, baseName + ALLOCATION_TABLE_SUFFIX);
            if (!allocationTableFile.canRead())
                throw new DataStoreException(
                        "Cannot access store allocation table : " + allocationTableFile.getAbsolutePath());
            allocationTableRandomAccessFile = new RandomAccessFile(allocationTableFile, "rw");

            dataFile = new File(dataFolder, baseName + DATA_FILE_SUFFIX);
            if (!dataFile.canRead())
                throw new DataStoreException("Cannot access store data file : " + dataFile.getAbsolutePath());
            dataRandomAccessFile = new RandomAccessFile(dataFile, "rw");
        } catch (FileNotFoundException e) {
            throw new DataStoreException("Cannot access file : " + e.getMessage());
        }
    }

    private final void loadAllocationTable() throws DataStoreException {
        log.debug(
                "[" + descriptor.getName() + "] Loading allocation table " + allocationTableFile.getAbsolutePath());
        DataInputStream in = null;
        try {
            in = new DataInputStream(new BufferedInputStream(new FileInputStream(allocationTableFile), 16384));

            this.blockCount = in.readInt();
            this.blockSize = in.readInt();
            this.firstBlock = in.readInt();

            this.flags = new byte[blockCount];
            this.allocatedSize = new int[blockCount];
            this.previousBlock = new int[blockCount];
            this.nextBlock = new int[blockCount];
            this.blocksInUse = 0;
            int msgCount = 0;
            for (int n = 0; n < blockCount; n++) {
                flags[n] = in.readByte();
                allocatedSize[n] = in.readInt();
                previousBlock[n] = in.readInt();
                nextBlock[n] = in.readInt();

                if (allocatedSize[n] != -1) {
                    blocksInUse++;

                    if ((flags[n] & FLAG_START_BLOCK) > 0)
                        msgCount++;
                }
            }
            this.locks = new FastBitSet(blockCount);
            this.size = msgCount;

            log.debug("[" + descriptor.getName() + "] " + msgCount + " entries found");
        } catch (EOFException e) {
            throw new DataStoreException("Allocation table is truncated : " + allocationTableFile.getAbsolutePath(),
                    e);
        } catch (IOException e) {
            throw new DataStoreException(
                    "Cannot initialize allocation table : " + allocationTableFile.getAbsolutePath(), e);
        } finally {
            if (in != null) {
                try {
                    in.close();
                } catch (IOException e) {
                    log.error("[" + descriptor.getName() + "] Could not close file input stream", e);
                }
            }
        }
    }

    private int findEmpty() throws DataStoreException {
        int pos = lastEmpty;
        for (int n = 0; n < blockCount; n++) {
            if (pos >= blockCount)
                pos = 0;

            if (allocatedSize[pos] == -1) {
                lastEmpty = pos + 1;
                return pos;
            }

            pos++;
        }
        throw new DataStoreException("Allocation table is full (" + blockCount + " blocks)");
    }

    /* (non-Javadoc)
     * @see net.timewalker.ffmq4.storage.data.impl.AbstractDataStore#checkHandle(int)
     */
    @Override
    protected void checkHandle(int handle) throws DataStoreException {
        if (handle < 0 || handle >= blockCount || allocatedSize[handle] == -1
                || (flags[handle] & FLAG_START_BLOCK) == 0)
            throw new DataStoreException("Invalid handle : " + handle);
    }

    /*
     * (non-Javadoc)
     * @see net.timewalker.ffmq4.utils.store.ChainedDataStore#first()
     */
    @Override
    public final int first() throws DataStoreException {
        return firstBlock;
    }

    /*
     * (non-Javadoc)
     * @see net.timewalker.ffmq4.utils.store.ChainedDataStore#next(int)
     */
    @Override
    public final int next(int handle) throws DataStoreException {
        if (SAFE_MODE)
            checkHandle(handle);

        int current = nextBlock[handle];
        while (current != -1 && (flags[current] & FLAG_START_BLOCK) == 0)
            current = nextBlock[current];

        return current;
    }

    /*
     * (non-Javadoc)
     * @see net.timewalker.ffmq4.utils.store.LinkedStore#previous(int)
     */
    @Override
    public final int previous(int handle) throws DataStoreException {
        if (SAFE_MODE)
            checkHandle(handle);

        int current = previousBlock[handle];
        while (current != -1 && (flags[current] & FLAG_START_BLOCK) == 0)
            current = previousBlock[current];

        return current;
    }

    private int computeSize(int handle) throws DataStoreException {
        int totalSize = 0;
        int current = handle;
        while (current != -1) {
            totalSize += allocatedSize[current];
            if ((flags[current] & FLAG_END_BLOCK) > 0)
                break;
            current = nextBlock[current];
        }
        if (current == -1)
            throw new DataStoreException("Can't find end block for " + handle);

        return totalSize;
    }

    private int computeBlocks(int handle) throws DataStoreException {
        int totalBlocks = 0;
        int current = handle;
        while (current != -1) {
            totalBlocks++;
            if ((flags[current] & FLAG_END_BLOCK) > 0)
                break;
            current = nextBlock[current];
        }
        if (current == -1)
            throw new DataStoreException("Can't find end block for " + handle);

        return totalBlocks;
    }

    public final byte[] retrieveHeader(int handle, int headerSize) throws DataStoreException {
        if (SAFE_MODE)
            checkHandle(handle);

        // Read block from map file
        byte[] data = new byte[headerSize];
        readDataBlock(data, 0, headerSize, handle);
        return data;
    }

    /*
     * (non-Javadoc)
     * @see net.timewalker.ffmq4.utils.store.ChainedDataStore#retrieve(int)
     */
    @Override
    public final Object retrieve(int handle) throws DataStoreException {
        if (SAFE_MODE)
            checkHandle(handle);

        // Compute total size
        int totalSize = computeSize(handle);

        // Retrieve all blocks
        byte[] data = new byte[totalSize];
        int offset = 0;
        int current = handle;
        while (current != -1) {
            int blockLen = allocatedSize[current];

            // Read block from map file
            readDataBlock(data, offset, blockLen, current);
            offset += blockLen;

            if ((flags[current] & FLAG_END_BLOCK) > 0)
                break;
            current = nextBlock[current];
        }
        if (current == -1)
            throw new DataStoreException("Can't find end block for " + handle);

        return data;
    }

    /*
     * (non-Javadoc)
     * @see net.timewalker.ffmq4.utils.store.ChainedDataStore#size()
     */
    @Override
    public final int size() {
        return size;
    }

    private byte[] asByteArray(Object obj) throws DataStoreException {
        if (obj instanceof byte[])
            return (byte[]) obj;
        throw new DataStoreException("Unsupported object type : " + obj.getClass().getName());
    }

    /*
     * (non-Javadoc)
     * @see net.timewalker.ffmq4.utils.store.ChainedDataStore#delete(int)
     */
    @Override
    public final int delete(int handle) throws DataStoreException {
        if (SAFE_MODE)
            checkHandle(handle);

        int previousHandle = previousBlock[handle];

        int current = handle;
        int nextHandle = -1;
        while (current != -1) {
            nextHandle = nextBlock[current];
            boolean isEndBlock = (flags[current] & FLAG_END_BLOCK) > 0;

            // Clear block entry
            flags[current] = 0;
            allocatedSize[current] = -1;
            previousBlock[current] = -1;
            nextBlock[current] = -1;
            locks.clear(current);

            // Update used blocks count
            blocksInUse--;

            writeAllocationBlock(current);

            if (isEndBlock)
                break;
            current = nextHandle;
        }
        // Reconnect chain
        if (nextHandle != -1) {
            previousBlock[nextHandle] = previousHandle;
            writeAllocationBlock(nextHandle);
        }
        if (previousHandle != -1) {
            nextBlock[previousHandle] = nextHandle;
            writeAllocationBlock(previousHandle);
        }

        // Update first block if necessary
        if (firstBlock == handle) {
            firstBlock = nextHandle;
            writeFirstBlock();
        }

        size--;

        flush();

        // Find previous object start from previous block handle
        while (previousHandle != -1 && (flags[previousHandle] & FLAG_START_BLOCK) == 0)
            previousHandle = previousBlock[previousHandle];

        return previousHandle;
    }

    /* (non-Javadoc)
     * @see net.timewalker.ffmq4.storage.data.LinkedDataStore#replace(int, java.lang.Object)
     */
    @Override
    public final int replace(int handle, Object obj) throws DataStoreException {
        if (SAFE_MODE)
            checkHandle(handle);

        byte[] data = asByteArray(obj);
        int previousHandle = previousBlock[handle];

        // Make sure we have enough space to do this
        int originalBlocks = computeBlocks(handle);
        int targetBlocks = (data.length / blockSize) + ((data.length % blockSize) > 0 ? 1 : 0);
        if (targetBlocks > originalBlocks) {
            int requiredFreeBlocks = targetBlocks - originalBlocks;
            if (requiredFreeBlocks + blocksInUse > blockCount) {
                log.error("[" + descriptor.getName() + "] Not enough free blocks to update message ("
                        + requiredFreeBlocks + " needed, " + (blockCount - blocksInUse) + " left), queue is full.");
                return -1;
            }
        }

        // Save locking status
        boolean wasLocked = isLocked(handle);

        // Delete the old message
        delete(handle);

        // Store the new version
        int newHandle = store(obj, previousHandle);

        // Re-lock if necessary
        if (wasLocked)
            lock(newHandle);

        flush();

        return newHandle;
    }

    /*
     * (non-Javadoc)
     * @see net.timewalker.ffmq4.utils.store.ChainedDataStore#store(byte[], int)
     */
    @Override
    public final int store(Object obj, int previousHandle) throws DataStoreException {
        byte[] data = asByteArray(obj);

        int lastBlockOfPreviousEntry;
        int firstBlockOfNextEntry;

        // Find blocks around insertion point
        if (previousHandle != -1) {
            //if (SAFE_MODE) checkHandle(previousHandle); FIXME previousHandle may already be lastBlockOfPreviousEntry

            // Find end block
            lastBlockOfPreviousEntry = previousHandle;
            while (lastBlockOfPreviousEntry != -1 && (flags[lastBlockOfPreviousEntry] & FLAG_END_BLOCK) == 0)
                lastBlockOfPreviousEntry = nextBlock[lastBlockOfPreviousEntry];
            if (lastBlockOfPreviousEntry == -1)
                throw new DataStoreException("Can't find end block for " + previousHandle);

            // Get successor block
            firstBlockOfNextEntry = nextBlock[lastBlockOfPreviousEntry];
        } else {
            lastBlockOfPreviousEntry = previousHandle;
            firstBlockOfNextEntry = firstBlock;
        }

        // Store data in split blocks
        int newHandle = storeData(data, lastBlockOfPreviousEntry, firstBlockOfNextEntry);
        if (newHandle == -1)
            return -1;

        // Update first block index if necessary
        // --> This should be done last for I/O failure reliability : 
        //      if something goes wrong before this, the firstBlock is still consistent
        if (previousHandle == -1) {
            firstBlock = newHandle;
            writeFirstBlock();
        }

        size++;

        flush();

        return newHandle;
    }

    private boolean autoExtendStore() throws DataStoreException {
        if (autoExtendAmount == 0)
            return false;

        int oldBlockCount = blockCount;
        int newBlockCount = Math.min(blockCount + autoExtendAmount, maxBlockCount);
        if (newBlockCount <= oldBlockCount)
            return false;

        log.debug("[" + descriptor.getName() + "] Auto-extending store to " + newBlockCount + " blocks");

        // Extend memory structures
        this.flags = ArrayTools.extend(flags, newBlockCount);
        this.allocatedSize = ArrayTools.extend(allocatedSize, newBlockCount);
        this.nextBlock = ArrayTools.extend(nextBlock, newBlockCount);
        this.previousBlock = ArrayTools.extend(previousBlock, newBlockCount);
        for (int n = blockCount; n < newBlockCount; n++) {
            allocatedSize[n] = -1;
            previousBlock[n] = -1;
            nextBlock[n] = -1;
        }
        this.locks.ensureCapacity(newBlockCount);
        this.blockCount = newBlockCount;

        // Extend physical storage
        extendStoreFiles(oldBlockCount, newBlockCount);

        return true;
    }

    private int storeData(byte[] data, int previousBlockHandle, int nextBlockHandle) throws DataStoreException {
        int fullBlocks = data.length / blockSize;
        int remaining = data.length % blockSize;

        // Check that we have enough space before changing anything
        int requiredFreeBlocks = fullBlocks + (remaining > 0 ? 1 : 0);
        if ((blocksInUse + requiredFreeBlocks) > blockCount) {
            if ((blocksInUse + requiredFreeBlocks) > maxBlockCount) {
                log.debug("[" + descriptor.getName() + "] Not enough free blocks to store message ("
                        + requiredFreeBlocks + " needed, " + (blockCount - blocksInUse) + " left), queue is full.");
                return -1;
            } else {
                // Auto-extend store
                if (!autoExtendStore())
                    return -1;
            }
        }

        int newHandle = -1;
        int lastBlock = previousBlockHandle;
        int offset = 0;
        for (int i = 0; i < fullBlocks; i++) {
            boolean isStartBlock = (i == 0);
            boolean isEndBlock = (remaining == 0 && i == fullBlocks - 1);

            lastBlock = storeDataBlock(data, offset, blockSize, lastBlock, isStartBlock, isEndBlock);
            offset += blockSize;
            if (isStartBlock)
                newHandle = lastBlock;
        }
        if (remaining > 0) {
            lastBlock = storeDataBlock(data, offset, remaining, lastBlock, fullBlocks == 0, true);
            if (newHandle == -1)
                newHandle = lastBlock;
        }

        // Connect end of sub-list to existing list
        if (nextBlockHandle != -1) {
            nextBlock[lastBlock] = nextBlockHandle;
            previousBlock[nextBlockHandle] = lastBlock;
            writeAllocationBlock(nextBlockHandle);
        }

        // Connect end of sub-list to existing list
        if (previousBlockHandle != -1) {
            nextBlock[previousBlockHandle] = newHandle;
            writeAllocationBlock(previousBlockHandle);
        }

        // Write intermediate allocation blocks
        int current = newHandle;
        for (int i = 0; i < requiredFreeBlocks; i++) {
            writeAllocationBlock(current);
            current = nextBlock[current];
        }

        // Update used blocks count
        blocksInUse += requiredFreeBlocks;

        return newHandle;
    }

    private int storeDataBlock(byte[] data, int offset, int len, int previousHandle, boolean startBlock,
            boolean endBlock) throws DataStoreException {
        int nextEmpty = findEmpty();

        byte flag = 0;
        if (startBlock)
            flag |= FLAG_START_BLOCK;
        if (endBlock)
            flag |= FLAG_END_BLOCK;
        flags[nextEmpty] = flag;
        nextBlock[nextEmpty] = -1;
        previousBlock[nextEmpty] = previousHandle;
        allocatedSize[nextEmpty] = len;

        // Connect block to previous if available
        if (previousHandle != -1)
            nextBlock[previousHandle] = nextEmpty;

        // Write data to map file
        writeDataBlock(data, offset, len, nextEmpty);

        return nextEmpty;
    }

    /*
     * (non-Javadoc)
     * @see net.timewalker.ffmq4.utils.store.ChainedDataStore#close()
     */
    @Override
    public void close() {
        // Close filesystem
        try {
            if (allocationTableRandomAccessFile != null)
                allocationTableRandomAccessFile.close();
        } catch (IOException e) {
            log.error("[" + descriptor.getName() + "] Could not close allocation table file : " + e.toString());
        }
        try {
            if (dataRandomAccessFile != null)
                dataRandomAccessFile.close();
        } catch (IOException e) {
            log.error("[" + descriptor.getName() + "] Could not close map file : " + e.toString());
        }
    }

    /* (non-Javadoc)
     * @see java.lang.Object#toString()
     */
    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder();

        sb.append("Allocation Table (size=" + size + ")\n");
        sb.append("------------------------------------\n");
        sb.append("first block index : ");
        sb.append(firstBlock);
        sb.append("\n");
        for (int n = 0; n < blockCount; n++) {
            sb.append(n);
            sb.append(": ");
            if (allocatedSize[n] == -1)
                sb.append("(free)\n");
            else {
                sb.append(previousBlock[n]);
                sb.append("\t");
                sb.append(nextBlock[n]);
                sb.append("\t");
                sb.append(allocatedSize[n]);
                if ((flags[n] & FLAG_START_BLOCK) > 0) {
                    sb.append("\t");
                    sb.append("START");
                }
                if ((flags[n] & FLAG_END_BLOCK) > 0) {
                    sb.append("\t");
                    sb.append("END");
                }
                sb.append("\n");
            }
        }
        sb.append("------------------------------------\n");

        return sb.toString();
    }

    /**
     * Run an integrity check on the store files and fix them as necessary
     * @throws DataStoreException if the files could not be fixed
     */
    protected void integrityCheck() throws DataStoreException {
        try {
            //========================
            // 1 - Check files sizes
            //========================
            // -- Allocation table
            long atFileSize = allocationTableRandomAccessFile.length();
            if (atFileSize < AT_HEADER_SIZE + AT_BLOCK_SIZE) /* Should have at least one entry */
                throw new DataStoreException(
                        "Allocation table is truncated : " + allocationTableFile.getAbsolutePath());

            // Read some header fields
            FileInputStream inFile = new FileInputStream(allocationTableFile);
            DataInputStream in = new DataInputStream(new BufferedInputStream(inFile, 16384));
            int blockCount = in.readInt();
            int blockSize = in.readInt();
            int firstBlock = in.readInt();
            // Fix AT size
            long expectedATFileSize = AT_HEADER_SIZE + AT_BLOCK_SIZE * (long) blockCount;
            if (atFileSize != expectedATFileSize) {
                log.error("[" + descriptor.getName() + "] Allocation table has an invalid size (actual:"
                        + atFileSize + ",expected:" + expectedATFileSize + "), fixing.");
                allocationTableRandomAccessFile.setLength(expectedATFileSize);
            }
            // Fix data size
            long dataFileSize = dataRandomAccessFile.length();
            long expectedDataFileSize = (long) blockSize * blockCount;
            if (dataFileSize != expectedDataFileSize) {
                log.error("[" + descriptor.getName() + "] Data file has an invalid size (actual:" + dataFileSize
                        + ",expected:" + expectedDataFileSize + "), fixing.");
                dataRandomAccessFile.setLength(expectedDataFileSize);
            }

            //============================
            // 2 - Check allocation table
            //============================
            // Read the AT into memory
            byte[] flags = new byte[blockCount];
            int[] allocatedSize = new int[blockCount];
            int[] previousBlock = new int[blockCount];
            int[] nextBlock = new int[blockCount];
            int blocksInUse = 0;
            int msgCount = 0;
            for (int n = 0; n < blockCount; n++) {
                flags[n] = in.readByte();
                allocatedSize[n] = in.readInt();
                previousBlock[n] = in.readInt();
                nextBlock[n] = in.readInt();
                if (allocatedSize[n] != -1) {
                    blocksInUse++;
                    if ((flags[n] & FLAG_START_BLOCK) > 0)
                        msgCount++;
                }
            }
            in.close();
            log.debug("[" + descriptor.getName() + "] Blocks in use before fix : " + blocksInUse);
            log.debug("[" + descriptor.getName() + "] Messages count before fix : " + msgCount);

            // Fix first block index
            boolean changed = false;
            if (firstBlock < -1 || firstBlock >= blockCount) {
                log.error("[" + descriptor.getName() + "] Invalid allocation table first block index (" + firstBlock
                        + "), guessing new one ...");
                firstBlock = guessFirstBlockIndex(blockCount, allocatedSize, nextBlock);
                log.debug("[" + descriptor.getName() + "] Guessed first block index : " + firstBlock);
                changed = true;
            }

            // Recover table
            if (msgCount == 0) {
                if (firstBlock == -1) {
                    // Table is empty, cleanup dirty entries
                    changed = changed
                            || cleanupEmptyBlocks(blockCount, flags, allocatedSize, previousBlock, nextBlock);
                } else {
                    log.error("[" + descriptor.getName() + "] First block index should be -1, clearing ...");
                    firstBlock = -1;
                    changed = true;
                }
            } else {
                if (firstBlock == -1) {
                    log.error("[" + descriptor.getName() + "] Invalid first block index, guessing value ...");
                    firstBlock = guessFirstBlockIndex(blockCount, allocatedSize, nextBlock);
                    log.debug("[" + descriptor.getName() + "] Guessed first block index : " + firstBlock);
                    changed = true;
                }

                changed = changed || fixBlocks(blockCount, blockSize, firstBlock, flags, allocatedSize,
                        previousBlock, nextBlock);
                changed = changed || cleanupEmptyBlocks(blockCount, flags, allocatedSize, previousBlock, nextBlock);
            }

            // Update the allocation file table
            if (changed) {
                // Re-compute size
                msgCount = 0;
                blocksInUse = 0;
                for (int n = 0; n < blockCount; n++) {
                    if (allocatedSize[n] != -1) {
                        blocksInUse++;
                        if ((flags[n] & FLAG_START_BLOCK) > 0)
                            msgCount++;
                    }
                }
                log.debug("[" + descriptor.getName() + "] Blocks in use after fix : " + blocksInUse);
                log.debug("[" + descriptor.getName() + "] Messages count after fix : " + msgCount);

                log.debug("[" + descriptor.getName() + "] Allocation table was altered, saving ...");
                allocationTableRandomAccessFile.seek(AT_HEADER_FIRSTBLOCK_OFFSET);
                allocationTableRandomAccessFile.writeInt(firstBlock);
                for (int n = 0; n < blockCount; n++) {
                    byte[] allocationBlock = new byte[AT_BLOCK_SIZE];

                    // Regroup I/O to improve performance
                    allocationBlock[AB_FLAGS_OFFSET] = flags[n];
                    allocationBlock[AB_ALLOCSIZE_OFFSET] = (byte) ((allocatedSize[n] >>> 24) & 0xFF);
                    allocationBlock[AB_ALLOCSIZE_OFFSET + 1] = (byte) ((allocatedSize[n] >>> 16) & 0xFF);
                    allocationBlock[AB_ALLOCSIZE_OFFSET + 2] = (byte) ((allocatedSize[n] >>> 8) & 0xFF);
                    allocationBlock[AB_ALLOCSIZE_OFFSET + 3] = (byte) ((allocatedSize[n] >>> 0) & 0xFF);
                    allocationBlock[AB_PREVBLOCK_OFFSET] = (byte) ((previousBlock[n] >>> 24) & 0xFF);
                    allocationBlock[AB_PREVBLOCK_OFFSET + 1] = (byte) ((previousBlock[n] >>> 16) & 0xFF);
                    allocationBlock[AB_PREVBLOCK_OFFSET + 2] = (byte) ((previousBlock[n] >>> 8) & 0xFF);
                    allocationBlock[AB_PREVBLOCK_OFFSET + 3] = (byte) ((previousBlock[n] >>> 0) & 0xFF);
                    allocationBlock[AB_NEXTBLOCK_OFFSET] = (byte) ((nextBlock[n] >>> 24) & 0xFF);
                    allocationBlock[AB_NEXTBLOCK_OFFSET + 1] = (byte) ((nextBlock[n] >>> 16) & 0xFF);
                    allocationBlock[AB_NEXTBLOCK_OFFSET + 2] = (byte) ((nextBlock[n] >>> 8) & 0xFF);
                    allocationBlock[AB_NEXTBLOCK_OFFSET + 3] = (byte) ((nextBlock[n] >>> 0) & 0xFF);

                    allocationTableRandomAccessFile.seek(AT_HEADER_SIZE + n * AT_BLOCK_SIZE);
                    allocationTableRandomAccessFile.write(allocationBlock);
                }
                allocationTableRandomAccessFile.getFD().sync();
            } else
                log.debug("[" + descriptor.getName() + "] Allocation table was not altered");
        } catch (IOException e) {
            throw new DataStoreException("Cannot check/fix store integrity : " + e);
        }
    }

    private int guessFirstBlockIndex(int blockCount, int[] allocatedSize, int[] nextBlock) {
        FastBitSet referenced = new FastBitSet(blockCount);

        // Flag all referenced blocks
        for (int n = 0; n < blockCount; n++)
            if (allocatedSize[n] != -1 && nextBlock[n] != -1)
                referenced.set(nextBlock[n]);

        // Find candidate
        for (int n = 0; n < blockCount; n++)
            if (allocatedSize[n] != -1 && nextBlock[n] != -1 && !referenced.get(n))
                return n;

        return -1;
    }

    private boolean fixBlocks(int blockCount, int blockSize, int firstBlock, byte[] flags, int[] allocatedSize,
            int[] previousBlock, int[] nextBlock) {
        boolean changed = false;
        FastBitSet fixedBlocks = new FastBitSet(blockCount);

        // Fix reverse links first
        int previous = -1;
        int current = firstBlock;
        while (current != -1) {
            if (previousBlock[current] != previous) {
                log.debug("[" + descriptor.getName() + "] Fixing previous reference " + previousBlock[current]
                        + " -> " + previous);
                previousBlock[current] = previous;
                changed = true;
            }
            fixedBlocks.set(current);
            previous = current;
            current = nextBlock[current];
        }

        // Look for lost blocks and fix allocated sizes
        for (int n = 0; n < blockCount; n++) {
            if (allocatedSize[n] != -1 && !fixedBlocks.get(n)) {
                log.warn("[" + descriptor.getName() + "] Lost block found : " + n);
                allocatedSize[n] = -1;
                changed = true;
            }
            if (fixedBlocks.get(n) && (allocatedSize[n] <= 0 || allocatedSize[n] > blockSize)) {
                log.warn("[" + descriptor.getName() + "] Block has an invalid size (" + allocatedSize[n]
                        + "), replacing by " + blockSize);
                allocatedSize[n] = blockSize;
                changed = true;
            }
        }

        // Fix flags
        boolean startExpected = true;
        previous = -1;
        current = firstBlock;
        while (current != -1) {
            if (startExpected) {
                if ((flags[current] & FLAG_START_BLOCK) == 0) {
                    log.warn("[" + descriptor.getName() + "] Missing start block flag, adding to block " + current);
                    flags[current] |= FLAG_START_BLOCK;
                    changed = true;
                }
                if ((flags[current] & FLAG_END_BLOCK) == 0)
                    startExpected = false;
            } else {
                if ((flags[current] & FLAG_START_BLOCK) > 0) {
                    log.warn("[" + descriptor.getName() + "] Missing end block flag, adding to previous block : "
                            + previous);
                    flags[previous] |= FLAG_END_BLOCK;
                    changed = true;
                }
                if ((flags[current] & FLAG_END_BLOCK) > 0)
                    startExpected = true;
            }

            previous = current;
            current = nextBlock[current];
        }
        // Fix end of chain if necessary
        if (!startExpected) {
            log.warn("[" + descriptor.getName() + "] Missing end block flag, adding to last block : " + previous);
            flags[previous] |= FLAG_END_BLOCK;
            changed = true;
        }

        return changed;
    }

    private boolean cleanupEmptyBlocks(int blockCount, byte[] flags, int[] allocatedSize, int[] previousBlock,
            int[] nextBlock) {
        boolean changed = false;
        for (int n = 0; n < blockCount; n++) {
            if (allocatedSize[n] == -1 && (flags[n] != 0 || nextBlock[n] != -1 || previousBlock[n] != -1)) {
                flags[n] = 0;
                nextBlock[n] = -1;
                previousBlock[n] = -1;
                changed = true;
            }
        }
        return changed;
    }

    public final int getBlockSize() {
        return blockSize;
    }

    /**
     * Write the first block index to disk
     * @throws DataStoreException
     */
    protected abstract void writeFirstBlock() throws DataStoreException;

    /**
     * Write an allocation block to disk
     * @throws DataStoreException
     */
    protected abstract void writeAllocationBlock(int blockIndex) throws DataStoreException;

    /**
     * Write a data block to disk
     * @throws DataStoreException
     */
    protected abstract void writeDataBlock(byte[] data, int offset, int len, int blockHandle)
            throws DataStoreException;

    /**
     * Read a data block to disk
     * @throws DataStoreException
     */
    protected abstract void readDataBlock(byte[] data, int offset, int len, int blockHandle)
            throws DataStoreException;

    /**
     * Extend the store files to newBlockCount
     * @throws DataStoreException
     */
    protected abstract void extendStoreFiles(int oldBlockCount, int newBlockCount) throws DataStoreException;

    /**
     * Flush internal buffers
     * @throws DataStoreException
     */
    protected abstract void flush() throws DataStoreException;

    /**
     * Serialize allocation block at index blockIndex
     * @param blockIndex the block index
     * @return the serialized allocation block
     */
    protected final byte[] serializeAllocationBlock(int blockIndex) {
        byte[] allocationBlock = new byte[AT_BLOCK_SIZE];

        // Regroup I/O to improve performance
        allocationBlock[AB_FLAGS_OFFSET] = flags[blockIndex];
        allocationBlock[AB_ALLOCSIZE_OFFSET] = (byte) ((allocatedSize[blockIndex] >>> 24) & 0xFF);
        allocationBlock[AB_ALLOCSIZE_OFFSET + 1] = (byte) ((allocatedSize[blockIndex] >>> 16) & 0xFF);
        allocationBlock[AB_ALLOCSIZE_OFFSET + 2] = (byte) ((allocatedSize[blockIndex] >>> 8) & 0xFF);
        allocationBlock[AB_ALLOCSIZE_OFFSET + 3] = (byte) ((allocatedSize[blockIndex] >>> 0) & 0xFF);
        allocationBlock[AB_PREVBLOCK_OFFSET] = (byte) ((previousBlock[blockIndex] >>> 24) & 0xFF);
        allocationBlock[AB_PREVBLOCK_OFFSET + 1] = (byte) ((previousBlock[blockIndex] >>> 16) & 0xFF);
        allocationBlock[AB_PREVBLOCK_OFFSET + 2] = (byte) ((previousBlock[blockIndex] >>> 8) & 0xFF);
        allocationBlock[AB_PREVBLOCK_OFFSET + 3] = (byte) ((previousBlock[blockIndex] >>> 0) & 0xFF);
        allocationBlock[AB_NEXTBLOCK_OFFSET] = (byte) ((nextBlock[blockIndex] >>> 24) & 0xFF);
        allocationBlock[AB_NEXTBLOCK_OFFSET + 1] = (byte) ((nextBlock[blockIndex] >>> 16) & 0xFF);
        allocationBlock[AB_NEXTBLOCK_OFFSET + 2] = (byte) ((nextBlock[blockIndex] >>> 8) & 0xFF);
        allocationBlock[AB_NEXTBLOCK_OFFSET + 3] = (byte) ((nextBlock[blockIndex] >>> 0) & 0xFF);

        return allocationBlock;
    }

    /* (non-Javadoc)
     * @see net.timewalker.ffmq4.storage.data.LinkedDataStore#getStoreUsage()
     */
    @Override
    public final int getStoreUsage() {
        long ratio = blockCount > 0 ? (long) blocksInUse * 100 / blockCount : 0;
        return (int) ratio;
    }

    /* (non-Javadoc)
     * @see net.timewalker.ffmq3.storage.data.DataStore#getAbsoluteStoreUsage()
     */
    @Override
    public int getAbsoluteStoreUsage() {
        long ratio = maxBlockCount > 0 ? (long) blocksInUse * 100 / maxBlockCount : 0;
        return (int) ratio;
    }
}