com.chinamobile.bcbsp.comm.MessageQueuesForDisk.java Source code

Java tutorial

Introduction

Here is the source code for com.chinamobile.bcbsp.comm.MessageQueuesForDisk.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.chinamobile.bcbsp.comm;

import com.chinamobile.bcbsp.BSPConfiguration;
import com.chinamobile.bcbsp.Constants;
import com.chinamobile.bcbsp.util.BSPJob;
import com.chinamobile.bcbsp.util.BSPJobID;
import com.chinamobile.bcbsp.util.ObjectSizer;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryUsage;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

/**
 * MessageQueuesForDisk Message queues manager for disk supported.
 */
public class MessageQueuesForDisk implements MessageQueuesInterface {
    /** class logger. */
    private static final Log LOG = LogFactory.getLog(MessageQueuesForDisk.class);
    /** default size of message. */
    private static final int MESSAGE_SIZE = 150;
    /** Converter byte to MByte. */
    private static final long MB_SIZE = 1048576;
    /** Converter byte to KByte. */
    private static final long KB_SIZE = 1000;
    /** time change. */
    private static final float TIME_CHANGE = 1000f;
    /** For time accumulation. */
    private long writeDiskTime = 0;
    /** Clock time. */
    private long readDiskTime = 0;

    /** The meta data for a bucket. */
    class BucketMeta {
        /** Is on disk flag. */
        public boolean onDiskFlag;
        /** The length of the bucket by Bytes. */
        public long length;
        /** The length of the part of the bucket still in memory by Bytes. */
        public long lengthInMemory;
        /** Number of messages in the bucket. */
        public int count;
        /** Number of messages of the part of the bucket still in memory. */
        public int countInMemory;
        /** The hash map of queues indexed by dstVertexID. */
        public ConcurrentHashMap<String, ConcurrentLinkedQueue<IMessage>> queueMap;
    }

    /**
     * The beta parameter for the proportion of data memory for the graph data,
     * 1-beta for messages data
     */
    private float beta;
    /**
     * The parameter for the percentage of the heap memory for the data memory
     * (graph & messages)
     */
    private float dataPercent;
    /** Hash bucket number. */
    private int hashBucketNumber;
    /** The total space for messages data(Bytes). */
    private long sizeOfMessagesSpace;
    /** The current size of messages data in memory(Bytes). */
    private long sizeOfMessagesDataInMem;
    /** The current count of messages data in memory. */
    private long countOfMessagesDataInMem;
    /** The size of hash maps structures in memory(Bytes). */
    private long sizeOfHashMapsInMem;
    /** The threshold size for messages data(Bytes). */
    private long sizeThreshold;
    /** The threshold count number for messages data. */
    private long countThreshold;
    /** The threshold count number for messages in an incoming bucket. */
    private long countThresholdForBucket;
    /** Accumulate the size fo messages totally.(Bytes). */
    private long totalSizeOfMessages;
    /** The total count number of messages. */
    private long totalCount;
    /** Size of IMessage type instance.(Bytes). */
    private int sizeOfMessage;
    /** Size of empty ConcurrentLinkedQueue<IMessage> object. */
    private int sizeOfEmptyMessageQueue;
    /** Size of a reference. */
    private final int sizeOfRef;
    /** Size of an Integer. */
    private final int sizeOfInteger;
    /** Size of a char. */
    private final int sizeOfChar;
    /** Object sizer. */
    private final ObjectSizer sizer;
    /** BSP Job ID. */
    private BSPJobID jobID;
    /** the partition ID. */
    private int partitionID;
    /** root file of message on disk. */
    private File fileRoot;
    /** message file on disk. */
    private File messagesDataFile;
    /**
     * Outgoing Message Queues hash indexed by <WorkerManagerName:Port>.
     */
    @SuppressWarnings("rawtypes")
    private ConcurrentHashMap<String, ConcurrentLinkedQueue<IMessage>> outgoingQueues;
    /**
     * Hash buckets of hash map of Incoming Message Queues. Hash indexed by
     * vertexID(int).
     */
    private ArrayList<BucketMeta> incomingQueues;
    /**
     * Hash buckets of hash map of Incomed Message Queues. Hash indexed by
     * vertexID(int).
     */
    private ArrayList<BucketMeta> incomedQueues;
    /** Index of the current accessed bucket of incomedQueues. */
    private volatile int currentBucket;
    /** incomed message file lock. */
    private Lock[] incomedFileLocks = null;
    /** incoming message file lock. */
    private Lock[] incomingFileLocks = null;
    /** counter of next outgoing queue. */
    private int nextOutgoingQueueCount = 1;

    /**
     * Constructor.
     * @param job
     * @param partitionID
     */
    public MessageQueuesForDisk(BSPJob job, int partitionID) {
        LOG.info("========== Initializing Message Queues Data For Disk ==========");
        this.dataPercent = job.getMemoryDataPercent(); // Default 0.8
        this.jobID = job.getJobID();
        this.partitionID = partitionID;
        // ratio of graph data
        this.beta = job.getBeta();
        this.hashBucketNumber = job.getHashBucketNumber();
        LOG.info("[beta] = " + this.beta);
        LOG.info("[hashBucketNumber] = " + this.hashBucketNumber);
        // the structure of three kinds message queues
        this.outgoingQueues = new ConcurrentHashMap<String, ConcurrentLinkedQueue<IMessage>>();
        this.incomingQueues = new ArrayList<BucketMeta>(this.hashBucketNumber);
        this.incomedQueues = new ArrayList<BucketMeta>(this.hashBucketNumber);
        for (int i = 0; i < this.hashBucketNumber; i++) {
            BucketMeta meta = new BucketMeta();
            meta.onDiskFlag = false;
            meta.length = 0;
            meta.lengthInMemory = 0;
            meta.count = 0;
            meta.countInMemory = 0;
            meta.queueMap = new ConcurrentHashMap<String, ConcurrentLinkedQueue<IMessage>>();
            this.incomingQueues.add(meta);
            meta = new BucketMeta();
            meta.onDiskFlag = false;
            meta.length = 0;
            meta.lengthInMemory = 0;
            meta.count = 0;
            meta.countInMemory = 0;
            meta.queueMap = new ConcurrentHashMap<String, ConcurrentLinkedQueue<IMessage>>();
            this.incomedQueues.add(meta);
        }
        // Initialize the bucket file locks
        this.incomedFileLocks = new ReentrantLock[this.hashBucketNumber];
        this.incomingFileLocks = new ReentrantLock[this.hashBucketNumber];
        for (int i = 0; i < this.hashBucketNumber; i++) {
            this.incomedFileLocks[i] = new ReentrantLock();
            this.incomingFileLocks[i] = new ReentrantLock();
        }
        // Initialize the size of objects.
        BSPConfiguration conf = new BSPConfiguration();
        if (conf.getInt(Constants.BC_BSP_JVM_VERSION, 32) == 64) {
            sizer = ObjectSizer.forSun64BitsVM();
        } else {
            sizer = ObjectSizer.forSun32BitsVM();
        }
        this.sizeOfRef = sizer.sizeOfRef();
        this.sizeOfInteger = sizer.sizeOf(new Integer(0));
        this.sizeOfChar = sizer.sizeOfChar();
        this.sizeOfEmptyMessageQueue = sizer.sizeOf(new ConcurrentLinkedQueue<IMessage>());
        // Size will be evaluted later based on first m received messages.
        this.sizeOfMessage = MESSAGE_SIZE; // Default
        // Get the memory mxBean.
        MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
        // Get the heap memory usage.
        MemoryUsage memoryUsage = memoryMXBean.getHeapMemoryUsage();
        long maxHeapSize = memoryUsage.getMax();
        LOG.info("[JVM max Heap size] = " + maxHeapSize / MB_SIZE + "MB");
        this.sizeOfMessagesSpace = (long) (maxHeapSize * dataPercent * (1.0f - beta));
        this.sizeThreshold = (long) (sizeOfMessagesSpace);
        this.countThreshold = (long) (sizeThreshold / sizeOfMessage);
        this.countThresholdForBucket = this.countThreshold / (3 * this.hashBucketNumber);
        LOG.info("[size of Messages Space Threshold] = " + this.sizeThreshold / MB_SIZE + "MB");
        LOG.info("[count of Messages In Memory Threshold] = " + this.countThreshold / KB_SIZE + "K");
        this.sizeOfMessagesDataInMem = 0;
        this.countOfMessagesDataInMem = 0;
        this.totalSizeOfMessages = 0;
        this.totalCount = 0;
        this.sizeOfHashMapsInMem = 0;
        this.fileRoot = new File("/tmp/bcbsp/" + this.jobID.toString() + "/" + "partition-" + this.partitionID);
        this.messagesDataFile = new File(this.fileRoot + "/" + "MessagesData");
        // If the root dir does not exit, create it.
        if (!this.fileRoot.exists()) {
            this.fileRoot.mkdirs();
        }
        // If the messages data dir does not exit, create it.
        if (!this.messagesDataFile.exists()) {
            this.messagesDataFile.mkdir();
        }
        // Initialize the current accessed bucket index.
        this.currentBucket = -1;
        LOG.info("===============================================================");
    }

    @Override
    public void clearAllQueues() {
        clearIncomedQueues();
        clearIncomingQueues();
        clearOutgoingQueues();
        this.sizeOfMessagesDataInMem = 0;
        this.countOfMessagesDataInMem = 0;
        this.totalSizeOfMessages = 0;
        this.totalCount = 0;
        this.sizeOfHashMapsInMem = 0;
        this.currentBucket = -1;
        try {
            deleteFile(this.messagesDataFile.toString());
        } catch (IOException e) {
            throw new RuntimeException("[File] Delete file:" + this.messagesDataFile + " failed!", e);
        }
    }

    @Override
    public void clearIncomedQueues() {
        File messagesDataFilebucket;
        for (int i = 0; i < this.hashBucketNumber; i++) {
            messagesDataFilebucket = new File(this.messagesDataFile + "/" + "incomed" + "/" + "bucket-" + i);
            if (messagesDataFilebucket.exists()) {
                if (!messagesDataFilebucket.delete()) {
                    LOG.warn("[File] Delete file:" + messagesDataFilebucket + " fail!");
                }
            }
            BucketMeta meta = this.incomedQueues.get(i);
            this.sizeOfMessagesDataInMem = this.sizeOfMessagesDataInMem - meta.lengthInMemory;
            this.countOfMessagesDataInMem = this.countOfMessagesDataInMem - meta.countInMemory;
            this.sizeOfHashMapsInMem = this.sizeOfHashMapsInMem
                    - (meta.queueMap.size() * (sizeOfRef + sizeOfInteger + sizeOfEmptyMessageQueue));
            meta.onDiskFlag = false;
            meta.length = 0;
            meta.lengthInMemory = 0;
            meta.count = 0;
            meta.countInMemory = 0;
            meta.queueMap.clear();
        }
        messagesDataFilebucket = new File(this.messagesDataFile + "/" + "incomed");
        if (messagesDataFilebucket.exists()) {
            if (!messagesDataFilebucket.delete()) {
                LOG.warn("[File] Delete directory:" + messagesDataFilebucket + " failed!");
            }
        }
    }

    @Override
    public void clearIncomingQueues() {
        File messagesDataFile_bucket;
        for (int i = 0; i < this.hashBucketNumber; i++) {
            messagesDataFile_bucket = new File(this.messagesDataFile + "/" + "incoming" + "/" + "bucket-" + i);
            if (messagesDataFile_bucket.exists()) {
                if (!messagesDataFile_bucket.delete()) {
                    LOG.warn("[File] Delete file:" + messagesDataFile_bucket + " fail!");
                }
            }
            BucketMeta meta = this.incomingQueues.get(i);
            this.sizeOfMessagesDataInMem = this.sizeOfMessagesDataInMem - meta.lengthInMemory;
            this.countOfMessagesDataInMem = this.countOfMessagesDataInMem - meta.countInMemory;
            this.sizeOfHashMapsInMem = this.sizeOfHashMapsInMem
                    - (meta.queueMap.size() * (sizeOfRef + sizeOfInteger + sizeOfEmptyMessageQueue));
            meta.onDiskFlag = false;
            meta.length = 0;
            meta.lengthInMemory = 0;
            meta.count = 0;
            meta.countInMemory = 0;
            meta.queueMap.clear();
        }
        messagesDataFile_bucket = new File(this.messagesDataFile + "/" + "incoming");
        if (messagesDataFile_bucket.exists()) {
            if (!messagesDataFile_bucket.delete()) {
                LOG.warn("[File] Delete directory:" + messagesDataFile_bucket + " failed!");
            }
        }
    }

    @Override
    public void clearOutgoingQueues() {
        Entry<String, ConcurrentLinkedQueue<IMessage>> entry = null;
        Iterator<Entry<String, ConcurrentLinkedQueue<IMessage>>> it = this.outgoingQueues.entrySet().iterator();
        ConcurrentLinkedQueue<IMessage> tmpQueue = null;
        int tmpCount = 0;
        while (it.hasNext()) {
            entry = it.next();
            tmpQueue = entry.getValue();
            tmpCount = tmpQueue.size();
            this.sizeOfMessagesDataInMem = this.sizeOfMessagesDataInMem - (tmpCount * this.sizeOfMessage);
            this.countOfMessagesDataInMem = this.countOfMessagesDataInMem - tmpCount;
            this.sizeOfHashMapsInMem = this.sizeOfHashMapsInMem
                    - (sizeOfRef * 2 + (entry.getKey().length() * sizeOfChar) + sizeOfEmptyMessageQueue);
        }
        this.outgoingQueues.clear();
    }

    /**
     * Delete all files in the filepath and the filepath.
     * @param filepath
     * @throws IOException
     */
    private void deleteFile(String filepath) throws IOException {
        File f = new File(filepath);
        if (f.exists() && f.isDirectory()) {
            if (f.listFiles().length == 0) {
                f.delete();
            } else {
                File[] delFile = f.listFiles();
                int i = f.listFiles().length;
                for (int j = 0; j < i; j++) {
                    if (delFile[j].isDirectory()) {
                        deleteFile(delFile[j].getAbsolutePath());
                    }
                    delFile[j].delete();
                }
            }
        }
    }

    @Override
    public void exchangeIncomeQueues() {
        LOG.info("[==>Clock<==] <MessageQueues: save bucket> totally used " + this.writeDiskTime / TIME_CHANGE
                + " seconds");
        /* Clock */
        LOG.info("[==>Clock<==] <MessageQueues: load bucket> totally used " + this.readDiskTime / TIME_CHANGE
                + " seconds");
        /* Clock */
        LOG.info("[==>Clock<==] <MessageQueues: Disk I/O> totally used "
                + (this.writeDiskTime + this.readDiskTime) / TIME_CHANGE + " seconds");
        /* Clock */
        this.writeDiskTime = 0;
        /* Clock */
        this.readDiskTime = 0;

        /* Clock */
        this.tidyIncomingQueues();
        clearIncomedQueues();
        // Rename the incoming path to incomed path.
        File incomingPath = new File(this.messagesDataFile + "/" + "incoming");
        if (incomingPath.exists()) {
            if (!incomingPath.renameTo(new File(this.messagesDataFile + "/" + "incomed"))) {
                LOG.warn("[MessageQueuesForDisk]:<exchangeIncomeQueues> !"
                        + " Rename the incoming path to incomed path failed!");
            }
        }
        ArrayList<BucketMeta> temp = this.incomedQueues;
        this.incomedQueues = this.incomingQueues;
        this.incomingQueues = temp;
        //    this.showHashBucketsInfo();
        // Reset the accumulative count and size of messages.
        this.totalCount = 0;
        this.totalSizeOfMessages = 0;
        LOG.info("[MessageQueuesForDisk: exchangeIncomeQueues] " + "After exchange, incomed size = "
                + getIncomedQueuesSize() + ", incoming size = " + getIncomingQueuesSize() + ".");
    }

    /**
     * Tidy the incoming queues To save the whole bucket onto disk if it has any
     * part on disk.
     */
    private void tidyIncomingQueues() {
        for (int i = 0; i < this.hashBucketNumber; i++) {
            BucketMeta meta = this.incomingQueues.get(i);
            if (meta.countInMemory < meta.count) {
                try {
                    saveBucket(this.incomingQueues, i, "incoming");
                } catch (IOException e) {
                    throw new RuntimeException("[MessageQueuesForDisk:tidyIncomingQueues]", e);
                }
            }
        }
    }

    @Override
    public int getIncomedQueuesSize() {
        int size = 0;
        for (int i = 0; i < this.hashBucketNumber; i++) {
            size = size + this.incomedQueues.get(i).count;
        }
        return size;
    }

    private int getIncomedQueuesSizeInMem() {
        int size = 0;
        for (int i = 0; i < this.hashBucketNumber; i++) {
            size = size + this.incomedQueues.get(i).countInMemory;
        }
        return size;
    }

    @Override
    public int getIncomingQueuesSize() {
        int size = 0;
        for (int i = 0; i < this.hashBucketNumber; i++) {
            size = size + this.incomingQueues.get(i).count;
        }
        return size;
    }

    private int getIncomingQueuesSizeInMem() {
        int size = 0;
        for (int i = 0; i < this.hashBucketNumber; i++) {
            size = size + this.incomingQueues.get(i).countInMemory;
        }
        return size;
    }

    /**
     * Current strategy is first find the max bucket in memory, and second get the
     * max queue in the bucket.
     */
    @Override
    public String getMaxIncomingQueueIndex() {
        int maxSize = 0;
        int maxBucket = 0;
        // Find the max buckets in memory.
        for (int i = 0; i < this.hashBucketNumber; i++) {
            BucketMeta meta = this.incomingQueues.get(i);
            if (meta.count > maxSize && !meta.onDiskFlag) {
                maxSize = meta.count;
                maxBucket = i;
            }
        }
        // Find the max queue in the maxBucket.
        String maxQueueIndex = null;
        ConcurrentHashMap<String, ConcurrentLinkedQueue<IMessage>> queueMap = this.incomingQueues
                .get(maxBucket).queueMap;
        Entry<String, ConcurrentLinkedQueue<IMessage>> entry = null;
        Iterator<Entry<String, ConcurrentLinkedQueue<IMessage>>> it = queueMap.entrySet().iterator();
        maxSize = 0;
        while (it.hasNext()) {
            entry = it.next();
            if (entry.getValue().size() > maxSize) {
                maxSize = entry.getValue().size();
                maxQueueIndex = entry.getKey();
            }
        }
        return maxQueueIndex;
    }

    /**
     * Current strategy is get the max outgoing queue in memory.
     */
    @Override
    public String getMaxOutgoingQueueIndex() {
        String maxIndex = null;
        long maxSize = 0;
        synchronized (this.outgoingQueues) {
            Entry<String, ConcurrentLinkedQueue<IMessage>> entry = null;
            Iterator<Entry<String, ConcurrentLinkedQueue<IMessage>>> it = this.outgoingQueues.entrySet().iterator();
            ConcurrentLinkedQueue<IMessage> tmpQueue = null;
            while (it.hasNext()) {
                entry = it.next();
                tmpQueue = entry.getValue();
                if (tmpQueue.size() > maxSize) {
                    maxSize = tmpQueue.size();
                    maxIndex = entry.getKey();
                }
            }
        }
        return maxIndex;
    }

    @Override
    public int getOutgoingQueuesSize() {
        int size = 0;
        Entry<String, ConcurrentLinkedQueue<IMessage>> entry = null;
        Iterator<Entry<String, ConcurrentLinkedQueue<IMessage>>> it = this.outgoingQueues.entrySet().iterator();
        ConcurrentLinkedQueue<IMessage> tmpQueue = null;
        while (it.hasNext()) {
            entry = it.next();
            tmpQueue = entry.getValue();
            size = size + tmpQueue.size();
        }
        return size;
    }

    @Override
    public void incomeAMessage(String dstVertexID, IMessage msg) {
        // Evaluate the length of the msg, 16 means 1 long and 2 int.
        int length = sizer.sizeOf(msg) + this.sizeOfRef;
        // Accumulate the total size of messages.
        this.totalSizeOfMessages = this.totalSizeOfMessages + length;
        this.totalCount = this.totalCount + 1;
        this.sizeOfMessagesDataInMem = this.sizeOfMessagesDataInMem + length;
        // Message number in memory
        this.countOfMessagesDataInMem = this.countOfMessagesDataInMem + 1;
        // Get the hash bucket index.
        int hashCode = dstVertexID.hashCode();
        int hashIndex = hashCode % this.hashBucketNumber; // bucket index
        hashIndex = (hashIndex < 0 ? hashIndex + this.hashBucketNumber : hashIndex);
        // Update the bucket meta data.
        BucketMeta meta = this.incomingQueues.get(hashIndex);
        meta.count = meta.count + 1;
        meta.countInMemory = meta.countInMemory + 1;
        meta.length = meta.length + length;
        meta.lengthInMemory = meta.lengthInMemory + length;
        // Add the msg into the incoming queue for the dstVertexID.
        ConcurrentLinkedQueue<IMessage> incomingQueue = meta.queueMap.get(dstVertexID);
        if (incomingQueue == null) {
            incomingQueue = new ConcurrentLinkedQueue<IMessage>();
            this.sizeOfHashMapsInMem = this.sizeOfHashMapsInMem
                    + (sizeOfRef * 2 + (dstVertexID.length() * sizeOfChar) + sizeOfEmptyMessageQueue);
        }
        incomingQueue.add(msg);
        meta.queueMap.put(dstVertexID, incomingQueue);
        // On a new message added.
        onMessageIncomed();
    }

    @Override
    public void outgoAMessage(String outgoingIndex, IMessage msg) {
        // Evaluate the length of the msg, 16 means 1 long and 2 int.
        int length = sizer.sizeOf(msg) + this.sizeOfRef;
        // Accumulate the total size of messages.
        this.totalSizeOfMessages = this.totalSizeOfMessages + length;
        this.totalCount = this.totalCount + 1;
        this.sizeOfMessagesDataInMem = this.sizeOfMessagesDataInMem + length;
        // Message number in memory
        this.countOfMessagesDataInMem = this.countOfMessagesDataInMem + 1;
        ConcurrentLinkedQueue<IMessage> queue = this.outgoingQueues.get(outgoingIndex);
        if (queue == null) {
            queue = new ConcurrentLinkedQueue<IMessage>();
            this.sizeOfHashMapsInMem = this.sizeOfHashMapsInMem
                    + (sizeOfRef * 2 + (outgoingIndex.length() * sizeOfChar) + sizeOfEmptyMessageQueue);
        }
        queue.add(msg);
        this.outgoingQueues.put(outgoingIndex, queue);
        // On a new message outgoed.
        onMessageOutgoed();
    }

    @Override
    @SuppressWarnings("unchecked")
    public ConcurrentLinkedQueue<IMessage> removeIncomedQueue(String dstVertID) {
        ConcurrentLinkedQueue<IMessage> incomedQueue = null;
        // Get the hash bucket index.
        int hashCode = dstVertID.hashCode();
        int hashIndex = hashCode % this.hashBucketNumber; // bucket index
        hashIndex = (hashIndex < 0 ? hashIndex + this.hashBucketNumber : hashIndex);
        BucketMeta meta = this.incomedQueues.get(hashIndex);
        // The bucket is on disk.
        if (meta.onDiskFlag) {
            this.incomedFileLocks[hashIndex].lock();
            /** Lock */
            try {
                loadBucket(this.incomedQueues, hashIndex, "incomed");
            } catch (IOException e) {
                LOG.info("==> bucket-" + hashIndex + ", VertexID = " + dstVertID);
                LOG.info("size = " + meta.queueMap.get(dstVertID).size());
                throw new RuntimeException("==> bucket-" + hashIndex + ", VertexID = " + dstVertID, e);
            } finally {
                this.incomedFileLocks[hashIndex].unlock();
                /** Unlock */
            }
        }
        meta = this.incomedQueues.get(hashIndex);
        this.currentBucket = hashIndex;
        incomedQueue = meta.queueMap.remove(dstVertID);
        if (incomedQueue == null) {
            incomedQueue = new ConcurrentLinkedQueue<IMessage>();
        }
        int removedCount = incomedQueue.size();
        long removedLength = removedCount * this.sizeOfMessage;
        // Update the meta data.
        meta.count = meta.count - removedCount;
        meta.countInMemory = meta.countInMemory - removedCount;
        meta.length = meta.length - removedLength;
        meta.lengthInMemory = meta.lengthInMemory - removedLength;
        this.sizeOfMessagesDataInMem = this.sizeOfMessagesDataInMem - removedLength;
        this.countOfMessagesDataInMem = this.countOfMessagesDataInMem - removedCount;
        this.sizeOfHashMapsInMem = this.sizeOfHashMapsInMem
                - (sizeOfRef * 2 + (dstVertID.length() * sizeOfChar) + sizeOfEmptyMessageQueue);
        return incomedQueue;
    }

    @Override
    public ConcurrentLinkedQueue<IMessage> removeIncomingQueue(String dstVerID) {
        ConcurrentLinkedQueue<IMessage> incomingQueue = null;
        // Get the hash bucket index.
        int hashCode = dstVerID.hashCode();
        int hashIndex = hashCode % this.hashBucketNumber; // bucket index
        hashIndex = (hashIndex < 0 ? hashIndex + this.hashBucketNumber : hashIndex);
        BucketMeta meta = this.incomingQueues.get(hashIndex);
        // The bucket is on disk.
        if (meta.onDiskFlag) {
            this.incomingFileLocks[hashIndex].lock();
            /** Lock */
            try {
                loadBucket(this.incomingQueues, hashIndex, "incoming");
            } catch (IOException e) {
                throw new RuntimeException("[MessageQueuesForDisk:removeIncomingQueue]", e);
            } finally {
                this.incomingFileLocks[hashIndex].unlock();
                /** Unlock */
            }
        }
        meta = this.incomingQueues.get(hashIndex);
        incomingQueue = meta.queueMap.remove(dstVerID);
        if (incomingQueue == null) {
            incomingQueue = new ConcurrentLinkedQueue<IMessage>();
        }
        int removedCount = incomingQueue.size();
        long removedLength = removedCount * this.sizeOfMessage;
        // Update the meta data.
        meta.count = meta.count - removedCount;
        meta.countInMemory = meta.countInMemory - removedCount;
        meta.length = meta.length - removedLength;
        meta.lengthInMemory = meta.lengthInMemory - removedLength;
        this.sizeOfMessagesDataInMem = this.sizeOfMessagesDataInMem - removedLength;
        this.countOfMessagesDataInMem = this.countOfMessagesDataInMem - removedCount;
        this.sizeOfHashMapsInMem = this.sizeOfHashMapsInMem
                - (sizeOfRef * 2 + (dstVerID.length() * sizeOfChar) + sizeOfEmptyMessageQueue);
        return incomingQueue;
    }

    @Override
    public ConcurrentLinkedQueue<IMessage> removeOutgoingQueue(String index) {
        ConcurrentLinkedQueue<IMessage> outgoingQueue = null;
        synchronized (this.outgoingQueues) {
            outgoingQueue = this.outgoingQueues.remove(index);
        }
        if (outgoingQueue == null) {
            return null;
        }
        int removedCount = outgoingQueue.size();
        long removeLength = this.sizeOfMessage * removedCount;
        this.sizeOfMessagesDataInMem = this.sizeOfMessagesDataInMem - removeLength;
        this.countOfMessagesDataInMem = this.countOfMessagesDataInMem - removedCount;
        this.sizeOfHashMapsInMem = this.sizeOfHashMapsInMem
                - (sizeOfRef * 2 + (index.length() * sizeOfChar) + sizeOfEmptyMessageQueue);
        return outgoingQueue;
    }

    /**
     * On a new message incomed.
     */
    private void onMessageIncomed() {
        if (this.totalCount % 10000 == 1) {
            // Evaluate the size of a single message.
            this.sizeOfMessage = (int) (this.totalSizeOfMessages / this.totalCount);
            // Update the count threshold.
            this.countThreshold = this.sizeThreshold / this.sizeOfMessage;
        }
        // To check the memory occupied, if over the threshold, cache some buckets
        // or queues onto disk.
        if (this.countOfMessagesDataInMem >= this.countThreshold) {
            // First get the longest bucket of incoming queues.
            // Find the longest hash buck index.
            int incomingIndex = findLongestBucket(this.incomingQueues);
            // Message number of longest hash buck.
            long incomingCountInMem = this.incomingQueues.get(incomingIndex).countInMemory;
            int incomedIndex = findLongestBucketWithOut(this.incomedQueues, this.currentBucket);
            long incomedCountInMem = this.incomedQueues.get(incomedIndex).countInMemory;
            // If there are still in memory buckets of incomingQueues.
            if (incomingCountInMem >= this.countThresholdForBucket) {
                this.incomingFileLocks[incomingIndex].lock();
                /** Lock */
                try {
                    saveBucket(this.incomingQueues, incomingIndex, "incoming");
                } catch (IOException e) {
                    throw new RuntimeException("[MessageQueuesForDisk:OnMessageAdded]", e);
                } finally {
                    this.incomingFileLocks[incomingIndex].unlock();
                    /** Unlock */
                }
            } else if (incomedCountInMem >= this.countThresholdForBucket) {
                // Else, begin to save incomed queues.
                // Find the longest incomed queue in memory wihtout the current accessed
                // bucket.
                this.incomedFileLocks[incomedIndex].lock();
                /** Lock */
                try {
                    saveBucket(this.incomedQueues, incomedIndex, "incomed");
                } catch (IOException e) {
                    throw new RuntimeException("[MessageQueuesForDisk:OnMessageAdded]", e);
                } finally {
                    this.incomedFileLocks[incomedIndex].unlock();
                    /** Unlock */
                }
            }
        }
    }

    /**
     * On a new message outgoed.
     */
    private void onMessageOutgoed() {
        if (this.totalCount % 10000 == 1) {
            // Evaluate the size of a single message.
            this.sizeOfMessage = (int) (this.totalSizeOfMessages / this.totalCount);
            // Update the count threshold.
            this.countThreshold = this.sizeThreshold / this.sizeOfMessage;
        }
        // To check the memory occupied, if over the threshold, cache some buckets
        // or queues onto disk.
        if (this.countOfMessagesDataInMem >= this.countThreshold) {
            // First get the longest bucket of incoming queues.
            int incomingIndex = findLongestBucket(this.incomingQueues);
            long incomingCountInMem = this.incomingQueues.get(incomingIndex).countInMemory;
            int incomedIndex = findLongestBucketWithOut(this.incomedQueues, this.currentBucket);
            long incomedCountInMem = this.incomedQueues.get(incomedIndex).countInMemory;
            // If there are still in memory buckets of incomingQueues.
            if (incomingCountInMem >= this.countThresholdForBucket) {
                this.incomingFileLocks[incomingIndex].lock();
                /* Lock */
                try {
                    saveBucket(this.incomingQueues, incomingIndex, "incoming");
                } catch (IOException e) {
                    LOG.error("[MessageQueuesForDisk:OnMessageAdded]", e);
                } finally {
                    this.incomingFileLocks[incomingIndex].unlock();
                    /* Unlock */
                }
            } else if (incomedCountInMem >= this.countThresholdForBucket) {
                // Else, begin to save incomed queues.
                // Find the longest incomed queue in memory wihtout the current accessed
                // bucket.
                this.incomedFileLocks[incomedIndex].lock();
                /** Lock */
                try {
                    saveBucket(this.incomedQueues, incomedIndex, "incomed");
                } catch (IOException e) {
                    LOG.error("[MessageQueuesForDisk:OnMessageAdded]", e);

                } finally {
                    this.incomedFileLocks[incomedIndex].unlock();
                    /** Unlock */
                }
            } else {
                try {
                    Thread.sleep(500);
                } catch (Exception e) {
                    LOG.error("[Sender] caught: ", e);
                }
            }
        }
    }

    /**
     * Find the longest bucket now in memory of the queuesBuckets.
     * @param queuesBuckets
     * @return int bucketIndex of the longest bucket in memory.
     */
    private synchronized int findLongestBucket(ArrayList<BucketMeta> queueBucks) {
        int bucketIndex = 0;
        long longestLength = 0;
        for (int i = 0; i < this.hashBucketNumber; i++) {
            BucketMeta meta = queueBucks.get(i);
            if (meta.lengthInMemory > longestLength) {
                longestLength = meta.lengthInMemory;
                bucketIndex = i;
            }
        }
        return bucketIndex;
    }

    /**
     * Find the longest bucket now in memory of the queuesBuckets without the
     * bucket indexed by withoutIndex.
     * @param queuesBuckets
     * @param withoutIndex
     * @return int bucketIndex of the longest bucket in memory.
     */
    private synchronized int findLongestBucketWithOut(ArrayList<BucketMeta> queuesBuckets, int withoutIndex) {
        int bucketIndex = 0;
        long longestLength = 0;
        for (int i = 0; i < this.hashBucketNumber; i++) {
            // Skip the bucket that is being accessed.
            if (i == withoutIndex) {
                continue;
            }
            BucketMeta meta = queuesBuckets.get(i);
            // Find the longest but lengthInMemory is not zero.
            if (meta.lengthInMemory > longestLength) {
                longestLength = meta.lengthInMemory;
                bucketIndex = i;
            }
        }
        return bucketIndex;
    }

    /**
     * Cache the bucket of messages indexed by bucketIndex onto disk file.
     * @param queuesBuckets
     * @param bucketIndex
     * @param queuePath
     * @throws IOException
     */
    private void saveBucket(ArrayList<BucketMeta> queuesBuckets, int bucketIndex, String queuePath)
            throws IOException {
        if (queuesBuckets.get(bucketIndex).countInMemory < this.countThresholdForBucket) {
            return;
        }
        LOG.info("[MessageQueuesForDisk] is saving the [" + queuePath + " Bucket-" + bucketIndex + "] >>> size = "
                + queuesBuckets.get(bucketIndex).countInMemory + ".");
        long start = System.currentTimeMillis();
        /* Clock */
        File messagesDataFileBucket;
        FileWriter fwMessagesData;
        BufferedWriter bwMessagesData;
        File messagesDataFileQueue = new File(this.messagesDataFile + "/" + queuePath);
        if (!messagesDataFileQueue.exists()) {
            if (!messagesDataFileQueue.mkdir()) {
                throw new IOException("Make dir " + messagesDataFileQueue + " fail!");
            }
        }
        messagesDataFileBucket = new File(messagesDataFileQueue + "/" + "bucket-" + bucketIndex);
        boolean isNewFile = false;
        // The bucket file does not exit, create it.
        if (!messagesDataFileBucket.exists()) {
            if (!messagesDataFileBucket.createNewFile()) {
                throw new IOException("Create bucket file" + messagesDataFileBucket + " failed!");
            }
            isNewFile = true;
        }
        // Append to the bucket file by line.
        fwMessagesData = new FileWriter(messagesDataFileBucket, true);
        bwMessagesData = new BufferedWriter(fwMessagesData, 65536);
        if (isNewFile) {
            // Write the file header.
            bwMessagesData.write(Constants.MSG_BUCKET_FILE_HEADER + "-" + queuePath + "-" + bucketIndex);
        }
        ConcurrentHashMap<String, ConcurrentLinkedQueue<IMessage>> queueMap = queuesBuckets
                .get(bucketIndex).queueMap;
        ConcurrentLinkedQueue<IMessage> tempQueue = null;
        Entry<String, ConcurrentLinkedQueue<IMessage>> entry = null;
        Iterator<Entry<String, ConcurrentLinkedQueue<IMessage>>> it = queueMap.entrySet().iterator();
        // Traverse the map of queues and cache them to disk file.
        while (it.hasNext()) {
            entry = it.next();
            String key = entry.getKey();
            tempQueue = entry.getValue();
            if (tempQueue.size() <= 0) {
                continue;
            }
            bwMessagesData.newLine();
            bwMessagesData.write(key + Constants.KV_SPLIT_FLAG + queueToString(tempQueue));
            this.sizeOfHashMapsInMem = this.sizeOfHashMapsInMem
                    - (sizeOfRef + sizeOfInteger + sizeOfEmptyMessageQueue);
        } // while
        bwMessagesData.close();
        fwMessagesData.close();
        // Update the meta data of the bucket.
        BucketMeta meta = queuesBuckets.get(bucketIndex);
        // Update the size of messages data in memory.
        this.sizeOfMessagesDataInMem = this.sizeOfMessagesDataInMem - meta.lengthInMemory;
        this.countOfMessagesDataInMem = this.countOfMessagesDataInMem - meta.countInMemory;
        meta.onDiskFlag = true;
        meta.lengthInMemory = 0;
        meta.countInMemory = 0;
        meta.queueMap.clear();
        this.writeDiskTime = this.writeDiskTime + (System.currentTimeMillis() - start);
    }

    /**
     * Load bucket from disk.
     * @param queuesBuckets
     * @param bucketIndex
     * @param queuePath
     * @throws IOException
     */
    private void loadBucket(ArrayList<BucketMeta> queuesBuckets, int bucketIndex, String queuePath)
            throws IOException {
        LOG.info("[MessageQueuesForDisk] is loading the [" + queuePath + " Bucket-" + bucketIndex + "] <<< size = "
                + queuesBuckets.get(bucketIndex).count + ".");
        long start = System.currentTimeMillis();
        /** Clock */
        File messagesDataFileBucket;
        FileReader frMessagesData;
        BufferedReader brMessagesData;
        messagesDataFileBucket = new File(this.messagesDataFile + "/" + queuePath + "/" + "bucket-" + bucketIndex);
        if (!messagesDataFileBucket.exists()) {
            throw new IOException("Bucket file" + messagesDataFileBucket + " does not exit!");
        }
        // Open file readers.
        frMessagesData = new FileReader(messagesDataFileBucket);
        brMessagesData = new BufferedReader(frMessagesData);
        // Read the file header.
        @SuppressWarnings("unused")
        String bucketHeader = brMessagesData.readLine();
        ConcurrentHashMap<String, ConcurrentLinkedQueue<IMessage>> queueMap = queuesBuckets
                .get(bucketIndex).queueMap;
        if (queueMap == null) {
            queueMap = new ConcurrentHashMap<String, ConcurrentLinkedQueue<IMessage>>();
        }
        String buffer;
        while ((buffer = brMessagesData.readLine()) != null) {
            String[] queueBuffer = buffer.split(Constants.KV_SPLIT_FLAG);
            if (queueBuffer[0] == "") {
                LOG.warn("[MessageQueuesForDisk] readLine = " + buffer);
            }
            String key = queueBuffer[0];
            ConcurrentLinkedQueue<IMessage> queue = queueMap.get(key);
            if (queue == null) {
                queue = stringToQueue(queueBuffer[1]);
                this.sizeOfHashMapsInMem = this.sizeOfHashMapsInMem
                        + (sizeOfRef + sizeOfInteger + sizeOfEmptyMessageQueue);
            } else {
                queue.addAll(stringToQueue(queueBuffer[1]));
            }
            queueMap.put(key, queue);
        }
        queuesBuckets.get(bucketIndex).queueMap = queueMap;
        brMessagesData.close();
        frMessagesData.close();
        // Update the meta data of the bucket.
        BucketMeta meta = queuesBuckets.get(bucketIndex);
        // Update the size of messages data in memory.
        this.sizeOfMessagesDataInMem = this.sizeOfMessagesDataInMem + (meta.length - meta.lengthInMemory);
        this.countOfMessagesDataInMem = this.countOfMessagesDataInMem + (meta.count - meta.countInMemory);
        meta.onDiskFlag = false;
        meta.lengthInMemory = meta.length;
        meta.countInMemory = meta.count;
        queuesBuckets.set(bucketIndex, meta);
        if (!messagesDataFileBucket.delete()) {
            throw new IOException("Bucket file delete failed!");
        }
        this.readDiskTime = this.readDiskTime + (System.currentTimeMillis() - start);
        /** Clock */
    }

    /**
     * ToString method for message queue.
     * @return String
     */
    private String queueToString(ConcurrentLinkedQueue<IMessage> queue) {
        String buffer;
        buffer = queue.poll().intoString();
        IMessage msg;
        while ((msg = queue.poll()) != null) {
            buffer = buffer + Constants.SPACE_SPLIT_FLAG + msg.intoString();
        }
        return buffer;
    }

    /**
     * Change String to queue.
     * @return ConcurrentLinkedQueue<IMessage>
     */
    @SuppressWarnings("unchecked")
    private ConcurrentLinkedQueue<IMessage> stringToQueue(String queueBuffer) {
        ConcurrentLinkedQueue<IMessage> queue = new ConcurrentLinkedQueue<IMessage>();
        if (queueBuffer != null) {
            String[] msgs = queueBuffer.split(Constants.SPACE_SPLIT_FLAG);
            for (int i = 0; i < msgs.length; i++) {
                // Note BSPMessage Is Temporayly Left.Should Be class.newInstance().
                // class Should Be Transferred In.
                IMessage msg = new BSPMessage();
                msg.fromString(msgs[i]);
                queue.add(msg);
            }
        }
        return queue;
    }

    @Override
    public int getIncomingQueueSize(String dstVertexID) {
        ConcurrentLinkedQueue<IMessage> incomingQueue = null;
        // Get the hash bucket index.
        int hashCode = dstVertexID.hashCode();
        int hashIndex = hashCode % this.hashBucketNumber; // bucket index
        hashIndex = (hashIndex < 0 ? hashIndex + this.hashBucketNumber : hashIndex);
        BucketMeta meta = this.incomingQueues.get(hashIndex);
        incomingQueue = meta.queueMap.get(dstVertexID);
        if (incomingQueue != null) {
            return incomingQueue.size();
        } else {
            return 0;
        }
    }

    @Override
    public int getOutgoingQueueSize(String index) {
        ConcurrentLinkedQueue<IMessage> queue = null;
        synchronized (this.outgoingQueues) {
            queue = this.outgoingQueues.get(index);
        }
        if (queue != null) {
            return queue.size();
        } else {
            return 0;
        }
    }

    /** Show the information of Memory. */
    public void showMemoryInfo() {
        LOG.info("---------------- Memory Info for Messages ------------------");
        MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
        MemoryUsage memoryUsage = memoryMXBean.getHeapMemoryUsage();
        long used = memoryUsage.getUsed();
        long committed = memoryUsage.getCommitted();
        LOG.info("<Real> [Memory used] = " + used / MB_SIZE + "MB");
        LOG.info("<Real> [Memory committed] = " + committed / MB_SIZE + "MB");
        LOG.info("<Evaluate> [size of Message] = " + this.sizeOfMessage + "B");
        LOG.info("<Evaluate> [size of Messages Data In Memory] = " + this.sizeOfMessagesDataInMem / MB_SIZE + "MB");
        LOG.info("<Evaluate> [size of HashMaps In Memory] = " + this.sizeOfHashMapsInMem / MB_SIZE + "MB");
        LOG.info("<Evaluate> [size of Messages Data Threshold] = " + this.sizeThreshold / MB_SIZE + "MB");
        LOG.info(
                "<Evaluate> [count of Messages Data In Memory] = " + this.countOfMessagesDataInMem / KB_SIZE + "K");
        LOG.info("<Evaluate> [count of Messages Data Threshold] = " + this.countThreshold / KB_SIZE + "K");
        LOG.info("----------------- ------------------------ -----------------");
        // showHashBucketsInfo();
    }

    /** Show the information of Hash Bucket. */
    private void showHashBucketsInfo() {
        LOG.info("------------ Buckets Info of Messages ------------");
        LOG.info("[Incoming Queues]:");
        long maxCount = 0;
        for (int i = 0; i < this.incomingQueues.size(); i++) {
            BucketMeta meta = this.incomingQueues.get(i);
            if (meta.count > maxCount) {
                maxCount = meta.count;
            }
        }
        for (int i = 0; i < this.incomingQueues.size(); i++) {
            BucketMeta meta = this.incomingQueues.get(i);
            String out = "[Incoming-" + i + "] ";
            if (meta.onDiskFlag) {
                out = out + "OnDisk ";
            } else {
                out = out + "       ";
            }
            out = out + meta.lengthInMemory / MB_SIZE + "MB - " + meta.length / MB_SIZE + "MB ";
            int nMax = 30;
            int nAll = (int) (nMax * ((float) meta.count / (float) maxCount));
            int nMem = (int) (nAll * ((float) meta.countInMemory / (float) meta.count));
            int nDisk = nAll - nMem;
            for (int j = 0; j < nMem; j++) {
                out = out + "-";
            }
            for (int j = 0; j < nDisk; j++) {
                out = out + "*";
            }
            LOG.info(out);
        }
        LOG.info("[Incomed Queues]:");
        maxCount = 0;
        for (int i = 0; i < this.incomedQueues.size(); i++) {
            BucketMeta meta = this.incomedQueues.get(i);
            if (meta.count > maxCount) {
                maxCount = meta.count;
            }
        }
        for (int i = 0; i < this.incomedQueues.size(); i++) {
            BucketMeta meta = this.incomedQueues.get(i);
            String out = "[Incomed-" + i + "] ";
            if (meta.onDiskFlag) {
                out = out + "OnDisk ";
            } else {
                out = out + "       ";
            }
            out = out + meta.lengthInMemory / MB_SIZE + "MB - " + meta.length / MB_SIZE + "MB ";
            int nMax = 30;
            int nAll = (int) (nMax * ((float) meta.count / (float) maxCount));
            int nMem = (int) (nAll * ((float) meta.countInMemory / (float) meta.count));
            int nDisk = nAll - nMem;
            for (int j = 0; j < nMem; j++) {
                out = out + "-";
            }
            for (int j = 0; j < nDisk; j++) {
                out = out + "*";
            }
            LOG.info(out);
        }
        LOG.info("------------ --------------------- ------------");
    }

    /** Show the information of Memory Occupation. */
    @SuppressWarnings("unused")
    private void showInMemoryOccupation() {
        LOG.info("------------ Messages In Memory ------------");
        float outgoPercent = (float) this.getOutgoingQueuesSize() / (float) this.countOfMessagesDataInMem;
        float incomedPercent = (float) this.getIncomedQueuesSizeInMem() / (float) this.countOfMessagesDataInMem;
        float incomingPercent = (float) this.getIncomingQueuesSizeInMem() / (float) this.countOfMessagesDataInMem;
        int maxHeight = 50;
        LOG.info("[Memory Threshold] = " + this.countThreshold / KB_SIZE + "K");
        LOG.info("[In memory Now] = " + this.countOfMessagesDataInMem / KB_SIZE + "K");
        LOG.info("[Outgo] = " + (int) (outgoPercent * 100) + "%, [Incomed] = " + (int) (incomedPercent * 100)
                + "%, [Incoming] = " + (int) (incomingPercent * 100) + "%");
        int outgoHeight = (int) (maxHeight * outgoPercent);
        int incomedHeight = (int) (maxHeight * incomedPercent);
        int incomingHeight = (int) (maxHeight * incomingPercent);
        for (int i = 0; i < outgoHeight; i++) {
            LOG.info("oooooooooooooooooooo");
        }
        for (int i = 0; i < incomedHeight; i++) {
            LOG.info("********************");
        }
        for (int i = 0; i < incomingHeight; i++) {
            LOG.info("!!!!!!!!!!!!!!!!!!!!");
        }
        LOG.info("--------------------------------------------");
    }

    @Override
    public String getNextOutgoingQueueIndex() throws Exception {
        String nextIndex = null;
        synchronized (this.outgoingQueues) {
            int size = this.outgoingQueues.size();
            if (size == 0) {
                return null;
            }
            if (this.nextOutgoingQueueCount > size) {
                this.nextOutgoingQueueCount = 1;
            }
            Entry<String, ConcurrentLinkedQueue<IMessage>> entry = null;
            Iterator<Entry<String, ConcurrentLinkedQueue<IMessage>>> it = this.outgoingQueues.entrySet().iterator();
            for (int i = 0; i < this.nextOutgoingQueueCount; i++) {
                entry = it.next();
                nextIndex = entry.getKey();
            }
            this.nextOutgoingQueueCount++;
        }
        return nextIndex;
    }

    /* Zhicheng Liu added */
    @Override
    public IMessage getAMessage() {
        ConcurrentLinkedQueue<IMessage> incomingQueue = null;
        // Get the hash bucket index.
        int hashIndex = (int) (Math.random() * 10);
        BucketMeta meta = this.incomingQueues.get(hashIndex);
        // The bucket is on disk.
        if (meta.onDiskFlag) {
            this.incomingFileLocks[hashIndex].lock();
            /* Lock */
            try {
                loadBucket(this.incomingQueues, hashIndex, "incoming");
            } catch (IOException e) {
                LOG.error("[MessageQueuesForDisk:removeIncomingQueue]", e);
            } finally {
                this.incomingFileLocks[hashIndex].unlock();
                /** Unlock */
            }
        }
        meta = this.incomingQueues.get(hashIndex);
        // incomingQueue = meta.queueMap.remove(dstVertexID);
        Iterator<String> it = meta.queueMap.keySet().iterator();
        IMessage mes = null;
        while (it.hasNext()) {
            String vertexID = it.next();
            if (vertexID != null) {
                ConcurrentLinkedQueue<IMessage> messages = meta.queueMap.get(vertexID);
                if (messages == null) {
                    continue;
                }
                mes = messages.peek();
            }
            break;
        }
        return mes;
    }

    public int getHashBucketNumber() {
        return hashBucketNumber;
    }

    public void setHashBucketNumber(int hashBucketNumber) {
        this.hashBucketNumber = hashBucketNumber;
    }

    public int getCurrentBucket() {
        return currentBucket;
    }

    public void setCurrentBucket(int currentBucket) {
        this.currentBucket = currentBucket;
    }

    public ConcurrentHashMap<String, ConcurrentLinkedQueue<IMessage>> getOutgoingQueues() {
        return outgoingQueues;
    }

    public void setOutgoingQueues(ConcurrentHashMap<String, ConcurrentLinkedQueue<IMessage>> outgoingQueues) {
        this.outgoingQueues = outgoingQueues;
    }
}