org.apache.hadoop.hive.ql.exec.persistence.BytesBytesMultiHashMap.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hive.ql.exec.persistence.BytesBytesMultiHashMap.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hive.ql.exec.persistence;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.common.MemoryEstimate;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.debug.Utils;
import org.apache.hadoop.hive.serde2.ByteStream.RandomAccessOutput;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.WriteBuffers;

import com.google.common.annotations.VisibleForTesting;

/**
 * HashMap that maps byte arrays to byte arrays with limited functionality necessary for
 * MapJoin hash tables, with small memory overhead. Supports multiple values for single key.
 * Values can be added for key (but cannot be removed); values can be gotten for the key.
 * Some things (like entrySet) are easy to add; some e.g. deletion are pretty hard to do well.
 * Additionally, for each key it contains a magic "state byte" which is not part of the key and
 * can be updated on every put for that key. That is really silly, we use it to store aliasFilter.
 * Magic byte could be removed for generality.
 * Initially inspired by HPPC LongLongOpenHashMap; however, the code is almost completely reworked
 * and there's very little in common left save for quadratic probing (and that with some changes).
 */
public final class BytesBytesMultiHashMap implements MemoryEstimate {
    public static final Logger LOG = LoggerFactory.getLogger(BytesBytesMultiHashMap.class);

    /*
     * This hashtable stores "references" in an array of longs;  index in the array is hash of
     * the key; these references point into infinite byte buffer (see below). This buffer contains
     * records written one after another. There are several simple record formats.
     * - single record for the key
     *    [key bytes][value bytes][vlong value length][vlong key length][padding]
     *    We leave padding to ensure we have at least 5 bytes after key and value.
     * - first of multiple records for the key (updated from "single value for the key")
     *    [key bytes][value bytes][5-byte long offset to a list start record]
     *  - list start record
     *    [vlong value length][vlong key length][5-byte long offset to the 2nd list record]
     *    Lengths are preserved from the first record. Offset is discussed above.
     *  - subsequent values in the list
     *    [value bytes][value length][vlong relative offset to next record].
     *
     * In summary, because we have separate list record, we have very little list overhead for
     * the typical case of primary key join, where there's no list for any key; large lists also
     * don't have a lot of relative overhead (also see the todo below).
     *
     * So the record looks as follows for one value per key (hash is fixed, 4 bytes, and is
     * stored to expand w/o rehashing, and to more efficiently deal with collision
     *
     *             i = key hash
     *           ._______.
     * REFS: ... |offset | ....
     *           `--\----'
     *               `-------------.
     *                            \|/
     *          .______._____._____'__.__._.
     * WBS: ... | hash | key | val |vl|kl| | ....
     *          `------'-----'-----'--'--'-'
     *
     * After that refs don't change so they are not pictured.
     * When we add the 2nd value, we rewrite lengths with relative offset to the list start record.
     * That way, the first record points to the "list record".
     *                         ref .---------.
     *                         \|/ |        \|/
     *       .______._____._____'__|___.     '__.__.______.
     * WBS:  | hash | key | val |offset| ... |vl|kl|      |
     *       `------'-----'-----'------'     '--'--'------'
     * After that refs don't change so they are not pictured. List record points to the 2nd value.
     *                         ref .---------.        .---------------.
     *                         \|/ |        \|/       |              \|/
     *       .______._____._____'__|___.     '__.__.__|___.     ._____'__._.
     * WBS:  | hash | key | val |offset| ... |vl|kl|offset| ... | val |vl|0|
     *       `------'-----'-----'------'     '--'--'------'     '-----'--'-'
     * If we add another value, we overwrite the list record.
     * We don't need to overwrite any vlongs and suffer because of that.
     *                         ref .---------.         .-------------------------------.
     *                         \|/ |        \|/        |                              \|/
     *       .______._____._____'__|___.     '__.__.___|__.     ._____.__._.     ._____'__.______.
     * WBS:  | hash | key | val |offset| ... |vl|kl|offset| ... | val |vl|0| ... | val |vl|offset|
     *       `------'-----'-----'------'     '--'--'------'     '-----:--'-'     '-----'--'--|---'
     *                                                               /|\                     |
     *                                                                `----------------------'
     * And another value (for example)
     * ... ---.         .-----------------------------------------------------.
     *       \|/        |                                                    \|/
     *        '__.__.___|__.     ._____.__._.     ._____.__.______.     ._____'__.______.
     * ...    |vl|kl|offset| ... | val |vl|0| ... | val |vl|offset| ... | val |vl|offset|
     *        '--'--'------'     '-----:--'-'     '-----'--:--|---'     '-----'--'--|---'
     *                                /|\                 /|\ |                     |
     *                                 `-------------------+--'                     |
     *                                                     `------------------------'
     */

    /**
     * Write buffers for keys and values. For the description of the structure above, think
     * of this as one infinite byte buffer.
     */
    private WriteBuffers writeBuffers;

    private final float loadFactor;

    private int resizeThreshold;
    private int keysAssigned;
    private int numValues;

    /**
     * Largest number of probe steps ever taken to find location for a key. When getting, we can
     * conclude that they key is not in hashtable when we make this many steps and don't find it.
     */
    private int largestNumberOfSteps = 0;

    /**
     * References to keys of the hashtable. The index is hash of the key; collisions are
     * resolved using open addressing with quadratic probing. Reference format
     * [40: offset into writeBuffers][8: state byte][1: has list flag]
     * [15: part of hash used to optimize probing]
     * Offset is tail offset of the first record for the key (the one containing the key).
     * It is not necessary to store 15 bits in particular to optimize probing; in fact when
     * we always store the hash it is not necessary. But we have nothing else to do with them.
     * TODO: actually we could also use few bits to store largestNumberOfSteps for each,
     *      so we'd stop earlier on read collision. Need to profile on real queries.
     */
    private long[] refs;
    private int startingHashBitCount, hashBitCount;

    private int metricPutConflict = 0, metricGetConflict = 0, metricExpands = 0, metricExpandsMs = 0;

    /** We have 39 bits to store list pointer from the first record; this is size limit */
    final static long MAX_WB_SIZE = ((long) 1) << 38;
    /** 8 Gb of refs is the max capacity if memory limit is not specified. If someone has 100s of
     * Gbs of memory (this might happen pretty soon) we'd need to string together arrays anyway. */
    private final static int DEFAULT_MAX_CAPACITY = 1024 * 1024 * 1024;
    /** Make sure maxCapacity has a lower limit */
    private final static int DEFAULT_MIN_MAX_CAPACITY = 16 * 1024 * 1024;

    public BytesBytesMultiHashMap(int initialCapacity, float loadFactor, int wbSize, long maxProbeSize) {
        if (loadFactor < 0 || loadFactor > 1) {
            throw new AssertionError("Load factor must be between (0, 1].");
        }
        assert initialCapacity > 0;
        initialCapacity = (Long.bitCount(initialCapacity) == 1) ? initialCapacity
                : nextHighestPowerOfTwo(initialCapacity);
        // 8 bytes per long in the refs, assume data will be empty. This is just a sanity check.
        int maxCapacity = (maxProbeSize <= 0) ? DEFAULT_MAX_CAPACITY
                : (int) Math.min((long) DEFAULT_MAX_CAPACITY, maxProbeSize / 8);
        if (maxCapacity < DEFAULT_MIN_MAX_CAPACITY) {
            maxCapacity = DEFAULT_MIN_MAX_CAPACITY;
        }
        if (maxCapacity < initialCapacity || initialCapacity <= 0) {
            // Either initialCapacity is too large, or nextHighestPowerOfTwo overflows
            initialCapacity = (Long.bitCount(maxCapacity) == 1) ? maxCapacity : nextLowestPowerOfTwo(maxCapacity);
        }

        validateCapacity(initialCapacity);
        startingHashBitCount = 63 - Long.numberOfLeadingZeros(initialCapacity);
        this.loadFactor = loadFactor;
        refs = new long[initialCapacity];
        writeBuffers = new WriteBuffers(wbSize, MAX_WB_SIZE);
        resizeThreshold = (int) (initialCapacity * this.loadFactor);
    }

    @VisibleForTesting
    BytesBytesMultiHashMap(int initialCapacity, float loadFactor, int wbSize) {
        this(initialCapacity, loadFactor, wbSize, -1);
    }

    /**
     * The result of looking up a key in the multi-hash map.
     *
     * This object can read through the 0, 1, or more values found for the key.
     */
    public static class Result {

        // Whether there are more than 0 rows.
        private boolean hasRows;

        // We need a pointer to the hash map since this class must be static to support having
        // multiple hash tables with Hybrid Grace partitioning.
        private BytesBytesMultiHashMap hashMap;

        // And, a mutable read position for thread safety when sharing a hash map.
        private WriteBuffers.Position readPos;

        // These values come from setValueResult when it finds a key.  These values allow this
        // class to read (and re-read) the values.
        private long firstOffset;
        private boolean hasList;
        private long offsetAfterListRecordKeyLen;

        // When we have multiple values, we save the next value record's offset here.
        private long nextTailOffset;

        // 0-based index of which row we are on.
        private long readIndex;

        // A reference to the current row.
        private WriteBuffers.ByteSegmentRef byteSegmentRef;

        public Result() {
            hasRows = false;
            byteSegmentRef = new WriteBuffers.ByteSegmentRef();
            readPos = new WriteBuffers.Position();
        }

        /**
         * Return the thread-safe read position.
         */
        public WriteBuffers.Position getReadPos() {
            return readPos;
        }

        /**
         * @return Whether there are 1 or more values.
         */
        public boolean hasRows() {
            // NOTE: Originally we named this isEmpty, but that name conflicted with another interface.
            return hasRows;
        }

        /**
         * @return Whether there is just 1 value row.
         */
        public boolean isSingleRow() {
            return !hasList;
        }

        /**
         * Set internal values for reading the values after finding a key.
         *
         * @param hashMap
         *          The hash map we found the key in.
         * @param firstOffset
         *          The absolute offset of the first record in the write buffers.
         * @param hasList
         *          Whether there are multiple values (true) or just a single value (false).
         * @param offsetAfterListRecordKeyLen
         *          The offset of just after the key length in the list record.  Or, 0 when single row.
         */
        public void set(BytesBytesMultiHashMap hashMap, long firstOffset, boolean hasList,
                long offsetAfterListRecordKeyLen) {

            this.hashMap = hashMap;

            this.firstOffset = firstOffset;
            this.hasList = hasList;
            this.offsetAfterListRecordKeyLen = offsetAfterListRecordKeyLen;

            // Position at first row.
            readIndex = 0;
            nextTailOffset = -1;

            hasRows = true;
        }

        public WriteBuffers.ByteSegmentRef first() {
            if (!hasRows) {
                return null;
            }

            // Position at first row.
            readIndex = 0;
            nextTailOffset = -1;

            return internalRead();
        }

        public WriteBuffers.ByteSegmentRef next() {
            if (!hasRows) {
                return null;
            }

            return internalRead();
        }

        /**
         * Read the current value.
         *
         * @return
         *           The ByteSegmentRef to the current value read.
         */
        private WriteBuffers.ByteSegmentRef internalRead() {

            if (!hasList) {

                /*
                 * Single value.
                 */

                if (readIndex > 0) {
                    return null;
                }

                // For a non-list (i.e. single value), the offset is for the variable length long (VLong)
                // holding the value length (followed by the key length).
                hashMap.writeBuffers.setReadPoint(firstOffset, readPos);
                int valueLength = (int) hashMap.writeBuffers.readVLong(readPos);

                // The value is before the offset.  Make byte segment reference absolute.
                byteSegmentRef.reset(firstOffset - valueLength, valueLength);
                hashMap.writeBuffers.populateValue(byteSegmentRef);

                readIndex++;
                return byteSegmentRef;
            }

            /*
             * Multiple values.
             */

            if (readIndex == 0) {
                // For a list, the value and key lengths of 1st record were overwritten with the
                // relative offset to a new list record.
                long relativeOffset = hashMap.writeBuffers.readNByteLong(firstOffset, 5, readPos);

                // At the beginning of the list record will be the value length.
                hashMap.writeBuffers.setReadPoint(firstOffset + relativeOffset, readPos);
                int valueLength = (int) hashMap.writeBuffers.readVLong(readPos);

                // The value is before the list record offset.  Make byte segment reference absolute.
                byteSegmentRef.reset(firstOffset - valueLength, valueLength);
                hashMap.writeBuffers.populateValue(byteSegmentRef);

                readIndex++;
                return byteSegmentRef;
            }

            if (readIndex == 1) {
                // We remembered the offset of just after the key length in the list record.
                // Read the absolute offset to the 2nd value.
                nextTailOffset = hashMap.writeBuffers.readNByteLong(offsetAfterListRecordKeyLen, 5, readPos);
                if (nextTailOffset <= 0) {
                    throw new Error("Expecting a second value");
                }
            } else if (nextTailOffset <= 0) {
                return null;
            }

            hashMap.writeBuffers.setReadPoint(nextTailOffset, readPos);

            // Get the value length.
            int valueLength = (int) hashMap.writeBuffers.readVLong(readPos);

            // Now read the relative offset to next record. Next record is always before the
            // previous record in the write buffers (see writeBuffers javadoc).
            long delta = hashMap.writeBuffers.readVLong(readPos);
            long newTailOffset = delta == 0 ? 0 : (nextTailOffset - delta);

            // The value is before the value record offset.  Make byte segment reference absolute.
            byteSegmentRef.reset(nextTailOffset - valueLength, valueLength);
            hashMap.writeBuffers.populateValue(byteSegmentRef);

            nextTailOffset = newTailOffset;
            readIndex++;
            return byteSegmentRef;
        }

        /**
         * Lets go of any references to a hash map.
         */
        public void forget() {
            hashMap = null;
            byteSegmentRef.reset(0, 0);
            hasRows = false;
            readIndex = 0;
            nextTailOffset = -1;
        }
    }

    /** The source of keys and values to put into hashtable; avoids byte copying. */
    public static interface KvSource {
        /** Write key into output. */
        public void writeKey(RandomAccessOutput dest) throws SerDeException;

        /** Write value into output. */
        public void writeValue(RandomAccessOutput dest) throws SerDeException;

        /**
         * Provide updated value for state byte for a key.
         * @param previousValue Previous value; null if this is the first call per key.
         * @return The updated value.
         */
        public byte updateStateByte(Byte previousValue);
    }

    /**
     * Adds new value to new or existing key in hashmap. Not thread-safe.
     * @param kv Keyvalue writer. Each method will be called at most once.
     */
    private static final byte[] FOUR_ZEROES = new byte[] { 0, 0, 0, 0 };

    public void put(KvSource kv, int keyHashCode) throws SerDeException {
        if (resizeThreshold <= keysAssigned) {
            expandAndRehash();
        }

        // Reserve 4 bytes for the hash (don't just reserve, there may be junk there)
        writeBuffers.write(FOUR_ZEROES);

        // Write key to buffer to compute hashcode and compare; if it's a new key, it will
        // become part of the record; otherwise, we will just write over it later.
        long keyOffset = writeBuffers.getWritePoint();

        kv.writeKey(writeBuffers);
        int keyLength = (int) (writeBuffers.getWritePoint() - keyOffset);
        int hashCode = (keyHashCode == -1) ? writeBuffers.unsafeHashCode(keyOffset, keyLength) : keyHashCode;

        int slot = findKeySlotToWrite(keyOffset, keyLength, hashCode);
        // LOG.info("Write hash code is " + Integer.toBinaryString(hashCode) + " - " + slot);

        long ref = refs[slot];
        if (ref == 0) {
            // This is a new key, keep writing the first record.
            long tailOffset = writeFirstValueRecord(kv, keyOffset, keyLength, hashCode);
            byte stateByte = kv.updateStateByte(null);
            refs[slot] = Ref.makeFirstRef(tailOffset, stateByte, hashCode, startingHashBitCount);
            ++keysAssigned;
        } else {
            // This is not a new key; we'll overwrite the key and hash bytes - not needed anymore.
            writeBuffers.setWritePoint(keyOffset - 4);
            long lrPtrOffset = createOrGetListRecord(ref);
            long tailOffset = writeValueAndLength(kv);
            addRecordToList(lrPtrOffset, tailOffset);
            byte oldStateByte = Ref.getStateByte(ref);
            byte stateByte = kv.updateStateByte(oldStateByte);
            if (oldStateByte != stateByte) {
                ref = Ref.setStateByte(ref, stateByte);
            }
            refs[slot] = Ref.setListFlag(ref);
        }
        ++numValues;
    }

    /**
     * Finds a key.  Values can be read with the supplied result object.
     *
     * Important Note: The caller is expected to pre-allocate the hashMapResult and not
     * share it among other threads.
     *
     * @param key Key buffer.
     * @param offset the offset to the key in the buffer
     * @param hashMapResult The object to fill in that can read the values.
     * @return The state byte.
     */
    public byte getValueResult(byte[] key, int offset, int length, Result hashMapResult) {

        hashMapResult.forget();

        WriteBuffers.Position readPos = hashMapResult.getReadPos();

        // First, find first record for the key.
        long ref = findKeyRefToRead(key, offset, length, readPos);
        if (ref == 0) {
            return 0;
        }

        boolean hasList = Ref.hasList(ref);

        // This relies on findKeyRefToRead doing key equality check and leaving read ptr where needed.
        long offsetAfterListRecordKeyLen = hasList ? writeBuffers.getReadPoint(readPos) : 0;

        hashMapResult.set(this, Ref.getOffset(ref), hasList, offsetAfterListRecordKeyLen);

        return Ref.getStateByte(ref);
    }

    /**
     * Take the segment reference from {@link #getValueRefs(byte[], int, List)}
     * result and makes it self-contained - adds byte array where the value is stored, and
     * updates the offset from "global" write buffers offset to offset within that array.
     */
    public void populateValue(WriteBuffers.ByteSegmentRef valueRef) {
        writeBuffers.populateValue(valueRef);
    }

    /**
     * Number of keys in the hashmap
     * @return number of keys
     */
    public int size() {
        return keysAssigned;
    }

    /**
     * Number of values in the hashmap
     * This is equal to or bigger than number of keys, since some values may share the same key
     * @return number of values
     */
    public int getNumValues() {
        return numValues;
    }

    /**
     * Number of bytes used by the hashmap
     * There are two main components that take most memory: writeBuffers and refs
     * Others include instance fields: 100
     * @return number of bytes
     */
    public long memorySize() {
        return getEstimatedMemorySize();
    }

    @Override
    public long getEstimatedMemorySize() {
        JavaDataModel jdm = JavaDataModel.get();
        long size = 0;
        size += writeBuffers.getEstimatedMemorySize();
        size += jdm.lengthForLongArrayOfSize(refs.length);
        // 11 primitive1 fields, 2 refs above with alignment
        size += JavaDataModel.alignUp(15 * jdm.primitive1(), jdm.memoryAlign());
        return size;
    }

    public void seal() {
        writeBuffers.seal();
    }

    public void clear() {
        // This will make the object completely unusable. Semantics of clear are not defined...
        this.writeBuffers.clear();
        this.refs = new long[1];
        this.keysAssigned = 0;
        this.numValues = 0;
    }

    public void expandAndRehashToTarget(int estimateNewRowCount) {
        int oldRefsCount = refs.length;
        int newRefsCount = oldRefsCount + estimateNewRowCount;
        if (resizeThreshold <= newRefsCount) {
            newRefsCount = (Long.bitCount(newRefsCount) == 1) ? estimateNewRowCount
                    : nextHighestPowerOfTwo(newRefsCount);
            expandAndRehashImpl(newRefsCount);
            LOG.info("Expand and rehash to " + newRefsCount + " from " + oldRefsCount);
        }
    }

    private static void validateCapacity(long capacity) {
        if (Long.bitCount(capacity) != 1) {
            throw new AssertionError("Capacity must be a power of two");
        }
        if (capacity <= 0) {
            throw new AssertionError("Invalid capacity " + capacity);
        }
        if (capacity > Integer.MAX_VALUE) {
            throw new RuntimeException("Attempting to expand the hash table to " + capacity
                    + " that overflows maximum array size. For this query, you may want to disable "
                    + ConfVars.HIVEDYNAMICPARTITIONHASHJOIN.varname + " or reduce "
                    + ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD.varname);
        }
    }

    /**
     * Finds the slot to use for writing, based on the key bytes already written to buffers.
     * @param keyOffset Offset to the key.
     * @param keyLength Length of the key.
     * @param hashCode Hash code of the key (passed in because java doesn't have ref/pointers).
     * @return The slot to use for writing; can be new, or matching existing key.
     */
    private int findKeySlotToWrite(long keyOffset, int keyLength, int hashCode) {
        final int bucketMask = (refs.length - 1);
        int slot = hashCode & bucketMask;
        long probeSlot = slot;
        int i = 0;
        while (true) {
            long ref = refs[slot];
            if (ref == 0 || isSameKey(keyOffset, keyLength, ref, hashCode)) {
                break;
            }
            ++metricPutConflict;
            // Some other key (collision) - keep probing.
            probeSlot += (++i);
            slot = (int) (probeSlot & bucketMask);
        }
        if (largestNumberOfSteps < i) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Probed " + i + " slots (the longest so far) to find space");
            }
            largestNumberOfSteps = i;
            // debugDumpKeyProbe(keyOffset, keyLength, hashCode, slot);
        }
        return slot;
    }

    /**
     * Finds the slot to use for reading.
     * @param key Read key array.
     * @param length Read key length.
     * @return The ref to use for reading.
     */
    private long findKeyRefToRead(byte[] key, int offset, int length, WriteBuffers.Position readPos) {
        final int bucketMask = (refs.length - 1);
        int hashCode = writeBuffers.hashCode(key, offset, length);
        int slot = hashCode & bucketMask;
        // LOG.info("Read hash code for " + Utils.toStringBinary(key, 0, length)
        //   + " is " + Integer.toBinaryString(hashCode) + " - " + slot);
        long probeSlot = slot;
        int i = 0;
        while (true) {
            long ref = refs[slot];
            // When we were inserting the key, we would have inserted here; so, there's no key.
            if (ref == 0) {
                return 0;
            }
            if (isSameKey(key, offset, length, ref, hashCode, readPos)) {
                return ref;
            }
            ++metricGetConflict;
            probeSlot += (++i);
            if (i > largestNumberOfSteps) {
                // We know we never went that far when we were inserting.
                return 0;
            }
            slot = (int) (probeSlot & bucketMask);
        }
    }

    /**
     * Puts ref into new refs array.
     * @param newRefs New refs array.
     * @param newRef New ref value.
     * @param hashCode Hash code to use.
     * @return The number of probe steps taken to find key position.
     */
    private int relocateKeyRef(long[] newRefs, long newRef, int hashCode) {
        final int bucketMask = newRefs.length - 1;
        int newSlot = hashCode & bucketMask;
        long probeSlot = newSlot;
        int i = 0;
        while (true) {
            long current = newRefs[newSlot];
            if (current == 0) {
                newRefs[newSlot] = newRef;
                break;
            }
            // New array cannot contain the records w/the same key, so just advance, don't check.
            probeSlot += (++i);
            newSlot = (int) (probeSlot & bucketMask);
        }
        return i;
    }

    /**
     * Verifies that the key matches a requisite key.
     * @param cmpOffset The offset to the key to compare with.
     * @param cmpLength The length of the key to compare with.
     * @param ref The ref that can be used to retrieve the candidate key.
     * @param hashCode
     * @return -1 if the key referenced by ref is different than the one referenced by cmp...
     *         0 if the keys match, and there's only one value for this key (no list).
     *         Offset if the keys match, and there are multiple values for this key (a list).
     */
    private boolean isSameKey(long cmpOffset, int cmpLength, long ref, int hashCode) {
        if (!compareHashBits(ref, hashCode)) {
            return false; // Hash bits in ref don't match.
        }
        writeBuffers.setUnsafeReadPoint(getFirstRecordLengthsOffset(ref, null));
        int valueLength = (int) writeBuffers.unsafeReadVLong(), keyLength = (int) writeBuffers.unsafeReadVLong();
        if (keyLength != cmpLength) {
            return false;
        }
        long keyOffset = Ref.getOffset(ref) - (valueLength + keyLength);
        // There's full hash code stored in front of the key. We could check that first. If keyLength
        // is <= 4 it obviously doesn't make sense, less bytes to check in a key. Then, if there's a
        // match, we check it in vain. But what is the proportion of matches? For writes it could be 0
        // if all keys are unique, for reads we hope it's really high. Then if there's a mismatch what
        // probability is there that key mismatches in <4 bytes (so just checking the key is faster)?
        // We assume the latter is pretty high, so we don't check for now.
        return writeBuffers.isEqual(cmpOffset, cmpLength, keyOffset, keyLength);
    }

    /**
     * Same as {@link #isSameKey(long, int, long, int)} but for externally stored key.
     */
    private boolean isSameKey(byte[] key, int offset, int length, long ref, int hashCode,
            WriteBuffers.Position readPos) {
        if (!compareHashBits(ref, hashCode)) {
            return false; // Hash bits don't match.
        }
        writeBuffers.setReadPoint(getFirstRecordLengthsOffset(ref, readPos), readPos);
        int valueLength = (int) writeBuffers.readVLong(readPos), keyLength = (int) writeBuffers.readVLong(readPos);
        long keyOffset = Ref.getOffset(ref) - (valueLength + keyLength);
        // See the comment in the other isSameKey
        if (offset == 0) {
            return writeBuffers.isEqual(key, length, keyOffset, keyLength);
        } else {
            return writeBuffers.isEqual(key, offset, length, keyOffset, keyLength);
        }
    }

    private boolean compareHashBits(long ref, int hashCode) {
        long fakeRef = Ref.makeFirstRef(0, (byte) 0, hashCode, startingHashBitCount);
        return (Ref.getHashBits(fakeRef) == Ref.getHashBits(ref));
    }

    /**
     * @param ref Reference.
     * @return The offset to value and key length vlongs of the first record referenced by ref.
     */
    private long getFirstRecordLengthsOffset(long ref, WriteBuffers.Position readPos) {
        long tailOffset = Ref.getOffset(ref);
        if (Ref.hasList(ref)) {
            long relativeOffset = (readPos == null) ? writeBuffers.unsafeReadNByteLong(tailOffset, 5)
                    : writeBuffers.readNByteLong(tailOffset, 5, readPos);
            tailOffset += relativeOffset;
        }
        return tailOffset;
    }

    private void expandAndRehash() {
        expandAndRehashImpl(((long) refs.length) << 1);
    }

    private void expandAndRehashImpl(long capacity) {
        long expandTime = System.currentTimeMillis();
        final long[] oldRefs = refs;
        validateCapacity(capacity);
        long[] newRefs = new long[(int) capacity];

        // We store some hash bits in ref; for every expansion, we need to add one bit to hash.
        // If we have enough bits, we'll do that; if we don't, we'll rehash.
        // LOG.info("Expanding the hashtable to " + capacity + " capacity");
        int newHashBitCount = hashBitCount + 1;

        // Relocate all assigned slots from the old hash table.
        int maxSteps = 0;
        for (int oldSlot = 0; oldSlot < oldRefs.length; ++oldSlot) {
            long oldRef = oldRefs[oldSlot];
            if (oldRef == 0) {
                continue;
            }
            // TODO: we could actually store a bit flag in ref indicating whether this is a hash
            //       match or a probe, and in the former case use hash bits (for a first few resizes).
            // int hashCodeOrPart = oldSlot | Ref.getNthHashBit(oldRef, startingHashBitCount, newHashBitCount);
            writeBuffers.setUnsafeReadPoint(getFirstRecordLengthsOffset(oldRef, null));
            // Read the value and key length for the first record.
            int hashCode = (int) writeBuffers.unsafeReadNByteLong(
                    Ref.getOffset(oldRef) - writeBuffers.unsafeReadVLong() - writeBuffers.unsafeReadVLong() - 4, 4);
            int probeSteps = relocateKeyRef(newRefs, oldRef, hashCode);
            maxSteps = Math.max(probeSteps, maxSteps);
        }
        this.refs = newRefs;
        this.largestNumberOfSteps = maxSteps;
        this.hashBitCount = newHashBitCount;
        this.resizeThreshold = (int) (capacity * loadFactor);
        metricExpandsMs += (System.currentTimeMillis() - expandTime);
        ++metricExpands;
    }

    /**
     * @param ref The ref.
     * @return The offset to list record pointer; list record is created if it doesn't exist.
     */
    private long createOrGetListRecord(long ref) {
        if (Ref.hasList(ref)) {
            // LOG.info("Found list record at " + writeBuffers.getReadPoint());
            return writeBuffers.getUnsafeReadPoint(); // Assumes we are here after key compare.
        }
        long firstTailOffset = Ref.getOffset(ref);
        // LOG.info("First tail offset to create list record is " + firstTailOffset);

        // Determine the length of storage for value and key lengths of the first record.
        writeBuffers.setUnsafeReadPoint(firstTailOffset);
        writeBuffers.unsafeSkipVLong();
        writeBuffers.unsafeSkipVLong();
        int lengthsLength = (int) (writeBuffers.getUnsafeReadPoint() - firstTailOffset);

        // Create the list record, copy first record value/key lengths there.
        writeBuffers.writeBytes(firstTailOffset, lengthsLength);
        long lrPtrOffset = writeBuffers.getWritePoint();
        // LOG.info("Creating list record: copying " + lengthsLength + ", lrPtrOffset " + lrPtrOffset);

        // Reserve 5 bytes for writeValueRecord to fill. There might be junk there so null them.
        writeBuffers.write(FIVE_ZEROES);
        // Link the list record to the first element.
        writeBuffers.writeFiveByteULong(firstTailOffset, lrPtrOffset - lengthsLength - firstTailOffset);
        return lrPtrOffset;
    }

    /**
     * Adds a newly-written record to existing list.
     * @param lrPtrOffset List record pointer offset.
     * @param tailOffset New record offset.
     */
    private void addRecordToList(long lrPtrOffset, long tailOffset) {
        // Now, insert this record into the list.
        long prevHeadOffset = writeBuffers.unsafeReadNByteLong(lrPtrOffset, 5);
        // LOG.info("Reading offset " + prevHeadOffset + " at " + lrPtrOffset);
        assert prevHeadOffset < tailOffset; // We replace an earlier element, must have lower offset.
        writeBuffers.writeFiveByteULong(lrPtrOffset, tailOffset);
        // LOG.info("Writing offset " + tailOffset + " at " + lrPtrOffset);
        writeBuffers.writeVLong(prevHeadOffset == 0 ? 0 : (tailOffset - prevHeadOffset));
    }

    /**
     * Writes first value and lengths to finish the first record after the key has been written.
     * @param kv Key-value writer.
     * @param keyOffset
     * @param keyLength Key length (already written).
     * @param hashCode
     * @return The offset of the new record.
     */
    private long writeFirstValueRecord(KvSource kv, long keyOffset, int keyLength, int hashCode)
            throws SerDeException {
        long valueOffset = writeBuffers.getWritePoint();
        kv.writeValue(writeBuffers);
        long tailOffset = writeBuffers.getWritePoint();
        int valueLength = (int) (tailOffset - valueOffset);
        // LOG.info("Writing value at " + valueOffset + " length " + valueLength);
        // In an unlikely case of 0-length key and value for the very first entry, we want to tell
        // this apart from an empty value. We'll just advance one byte; this byte will be lost.
        if (tailOffset == 0) {
            writeBuffers.reserve(1);
            ++tailOffset;
        }
        // LOG.info("First tail offset " + writeBuffers.getWritePoint());
        writeBuffers.writeVLong(valueLength);
        writeBuffers.writeVLong(keyLength);
        long lengthsLength = writeBuffers.getWritePoint() - tailOffset;
        if (lengthsLength < 5) { // Reserve space for potential future list
            writeBuffers.reserve(5 - (int) lengthsLength);
        }
        // Finally write the hash code.
        writeBuffers.writeInt(keyOffset - 4, hashCode);
        return tailOffset;
    }

    /**
     * Writes the value and value length for non-first record.
     * @param kv Key-value writer.
     * @return The offset of the new record.
     */
    private long writeValueAndLength(KvSource kv) throws SerDeException {
        long valueOffset = writeBuffers.getWritePoint();
        kv.writeValue(writeBuffers);
        long tailOffset = writeBuffers.getWritePoint();
        writeBuffers.writeVLong(tailOffset - valueOffset);
        // LOG.info("Writing value at " + valueOffset + " length " + (tailOffset - valueOffset));
        return tailOffset;
    }

    /** Writes the debug dump of the table into logs. Not thread-safe. */
    public void debugDumpTable() {
        StringBuilder dump = new StringBuilder(keysAssigned + " keys\n");
        TreeMap<Long, Integer> byteIntervals = new TreeMap<Long, Integer>();
        int examined = 0;
        for (int slot = 0; slot < refs.length; ++slot) {
            long ref = refs[slot];
            if (ref == 0) {
                continue;
            }
            ++examined;
            long recOffset = getFirstRecordLengthsOffset(ref, null);
            long tailOffset = Ref.getOffset(ref);
            writeBuffers.setUnsafeReadPoint(recOffset);
            int valueLength = (int) writeBuffers.unsafeReadVLong(),
                    keyLength = (int) writeBuffers.unsafeReadVLong();
            long ptrOffset = writeBuffers.getUnsafeReadPoint();
            if (Ref.hasList(ref)) {
                byteIntervals.put(recOffset, (int) (ptrOffset + 5 - recOffset));
            }
            long keyOffset = tailOffset - valueLength - keyLength;
            byte[] key = new byte[keyLength];
            WriteBuffers.ByteSegmentRef fakeRef = new WriteBuffers.ByteSegmentRef(keyOffset, keyLength);
            byteIntervals.put(keyOffset - 4, keyLength + 4);
            writeBuffers.populateValue(fakeRef);
            System.arraycopy(fakeRef.getBytes(), (int) fakeRef.getOffset(), key, 0, keyLength);
            dump.append(Utils.toStringBinary(key, 0, key.length)).append(" ref [").append(dumpRef(ref))
                    .append("]: ");
            Result hashMapResult = new Result();
            getValueResult(key, 0, key.length, hashMapResult);
            List<WriteBuffers.ByteSegmentRef> results = new ArrayList<WriteBuffers.ByteSegmentRef>();
            WriteBuffers.ByteSegmentRef byteSegmentRef = hashMapResult.first();
            while (byteSegmentRef != null) {
                results.add(hashMapResult.byteSegmentRef);
                byteSegmentRef = hashMapResult.next();
            }
            dump.append(results.size()).append(" rows\n");
            for (int i = 0; i < results.size(); ++i) {
                WriteBuffers.ByteSegmentRef segment = results.get(i);
                byteIntervals.put(segment.getOffset(), segment.getLength() + ((i == 0) ? 1 : 0)); // state byte in the first record
            }
        }
        if (examined != keysAssigned) {
            dump.append("Found " + examined + " keys!\n");
        }
        // Report suspicious gaps in writeBuffers
        long currentOffset = 0;
        for (Map.Entry<Long, Integer> e : byteIntervals.entrySet()) {
            long start = e.getKey(), len = e.getValue();
            if (start - currentOffset > 4) {
                dump.append("Gap! [" + currentOffset + ", " + start + ")\n");
            }
            currentOffset = start + len;
        }
        LOG.info("Hashtable dump:\n " + dump.toString());
    }

    private final static byte[] FIVE_ZEROES = new byte[] { 0, 0, 0, 0, 0 };

    private static int nextHighestPowerOfTwo(int v) {
        return Integer.highestOneBit(v) << 1;
    }

    private static int nextLowestPowerOfTwo(int v) {
        return Integer.highestOneBit(v);
    }

    @VisibleForTesting
    int getCapacity() {
        return refs.length;
    }

    /** Static helper for manipulating refs */
    private final static class Ref {
        private final static int OFFSET_SHIFT = 24;
        private final static int STATE_BYTE_SHIFT = 16;
        private final static long STATE_BYTE_MASK = ((long) 1 << (OFFSET_SHIFT - STATE_BYTE_SHIFT)) - 1;
        public final static long HASH_BITS_COUNT = STATE_BYTE_SHIFT - 1;
        private final static long HAS_LIST_MASK = (long) 1 << HASH_BITS_COUNT;
        private final static long HASH_BITS_MASK = HAS_LIST_MASK - 1;

        private final static long REMOVE_STATE_BYTE = ~(STATE_BYTE_MASK << STATE_BYTE_SHIFT);

        public static long getOffset(long ref) {
            return ref >>> OFFSET_SHIFT;
        }

        public static byte getStateByte(long ref) {
            return (byte) ((ref >>> STATE_BYTE_SHIFT) & STATE_BYTE_MASK);
        }

        public static boolean hasList(long ref) {
            return (ref & HAS_LIST_MASK) == HAS_LIST_MASK;
        }

        public static long getHashBits(long ref) {
            return ref & HASH_BITS_MASK;
        }

        public static long makeFirstRef(long offset, byte stateByte, int hashCode, int skipHashBits) {
            long hashPart = (hashCode >>> skipHashBits) & HASH_BITS_MASK;
            return offset << OFFSET_SHIFT | hashPart | ((stateByte & 0xffl) << STATE_BYTE_SHIFT);
        }

        public static int getNthHashBit(long ref, int skippedBits, int position) {
            long hashPart = getHashBits(ref) << skippedBits; // Original hashcode, with 0-d low bits.
            return (int) (hashPart & (1 << (position - 1)));
        }

        public static long setStateByte(long ref, byte stateByte) {
            return (ref & REMOVE_STATE_BYTE) | ((stateByte & 0xffl) << STATE_BYTE_SHIFT);
        }

        public static long setListFlag(long ref) {
            return ref | HAS_LIST_MASK;
        }
    }

    private static String dumpRef(long ref) {
        return StringUtils.leftPad(Long.toBinaryString(ref), 64, "0") + " o=" + Ref.getOffset(ref) + " s="
                + Ref.getStateByte(ref) + " l=" + Ref.hasList(ref) + " h="
                + Long.toBinaryString(Ref.getHashBits(ref));
    }

    public void debugDumpMetrics() {
        LOG.info("Map metrics: keys allocated " + this.refs.length + ", keys assigned " + keysAssigned
                + ", write conflict " + metricPutConflict + ", write max dist " + largestNumberOfSteps
                + ", read conflict " + metricGetConflict + ", expanded " + metricExpands + " times in "
                + metricExpandsMs + "ms");
    }

    private void debugDumpKeyProbe(long keyOffset, int keyLength, int hashCode, int finalSlot) {
        final int bucketMask = refs.length - 1;
        WriteBuffers.ByteSegmentRef fakeRef = new WriteBuffers.ByteSegmentRef(keyOffset, keyLength);
        writeBuffers.populateValue(fakeRef);
        int slot = hashCode & bucketMask;
        long probeSlot = slot;
        StringBuilder sb = new StringBuilder("Probe path debug for [");
        sb.append(Utils.toStringBinary(fakeRef.getBytes(), (int) fakeRef.getOffset(), fakeRef.getLength()));
        sb.append("] hashCode ").append(Integer.toBinaryString(hashCode)).append(" is: ");
        int i = 0;
        while (slot != finalSlot) {
            probeSlot += (++i);
            slot = (int) (probeSlot & bucketMask);
            sb.append(slot).append(" - ").append(probeSlot).append(" - ").append(Long.toBinaryString(refs[slot]))
                    .append("\n");
        }
        LOG.info(sb.toString());
    }
}