org.apache.lucene.store.BufferedIndexInput.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.lucene.store.BufferedIndexInput.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.store;

import java.io.EOFException;
import java.io.IOException;

/** Base implementation class for buffered {@link IndexInput}. */
public abstract class BufferedIndexInput extends IndexInput implements RandomAccessInput {

    /** Default buffer size set to {@value #BUFFER_SIZE}. */
    public static final int BUFFER_SIZE = 1024;

    /** Minimum buffer size allowed */
    public static final int MIN_BUFFER_SIZE = 8;

    // The normal read buffer size defaults to 1024, but
    // increasing this during merging seems to yield
    // performance gains.  However we don't want to increase
    // it too much because there are quite a few
    // BufferedIndexInputs created during merging.  See
    // LUCENE-888 for details.
    /**
     * A buffer size for merges set to {@value #MERGE_BUFFER_SIZE}.
     */
    public static final int MERGE_BUFFER_SIZE = 4096;

    private int bufferSize = BUFFER_SIZE;

    protected byte[] buffer;

    private long bufferStart = 0; // position in file of buffer
    private int bufferLength = 0; // end of valid bytes
    private int bufferPosition = 0; // next byte to read

    @Override
    public final byte readByte() throws IOException {
        if (bufferPosition >= bufferLength)
            refill();
        return buffer[bufferPosition++];
    }

    public BufferedIndexInput(String resourceDesc) {
        this(resourceDesc, BUFFER_SIZE);
    }

    public BufferedIndexInput(String resourceDesc, IOContext context) {
        this(resourceDesc, bufferSize(context));
    }

    /** Inits BufferedIndexInput with a specific bufferSize */
    public BufferedIndexInput(String resourceDesc, int bufferSize) {
        super(resourceDesc);
        checkBufferSize(bufferSize);
        this.bufferSize = bufferSize;
    }

    /** Change the buffer size used by this IndexInput */
    public final void setBufferSize(int newSize) {
        assert buffer == null || bufferSize == buffer.length : "buffer=" + buffer + " bufferSize=" + bufferSize
                + " buffer.length=" + (buffer != null ? buffer.length : 0);
        if (newSize != bufferSize) {
            checkBufferSize(newSize);
            bufferSize = newSize;
            if (buffer != null) {
                // Resize the existing buffer and carefully save as
                // many bytes as possible starting from the current
                // bufferPosition
                byte[] newBuffer = new byte[newSize];
                final int leftInBuffer = bufferLength - bufferPosition;
                final int numToCopy;
                if (leftInBuffer > newSize)
                    numToCopy = newSize;
                else
                    numToCopy = leftInBuffer;
                System.arraycopy(buffer, bufferPosition, newBuffer, 0, numToCopy);
                bufferStart += bufferPosition;
                bufferPosition = 0;
                bufferLength = numToCopy;
                newBuffer(newBuffer);
            }
        }
    }

    protected void newBuffer(byte[] newBuffer) {
        // Subclasses can do something here
        buffer = newBuffer;
    }

    /** Returns buffer size.  @see #setBufferSize */
    public final int getBufferSize() {
        return bufferSize;
    }

    private void checkBufferSize(int bufferSize) {
        if (bufferSize < MIN_BUFFER_SIZE)
            throw new IllegalArgumentException(
                    "bufferSize must be at least MIN_BUFFER_SIZE (got " + bufferSize + ")");
    }

    @Override
    public final void readBytes(byte[] b, int offset, int len) throws IOException {
        readBytes(b, offset, len, true);
    }

    @Override
    public final void readBytes(byte[] b, int offset, int len, boolean useBuffer) throws IOException {
        int available = bufferLength - bufferPosition;
        if (len <= available) {
            // the buffer contains enough data to satisfy this request
            if (len > 0) // to allow b to be null if len is 0...
                System.arraycopy(buffer, bufferPosition, b, offset, len);
            bufferPosition += len;
        } else {
            // the buffer does not have enough data. First serve all we've got.
            if (available > 0) {
                System.arraycopy(buffer, bufferPosition, b, offset, available);
                offset += available;
                len -= available;
                bufferPosition += available;
            }
            // and now, read the remaining 'len' bytes:
            if (useBuffer && len < bufferSize) {
                // If the amount left to read is small enough, and
                // we are allowed to use our buffer, do it in the usual
                // buffered way: fill the buffer and copy from it:
                refill();
                if (bufferLength < len) {
                    // Throw an exception when refill() could not read len bytes:
                    System.arraycopy(buffer, 0, b, offset, bufferLength);
                    throw new EOFException("read past EOF: " + this);
                } else {
                    System.arraycopy(buffer, 0, b, offset, len);
                    bufferPosition = len;
                }
            } else {
                // The amount left to read is larger than the buffer
                // or we've been asked to not use our buffer -
                // there's no performance reason not to read it all
                // at once. Note that unlike the previous code of
                // this function, there is no need to do a seek
                // here, because there's no need to reread what we
                // had in the buffer.
                long after = bufferStart + bufferPosition + len;
                if (after > length())
                    throw new EOFException("read past EOF: " + this);
                readInternal(b, offset, len);
                bufferStart = after;
                bufferPosition = 0;
                bufferLength = 0; // trigger refill() on read
            }
        }
    }

    @Override
    public final short readShort() throws IOException {
        if (2 <= (bufferLength - bufferPosition)) {
            return (short) (((buffer[bufferPosition++] & 0xFF) << 8) | (buffer[bufferPosition++] & 0xFF));
        } else {
            return super.readShort();
        }
    }

    @Override
    public final int readInt() throws IOException {
        if (4 <= (bufferLength - bufferPosition)) {
            return ((buffer[bufferPosition++] & 0xFF) << 24) | ((buffer[bufferPosition++] & 0xFF) << 16)
                    | ((buffer[bufferPosition++] & 0xFF) << 8) | (buffer[bufferPosition++] & 0xFF);
        } else {
            return super.readInt();
        }
    }

    @Override
    public final long readLong() throws IOException {
        if (8 <= (bufferLength - bufferPosition)) {
            final int i1 = ((buffer[bufferPosition++] & 0xff) << 24) | ((buffer[bufferPosition++] & 0xff) << 16)
                    | ((buffer[bufferPosition++] & 0xff) << 8) | (buffer[bufferPosition++] & 0xff);
            final int i2 = ((buffer[bufferPosition++] & 0xff) << 24) | ((buffer[bufferPosition++] & 0xff) << 16)
                    | ((buffer[bufferPosition++] & 0xff) << 8) | (buffer[bufferPosition++] & 0xff);
            return (((long) i1) << 32) | (i2 & 0xFFFFFFFFL);
        } else {
            return super.readLong();
        }
    }

    @Override
    public final int readVInt() throws IOException {
        if (5 <= (bufferLength - bufferPosition)) {
            byte b = buffer[bufferPosition++];
            if (b >= 0)
                return b;
            int i = b & 0x7F;
            b = buffer[bufferPosition++];
            i |= (b & 0x7F) << 7;
            if (b >= 0)
                return i;
            b = buffer[bufferPosition++];
            i |= (b & 0x7F) << 14;
            if (b >= 0)
                return i;
            b = buffer[bufferPosition++];
            i |= (b & 0x7F) << 21;
            if (b >= 0)
                return i;
            b = buffer[bufferPosition++];
            // Warning: the next ands use 0x0F / 0xF0 - beware copy/paste errors:
            i |= (b & 0x0F) << 28;
            if ((b & 0xF0) == 0)
                return i;
            throw new IOException("Invalid vInt detected (too many bits)");
        } else {
            return super.readVInt();
        }
    }

    @Override
    public final long readVLong() throws IOException {
        if (9 <= bufferLength - bufferPosition) {
            byte b = buffer[bufferPosition++];
            if (b >= 0)
                return b;
            long i = b & 0x7FL;
            b = buffer[bufferPosition++];
            i |= (b & 0x7FL) << 7;
            if (b >= 0)
                return i;
            b = buffer[bufferPosition++];
            i |= (b & 0x7FL) << 14;
            if (b >= 0)
                return i;
            b = buffer[bufferPosition++];
            i |= (b & 0x7FL) << 21;
            if (b >= 0)
                return i;
            b = buffer[bufferPosition++];
            i |= (b & 0x7FL) << 28;
            if (b >= 0)
                return i;
            b = buffer[bufferPosition++];
            i |= (b & 0x7FL) << 35;
            if (b >= 0)
                return i;
            b = buffer[bufferPosition++];
            i |= (b & 0x7FL) << 42;
            if (b >= 0)
                return i;
            b = buffer[bufferPosition++];
            i |= (b & 0x7FL) << 49;
            if (b >= 0)
                return i;
            b = buffer[bufferPosition++];
            i |= (b & 0x7FL) << 56;
            if (b >= 0)
                return i;
            throw new IOException("Invalid vLong detected (negative values disallowed)");
        } else {
            return super.readVLong();
        }
    }

    @Override
    public final byte readByte(long pos) throws IOException {
        long index = pos - bufferStart;
        if (index < 0 || index >= bufferLength) {
            bufferStart = pos;
            bufferPosition = 0;
            bufferLength = 0; // trigger refill() on read()
            seekInternal(pos);
            refill();
            index = 0;
        }
        return buffer[(int) index];
    }

    @Override
    public final short readShort(long pos) throws IOException {
        long index = pos - bufferStart;
        if (index < 0 || index >= bufferLength - 1) {
            bufferStart = pos;
            bufferPosition = 0;
            bufferLength = 0; // trigger refill() on read()
            seekInternal(pos);
            refill();
            index = 0;
        }
        return (short) (((buffer[(int) index] & 0xFF) << 8) | (buffer[(int) index + 1] & 0xFF));
    }

    @Override
    public final int readInt(long pos) throws IOException {
        long index = pos - bufferStart;
        if (index < 0 || index >= bufferLength - 3) {
            bufferStart = pos;
            bufferPosition = 0;
            bufferLength = 0; // trigger refill() on read()
            seekInternal(pos);
            refill();
            index = 0;
        }
        return ((buffer[(int) index] & 0xFF) << 24) | ((buffer[(int) index + 1] & 0xFF) << 16)
                | ((buffer[(int) index + 2] & 0xFF) << 8) | (buffer[(int) index + 3] & 0xFF);
    }

    @Override
    public final long readLong(long pos) throws IOException {
        long index = pos - bufferStart;
        if (index < 0 || index >= bufferLength - 7) {
            bufferStart = pos;
            bufferPosition = 0;
            bufferLength = 0; // trigger refill() on read()
            seekInternal(pos);
            refill();
            index = 0;
        }
        final int i1 = ((buffer[(int) index] & 0xFF) << 24) | ((buffer[(int) index + 1] & 0xFF) << 16)
                | ((buffer[(int) index + 2] & 0xFF) << 8) | (buffer[(int) index + 3] & 0xFF);
        final int i2 = ((buffer[(int) index + 4] & 0xFF) << 24) | ((buffer[(int) index + 5] & 0xFF) << 16)
                | ((buffer[(int) index + 6] & 0xFF) << 8) | (buffer[(int) index + 7] & 0xFF);
        return (((long) i1) << 32) | (i2 & 0xFFFFFFFFL);
    }

    private void refill() throws IOException {
        long start = bufferStart + bufferPosition;
        long end = start + bufferSize;
        if (end > length()) // don't read past EOF
            end = length();
        int newLength = (int) (end - start);
        if (newLength <= 0)
            throw new EOFException("read past EOF: " + this);

        if (buffer == null) {
            newBuffer(new byte[bufferSize]); // allocate buffer lazily
            seekInternal(bufferStart);
        }
        readInternal(buffer, 0, newLength);
        bufferLength = newLength;
        bufferStart = start;
        bufferPosition = 0;
    }

    /** Expert: implements buffer refill.  Reads bytes from the current position
     * in the input.
     * @param b the array to read bytes into
     * @param offset the offset in the array to start storing bytes
     * @param length the number of bytes to read
     */
    protected abstract void readInternal(byte[] b, int offset, int length) throws IOException;

    @Override
    public final long getFilePointer() {
        return bufferStart + bufferPosition;
    }

    @Override
    public final void seek(long pos) throws IOException {
        if (pos >= bufferStart && pos < (bufferStart + bufferLength))
            bufferPosition = (int) (pos - bufferStart); // seek within buffer
        else {
            bufferStart = pos;
            bufferPosition = 0;
            bufferLength = 0; // trigger refill() on read()
            seekInternal(pos);
        }
    }

    /** Expert: implements seek.  Sets current position in this file, where the
     * next {@link #readInternal(byte[],int,int)} will occur.
     * @see #readInternal(byte[],int,int)
     */
    protected abstract void seekInternal(long pos) throws IOException;

    @Override
    public BufferedIndexInput clone() {
        BufferedIndexInput clone = (BufferedIndexInput) super.clone();

        clone.buffer = null;
        clone.bufferLength = 0;
        clone.bufferPosition = 0;
        clone.bufferStart = getFilePointer();

        return clone;
    }

    @Override
    public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
        return wrap(sliceDescription, this, offset, length);
    }

    /**
     * Flushes the in-memory buffer to the given output, copying at most
     * <code>numBytes</code>.
     * <p>
     * <b>NOTE:</b> this method does not refill the buffer, however it does
     * advance the buffer position.
     * 
     * @return the number of bytes actually flushed from the in-memory buffer.
     */
    protected final int flushBuffer(IndexOutput out, long numBytes) throws IOException {
        int toCopy = bufferLength - bufferPosition;
        if (toCopy > numBytes) {
            toCopy = (int) numBytes;
        }
        if (toCopy > 0) {
            out.writeBytes(buffer, bufferPosition, toCopy);
            bufferPosition += toCopy;
        }
        return toCopy;
    }

    /**
     * Returns default buffer sizes for the given {@link IOContext}
     */
    public static int bufferSize(IOContext context) {
        switch (context.context) {
        case MERGE:
            return MERGE_BUFFER_SIZE;
        default:
            return BUFFER_SIZE;
        }
    }

    /** 
     * Wraps a portion of another IndexInput with buffering.
     * <p><b>Please note:</b> This is in most cases ineffective, because it may double buffer!
     */
    public static BufferedIndexInput wrap(String sliceDescription, IndexInput other, long offset, long length) {
        return new SlicedIndexInput(sliceDescription, other, offset, length);
    }

    /** 
     * Implementation of an IndexInput that reads from a portion of a file.
     */
    private static final class SlicedIndexInput extends BufferedIndexInput {
        IndexInput base;
        long fileOffset;
        long length;

        SlicedIndexInput(String sliceDescription, IndexInput base, long offset, long length) {
            super((sliceDescription == null) ? base.toString()
                    : (base.toString() + " [slice=" + sliceDescription + "]"), BufferedIndexInput.BUFFER_SIZE);
            if (offset < 0 || length < 0 || offset + length > base.length()) {
                throw new IllegalArgumentException("slice() " + sliceDescription + " out of bounds: " + base);
            }
            this.base = base.clone();
            this.fileOffset = offset;
            this.length = length;
        }

        @Override
        public SlicedIndexInput clone() {
            SlicedIndexInput clone = (SlicedIndexInput) super.clone();
            clone.base = base.clone();
            clone.fileOffset = fileOffset;
            clone.length = length;
            return clone;
        }

        @Override
        protected void readInternal(byte[] b, int offset, int len) throws IOException {
            long start = getFilePointer();
            if (start + len > length) {
                throw new EOFException("read past EOF: " + this);
            }
            base.seek(fileOffset + start);
            base.readBytes(b, offset, len, false);
        }

        @Override
        protected void seekInternal(long pos) {
        }

        @Override
        public void close() throws IOException {
            base.close();
        }

        @Override
        public long length() {
            return length;
        }
    }
}