org.apache.hadoop.hdfs.hoss.db.FileStreamStore.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hdfs.hoss.db.FileStreamStore.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hdfs.hoss.db;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.hoss.util.StringSerializer;

/**
 * File based Stream Storage This class is Thread-Safe
 */
public final class FileStreamStore {

    private static final Log LOG = LogFactory.getLog(FileStreamStore.class);
    private final static short MAGIC = 0x754C;
    private final static byte MAGIC_PADDING = 0x42;
    private final static byte MAGIC_FOOT = 0x24;//$
    private static final int HEADER_LEN = 6;
    private static final int FOOTER_LEN = 1;

    /**
     * File associated to this store
     */
    private File file = null;
    /**
     * Size/Power-of-2 for size of buffers/align ^9=512 ^12=4096 ^16=65536
     */
    private final int bits;

    /**
     * RamdomAccessFile for Input this store
     */
    private RandomAccessFile rafInput = null;
    /**
     * FileChannel for Input this store
     */
    private FileChannel fcInput = null;
    /**
     * ByteBuffer for Input (internal used)
     */
    private final ByteBuffer bufInput;

    /**
     * FileOutputStream for Output this store
     */
    private FileOutputStream osOutput = null;
    /**
     * FileChannel for Output this store
     */
    private FileChannel fcOutput = null;
    /**
     * Current output offset for blocks (commited to disk)
     */
    private long offsetOutputCommited = 0;
    /**
     * Current output offset for blocks (uncommited to disk)
     */
    private long offsetOutputUncommited = 0;
    /**
     * ByteBuffer for Output (internal used)
     */
    private final ByteBuffer bufOutput;
    /**
     * In Valid State?
     */
    private boolean validState = false;
    /**
     * flush buffer on every write?
     */
    private boolean flushOnWrite = false;
    /**
     * sync to disk on flushbuffer?
     */
    private boolean syncOnFlush = true;
    /**
     * align data to buffer boundary?
     */
    private boolean alignBlocks = true;
    /**
     * Callback called when flush buffers to disk
     */
    private CallbackSync callback = null;

    /**
     * Instantiate FileStreamStore
     * 
     * @param file
     *            name of file to open
     * @param size
     *            for buffer to reduce context switching (minimal is 512bytes,
     *            recommended 64KBytes)
     */
    public FileStreamStore(final String file, final int bufferSize) {
        this(new File(file), bufferSize);
    }

    /**
     * Instantiate FileStreamStore
     * 
     * @param file
     *            file to open
     * @param size
     *            for buffer to reduce context switching (minimal is 512bytes)
     */
    public FileStreamStore(final File file, final int bufferSize) {
        this.file = file;
        this.bits = ((int) Math.ceil(Math.log(Math.max(bufferSize, 512)) / Math.log(2))); // round to power of 2
        this.bufInput = ByteBuffer.allocate(512); // default HDD sector size
        this.bufOutput = ByteBuffer.allocate(1 << bits);
    }

    // ========= Open / Close =========

    /**
     * Open file for read/write
     * 
     * @return true if valid state
     */
    public boolean open() {
        return open(false);
    }

    /**
     * Open file
     * 
     * @param readOnly
     *            open in readOnly mode?
     * @return true if valid state
     */
    public synchronized boolean open(final boolean readOnly) {
        if (isOpen()) {
            close();
        }
        if (LOG.isDebugEnabled())
            LOG.debug("open(" + file + ", " + (readOnly ? "r" : "rw") + ")");
        try {
            if (!readOnly) {
                //append manner
                osOutput = new FileOutputStream(file, true);
                fcOutput = osOutput.getChannel();
            }
            rafInput = new RandomAccessFile(file, "r");
            fcInput = rafInput.getChannel();
            if (readOnly)
                fcOutput = fcInput;
            offsetOutputUncommited = offsetOutputCommited = fcOutput.size();
        } catch (Exception e) {
            LOG.error("Exception in open()", e);
            try {
                close();
            } catch (Exception ign) {
            }
        }
        validState = isOpen();
        return validState;
    }

    /**
     * Close file
     */
    public synchronized void close() {
        if (validState)
            sync();
        try {
            fcInput.close();
        } catch (Exception ign) {
        }
        try {
            rafInput.close();
        } catch (Exception ign) {
        }
        try {
            osOutput.close();
        } catch (Exception ign) {
        }
        try {
            fcOutput.close();
        } catch (Exception ign) {
        }
        rafInput = null;
        fcInput = null;
        osOutput = null;
        fcOutput = null;
        //
        validState = false;
    }

    // ========= Info =========

    /**
     * @return true if file is open
     */
    public synchronized boolean isOpen() {
        try {
            if ((fcInput != null) && (fcOutput != null)) {
                return (fcInput.isOpen() && fcOutput.isOpen());
            }
        } catch (Exception ign) {
        }
        return false;
    }

    /**
     * @return size of file in bytes
     * @see #getBlockSize()
     */
    public synchronized long size() {
        try {
            return (file.length() + bufOutput.position());
        } catch (Exception e) {
            LOG.error("Exception in size()", e);
        }
        return -1;
    }

    /**
     * check if empty
     * 
     * @return true if empty
     */
    public synchronized boolean isEmpty() {
        if (!validState)
            throw new InvalidStateException();
        return (size() == 0);
    }

    /**
     * Read end of valid and check last magic footer
     * 
     * @return true if valid
     */
    public synchronized boolean isValid() {
        if (!validState)
            throw new InvalidStateException();
        final long size = size();
        if (size == 0)
            return true;
        try {
            final long offset = (size - FOOTER_LEN);
            if (offset < 0)
                return false;
            if (offset >= offsetOutputCommited) {
                if (bufOutput.position() > 0) {
                    LOG.warn("WARN: autoflush forced");
                    flushBuffer();
                }
            }
            bufInput.clear();
            bufInput.limit(FOOTER_LEN);
            final int readed = fcInput.position(offset).read(bufInput);
            if (readed < FOOTER_LEN) {
                return false;
            }
            bufInput.flip();
            final int footer = bufInput.get(); // Footer (byte)
            if (footer != MAGIC_FOOT) {
                LOG.error("MAGIC FOOT fake=" + Integer.toHexString(footer) + " expected="
                        + Integer.toHexString(MAGIC_FOOT));
                return false;
            }
            return true;
        } catch (Exception e) {
            LOG.error("Exception in isValid()", e);
        }
        return false;
    }

    // ========= Destroy =========

    /**
     * Truncate file
     */
    public synchronized void clear() {
        if (!validState)
            throw new InvalidStateException();
        try {
            bufOutput.clear();
            fcOutput.position(0).truncate(0).force(true);
            offsetOutputUncommited = offsetOutputCommited = fcOutput.position();
            if (callback != null)
                callback.synched(offsetOutputCommited);
            close();
            open();
        } catch (Exception e) {
            LOG.error("Exception in clear()", e);
        }
    }

    /**
     * Delete file
     */
    public synchronized void delete() {
        bufOutput.clear();
        close();
        try {
            file.delete();
        } catch (Exception ign) {
        }
    }

    // ========= Operations =========

    /**
     * set flush buffer on write to true/false, default false
     * 
     * @param syncOnFlush
     */
    public synchronized void setFlushOnWrite(final boolean flushOnWrite) {
        this.flushOnWrite = flushOnWrite;
    }

    /**
     * set sync to disk flush buffer to true/false, default true
     * 
     * @param syncOnFlush
     */
    public synchronized void setSyncOnFlush(final boolean syncOnFlush) {
        this.syncOnFlush = syncOnFlush;
    }

    /**
     * set align blocks to buffer boundary to true/false, default true
     * 
     * @param alignBlocks
     */
    public synchronized void setAlignBlocks(final boolean alignBlocks) {
        this.alignBlocks = alignBlocks;
    }

    /**
     * set callback called when buffers where synched to disk
     * 
     * @param callback
     */
    public synchronized void setCallback(final CallbackSync callback) {
        this.callback = callback;
    }

    /**
     * Read desired block of datalen from end of file
     * 
     * @param datalen
     *            expected
     * @param ByteBuffer
     * @return new offset (offset+headerlen+datalen+footer)
     */
    public synchronized long readFromEnd(final long datalen, final ByteBuffer buf) {
        if (!validState)
            throw new InvalidStateException();
        final long size = size();
        final long offset = (size - HEADER_LEN - datalen - FOOTER_LEN);
        return read(offset, buf);
    }

    /**
     * Read block from file
     * 
     * @param offset
     *            of block
     * @param ByteBuffer
     * @return new offset (offset+headerlen+datalen+footer)
     */
    public synchronized long read(long offset, final ByteBuffer buf) {
        if (!validState)
            throw new InvalidStateException();
        try {
            int readed;
            while (true) {
                if (offset >= offsetOutputCommited) {
                    if (bufOutput.position() > 0) {
                        LOG.warn("WARN: autoflush forced");
                        flushBuffer();
                    }
                }
                bufInput.clear();
                readed = fcInput.position(offset).read(bufInput); // Read 1
                // sector
                if (readed < HEADER_LEN) { // short+int (6 bytes)
                    return -1;
                }
                bufInput.flip();
                final int magicB1 = (bufInput.get() & 0xFF); // Header - Magic
                // (short, 2 bytes, msb-first)
                final int magicB2 = (bufInput.get() & 0xFF); // Header - Magic
                // (short, 2 bytes, lsb-last)
                if (alignBlocks && (magicB1 == MAGIC_PADDING)) {
                    final int diffOffset = nextBlockBoundary(offset);
                    if (diffOffset > 0) {
                        offset += diffOffset;
                        continue;
                    }
                }
                final int magic = ((magicB1 << 8) | magicB2);
                if (magic != MAGIC) {
                    LOG.error("MAGIC HEADER fake=" + Integer.toHexString(magic) + " expected="
                            + Integer.toHexString(MAGIC));
                    return -1;
                }
                break;
            }
            // Header - Data Size (int, 4 bytes)
            final int datalen = bufInput.getInt();
            final int dataUnderFlow = (datalen - (readed - HEADER_LEN));
            int footer = -12345678;
            if (dataUnderFlow < 0) {
                footer = bufInput.get(datalen + HEADER_LEN); // Footer (byte)
            }
            bufInput.limit(Math.min(readed, datalen + HEADER_LEN));
            buf.put(bufInput);
            if (dataUnderFlow > 0) {
                buf.limit(datalen);
                int len = fcInput.read(buf);
                if (len < dataUnderFlow) {
                    LOG.error("Unable to read payload readed=" + len + " expected=" + dataUnderFlow);
                    return -1;
                }
            }
            if (dataUnderFlow >= 0) {
                // Read Footer (byte)
                bufInput.clear();
                bufInput.limit(FOOTER_LEN);
                if (fcInput.read(bufInput) < FOOTER_LEN)
                    return -1;
                bufInput.flip();
                footer = bufInput.get();
            }
            if (footer != MAGIC_FOOT) {
                LOG.error("MAGIC FOOT fake=" + Integer.toHexString(footer) + " expected="
                        + Integer.toHexString(MAGIC_FOOT));
                return -1;
            }
            buf.flip();
            return (offset + HEADER_LEN + datalen + FOOTER_LEN);
        } catch (Exception e) {
            LOG.error("Exception in read(" + offset + ")", e);
        }
        return -1;
    }

    /**
     * Write from buf to file
     * 
     * @param offset
     *            of block
     * @param buf
     *            ByteBuffer to write
     * @return long offset where buffer begin was write or -1 if error
     */
    public synchronized long write(final ByteBuffer buf) {
        if (!validState)
            throw new InvalidStateException();
        final int packet_size = (HEADER_LEN + buf.limit() + FOOTER_LEN);
        // short + int + data + byte
        final boolean useDirectIO = (packet_size > (1 << bits));
        try {
            if (useDirectIO) {
                LOG.warn("WARN: usingDirectIO packet size is greater (" + packet_size + ") than file buffer ("
                        + bufOutput.capacity() + ")");
            }
            // Align output
            if (alignBlocks && !useDirectIO) {
                final int diffOffset = nextBlockBoundary(offsetOutputUncommited);
                if (packet_size > diffOffset) {
                    alignBuffer(diffOffset);
                    offsetOutputUncommited += diffOffset;
                }
            }
            // Remember current offset
            final long offset = offsetOutputUncommited;
            // Write pending buffered data to disk
            if (bufOutput.remaining() < packet_size) {
                flushBuffer();
            }
            // Write new data to buffer
            bufOutput.put((byte) ((MAGIC >> 8) & 0xFF)); // Header - Magic
            // (short, 2 bytes, msb-first)
            // Header - Magic (short, 2 bytes, lsb-last)
            bufOutput.put((byte) (MAGIC & 0xFF));
            // Header - Data Size (int, 4 bytes)
            bufOutput.putInt(buf.limit());
            if (useDirectIO) {
                bufOutput.flip();
                // Write Header + Data + Footer
                fcOutput.write(new ByteBuffer[] { bufOutput, buf, ByteBuffer.wrap(new byte[] { MAGIC_FOOT }) });
                bufOutput.clear();
                offsetOutputUncommited = offsetOutputCommited = fcOutput.position();
                if (syncOnFlush) {
                    fcOutput.force(false);
                    if (callback != null)
                        callback.synched(offsetOutputCommited);
                }
            } else {
                bufOutput.put(buf); // Data Body
                bufOutput.put(MAGIC_FOOT); // Footer
                // Increment offset of buffered data (header + user-data)
                offsetOutputUncommited += packet_size;
                if (flushOnWrite)
                    flushBuffer();
            }
            return offset;
        } catch (Exception e) {
            LOG.error("Exception in write()", e);
        }
        return -1L;
    }

    /**
     * How much bytes left to next block boundary
     * 
     * @param offset
     * @return bytes left
     */
    private final int nextBlockBoundary(final long offset) {
        return (int) ((((offset >> bits) + 1) << bits) - offset);
    }

    /**
     * Pad output buffer with NULL to complete alignment
     * 
     * @param diff
     *            bytes
     * @throws IOException
     */
    private final void alignBuffer(final int diff) throws IOException {
        if (bufOutput.remaining() < diff) {
            flushBuffer();
        }
        bufOutput.put(MAGIC_PADDING); // Magic for Padding
        int i = 1;
        for (; i + 8 <= diff; i += 8) {
            bufOutput.putLong(0L);
        }
        for (; i + 4 <= diff; i += 4) {
            bufOutput.putInt(0);
        }
        switch (diff - i) {
        case 3:
            bufOutput.put((byte) 0);
        case 2:
            bufOutput.putShort((short) 0);
            break;
        case 1:
            bufOutput.put((byte) 0);
        }
    }

    /**
     * Flush buffer to file
     * 
     * @return false if exception occur
     */
    public synchronized boolean flush() {
        if (!validState)
            throw new InvalidStateException();
        try {
            flushBuffer();
            return true;
        } catch (Exception e) {
            LOG.error("Exception in flush()", e);
        }
        return false;
    }

    /**
     * Write uncommited data to disk
     * 
     * @throws IOException
     */
    private final void flushBuffer() throws IOException {
        if (bufOutput.position() > 0) {
            bufOutput.flip();
            fcOutput.write(bufOutput);
            bufOutput.clear();
            offsetOutputUncommited = offsetOutputCommited = fcOutput.position();
            if (syncOnFlush) {
                fcOutput.force(false);
                if (callback != null)
                    callback.synched(offsetOutputCommited);
            }
        }
    }

    /**
     * Forces any updates to this file to be written to the storage device that
     * contains it.
     * 
     * @return false if exception occur
     */
    public synchronized boolean sync() {
        if (!validState)
            throw new InvalidStateException();
        try {
            flushBuffer();
            if (!syncOnFlush) {
                fcOutput.force(false);
                if (callback != null)
                    callback.synched(offsetOutputCommited);
            }
            return true;
        } catch (Exception e) {
            LOG.error("Exception in sync()", e);
        }
        return false;
    }

    public static interface CallbackSync {
        public void synched(final long offset);
    }

    // ========= Exceptions =========

    /**
     * Exception throwed when store is in invalid state (closed)
     */
    public static class InvalidStateException extends RuntimeException {
        private static final long serialVersionUID = 42L;
    }

    // ========= END =========

    public static void main(String[] args) {
        final int BUFFER_LEN = 4096;//65536
        final int TOTAL = 10000000;
        FileStreamStore fss = new FileStreamStore("./data/stream", BUFFER_LEN);
        final ByteBuffer buf = ByteBuffer.allocate(BUFFER_LEN);
        final long[] offset = new long[TOTAL];

        fss.delete();
        fss.open();
        // Parameters
        fss.setFlushOnWrite(false);
        fss.setSyncOnFlush(false);
        fss.setAlignBlocks(true);
        long start = System.currentTimeMillis();
        for (int i = 0; i < TOTAL; i++) {
            buf.clear();
            StringSerializer.fromStringToBuffer(buf, "hehe" + i);
            buf.flip();
            offset[i] = fss.write(buf);
        }
        fss.sync();
        //System.out.println(ArrayUtils.toString(offset));
        System.out.println("write time: " + (System.currentTimeMillis() - start) / 1000);
        start = System.currentTimeMillis();
        long newOffset = 0;
        for (int i = 0; i < TOTAL; i++) {
            buf.clear();
            newOffset = fss.read(newOffset, buf);
            if (newOffset < 0) {
                System.out.println("Error trying read offset " + i + " size=" + fss.size());
                break;
            }
            StringSerializer.fromBufferToString(buf);
            //System.out.print(hehe + "\t");
        }
        System.out.println("read time: " + (System.currentTimeMillis() - start) / 1000);
    }

}