Java tutorial
/** * LargeFile.java * * Cytobank (TM) is server and client software for web-based management, analysis, * and sharing of flow cytometry data. * * Copyright (C) 2010 Cytobank, Inc. All rights reserved. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. * * Cytobank, Inc. * 659 Oak Grove Avenue #205 * Menlo Park, CA 94025 * * http://www.cytobank.org */ package org.cytobank.io; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.codec.binary.StringUtils; import org.apache.commons.pool.impl.GenericObjectPool; import org.cytobank.FACS; public abstract class LargeFile { /** The size of each buffer in bytes */ public static final int BUFFER_SIZE_IN_BYTES = FACS.FILE_BUFFER_SIZE_IN_DOUBLES * DoubleFile.BYTES_PER_DOUBLE; /** The maximun number of file buffers to hold in memory */ public static final int MAX_BUFFERS = FACS.NUMBER_OF_FILE_BUFFERS; /** Maximum wait time for a buffer before an exception */ public static final int MAX_WAIT_MS = FACS.MAX_FILE_BUFFER_WAIT_MS; /** Maximum wait time for a buffer before an exception */ public static final int MAX_IDLE_MS = FACS.MAX_FILE_BUFFER_IDLE_MS; /** * How often to display monitor updates in ms */ public static final int UPDATES = 2000; public static final String READ_ONLY = "r"; public static final String WRITE_ONLY = "w"; public static final String READ_WRITE = READ_ONLY + WRITE_ONLY; public static final String WRITE_APPEND_ONLY = WRITE_ONLY + "a"; public static final String READ_WRITE_APPEND = READ_ONLY + WRITE_APPEND_ONLY; public static final ByteOrder DEFAULT_BYTE_ORDER = ByteOrder.BIG_ENDIAN; protected static Thread monitorThread; // The finals are in an attempt to give hints to the just in time compiler. // Depending on the Java implementation they may speed things up or serve no purpose. protected final FileChannel fileChannel; final long absoluteDataStart; long absoluteDataEnd; protected final int relativeBufferSize; long absoluteLength; long relativeSize; protected final int bytesPerRead; protected final boolean readable; protected final boolean writable; protected final boolean appendable; protected long relativePosition = 0; // Setting to max value forces a load the first time around protected long relativeBufferStart = Long.MAX_VALUE; protected long relativeBufferEnd = -1; protected int relativeAvailable; protected ByteOrder byteOrder = DEFAULT_BYTE_ORDER; protected ByteBuffer byteBuffer; protected boolean written = false; protected boolean closed = false; protected String closedByThreadName; protected long closedByThreadId; protected static final GenericObjectPool objectPool; static { BufferPoolFactory eventBufferPoolFactory = new BufferPoolFactory(BUFFER_SIZE_IN_BYTES); objectPool = new GenericObjectPool(eventBufferPoolFactory, MAX_BUFFERS, GenericObjectPool.WHEN_EXHAUSTED_BLOCK, MAX_WAIT_MS, MAX_IDLE_MS, true, false); } protected LargeFile(FileChannel fileChannel, int bytesPerRead, String mode) throws IOException { this(fileChannel, 0, fileChannel.size(), bytesPerRead, mode); } protected LargeFile(FileChannel fileChannel, long dataAbsoluteOffset, long absoluteLength, int bytesPerRead, String mode) throws IOException { this.fileChannel = fileChannel; this.absoluteDataStart = dataAbsoluteOffset; this.absoluteDataEnd = dataAbsoluteOffset + absoluteLength - 1; this.absoluteLength = absoluteLength; this.bytesPerRead = bytesPerRead; this.relativeSize = toRelative(absoluteLength); if (mode == null || mode.trim().isEmpty()) { this.readable = true; this.writable = false; this.appendable = false; } else { mode = mode.toLowerCase(); this.readable = mode.contains(READ_ONLY); this.writable = mode.contains(WRITE_ONLY); this.appendable = mode.contains(WRITE_APPEND_ONLY); } byteBuffer = getBufferFromPool(); if (byteBuffer == null) throw new IOException("Could not get buffer from pool."); this.relativeBufferSize = toRelative(byteBuffer.capacity()); if (relativeBufferSize < 1) throw new IOException("Cannot work with a zero length buffer."); // Guard against attempts to specify a length needed that is larger than the actual file length by checking if the dataAbsoluteOffset + absoluteLength exceeds the length // of the file when in read only. If the file on disk is too short, throwing an IOException. if (!this.appendable) { if (fileChannel.size() < absoluteDataEnd) throw new IOException("File too short. Expected file to be at least " + absoluteDataEnd + " bytes, but it only contains " + fileChannel.size() + " bytes."); } } // Callbacks protected abstract void afterReadBuffer(); protected boolean positionRequiresRead(long position) { return (position < relativeBufferStart || position >= (relativeBufferStart + relativeBufferSize)); } protected boolean positionAtEof(long position) { return position >= relativeSize; } public ByteOrder getByteOrder() { return byteOrder; } public void setByteOrder(ByteOrder byteOrder) { if (byteOrder == null) byteOrder = DEFAULT_BYTE_ORDER; this.byteOrder = byteOrder; } public void setRelativePosition(long relativePosition) throws IOException { if (closed) { throw new IOException("Read from closed file. This large file was alrady closed by " + this.closedByThreadId + "/" + this.closedByThreadName); } if (!writable && positionAtEof(relativePosition)) { // It's alright to set a zero length file to position 0 if (relativePosition == 0) return; throw new IOException("Read past end of file in " + this.getClass().getName() + ". relativePosition: " + relativePosition + ", relativeSize: " + relativeSize + ", fileChannel.size(): " + fileChannel.size() + ", fileChannel.size() double length : " + fileChannel.size() / (1.0 * DoubleFile.BYTES_PER_DOUBLE)); } // Load the next frame if necessary if (positionRequiresRead(relativePosition)) loadFrame(relativePosition); this.relativePosition = relativePosition; } /** * Load the frame for the relative position. This will force a load. byteBuffer.position is set * to zero, since it assumed indexes will be used with gets and puts. Note: byteBuffer position * zero will not necessarily be forRelativePosition. * @param forRelativePosition This is the requested position needed from the frame. It will * be converted to the position needed on disk through lots of math and science. * @throws IOException */ protected void loadFrame(long forRelativePosition) throws IOException { // Flush out any writes before loading the next frame. flush(); // Everywhere else we try to ignore that the offset, absoluteDataStart, doesn't exist, here it is needed. long positionOnDisk = (toAbsolute(forRelativePosition) / BUFFER_SIZE_IN_BYTES) * BUFFER_SIZE_IN_BYTES + absoluteDataStart; // The byteBuffer must be cleared for the read fileChannel.read to write new data to it byteBuffer.clear(); // Byte order needs to be set after every clear/read byteBuffer.order(byteOrder); int read = 0; if (positionOnDisk <= absoluteDataEnd && readable) read = fileChannel.read(byteBuffer, positionOnDisk); relativeBufferStart = (forRelativePosition / relativeBufferSize) * relativeBufferSize; relativeAvailable = toRelative(read); relativeBufferEnd = relativeBufferStart + relativeAvailable - 1; // Use relativeAvailable to make sure that the amount read is divisible by bytesPerRead if (relativeAvailable > 0) { // Position needs to be set to zero for byteBuffer.toDouble() and its ilk to work // Set position to forRelativePosition byteBuffer.position(0); } // Do any follow up work required afterReadBuffer(); } protected int toRelative(int absolute) { return absolute / bytesPerRead; } protected int toAbsolute(int relative) { return relative * bytesPerRead; } protected long toRelative(long absolute) { return absolute / bytesPerRead; } protected long toAbsolute(long relative) { return relative * bytesPerRead; } protected int getNeededPosition() throws IOException { if (closed) throw new IOException("Read from closed file."); if (!appendable && positionAtEof(relativePosition)) throw new IOException("Opertation past end of file."); if (positionRequiresRead(relativePosition)) loadFrame(relativePosition); int neededPosition = (int) (relativePosition - relativeBufferStart); relativePosition++; return neededPosition; } protected int getNeededWritePosition() throws IOException { int neededPosition = getNeededPosition(); written = true; if (neededPosition >= relativeAvailable) relativeAvailable = neededPosition + 1; return neededPosition; } protected int write() throws IOException { byteBuffer.limit(toAbsolute(relativeAvailable)); long positionOnDisk = toAbsolute(relativeBufferStart) + absoluteDataStart; int written = fileChannel.write(byteBuffer, positionOnDisk); long currentPosition = toAbsolute(relativeBufferStart) + written; // Grow the length of the file if (currentPosition > absoluteLength) setLengthInBytes(currentPosition); return written; } public long getLengthInBytes() { return absoluteLength; } /** * Return the size of the file in terms of the data type that is being used to represent the file. * If the file is being represented in doubles, there are 8 bytes per double, and there are 800 bytes * in the file then the size would be 100 because there would be 100 doubles in the file. * @return * @throws IOException */ public long getRelativeLength() { return toRelative(absoluteLength); } protected void setLengthInBytes(long length) { this.absoluteLength = length; this.relativeSize = toRelative(length); this.absoluteDataEnd = absoluteDataStart + absoluteLength - 1; } public int getRelativeBufferSize() { return relativeBufferSize; } public int getAbsoluteBufferSize() { return BUFFER_SIZE_IN_BYTES; } public boolean isWritable() { return writable; } public void flush() throws IOException { if (written) write(); written = false; } /** * Close this instance of LargeFile. Note, this method intentionally does not close the fileChannel. * It is up to the caller to do that. * @throws IOException */ public void close() throws IOException { this.closedByThreadName = Thread.currentThread().getName(); this.closedByThreadId = Thread.currentThread().getId(); closed = true; flush(); returnBufferToPool(byteBuffer); byteBuffer = null; } // protected void finalize() { // try { // close(); // } catch (IOException ignore) {} // } protected static ByteBuffer getBufferFromPool() throws IOException { try { ByteBuffer byteBuffer = (ByteBuffer) objectPool.borrowObject(); byteBuffer.clear(); return byteBuffer; } catch (Exception e) { throw new IOException("Could not get buffer from buffer pool."); } } protected static void returnBufferToPool(ByteBuffer buffer) { if (buffer == null) return; try { objectPool.returnObject(buffer); } catch (Exception e) { System.err.println(e); } } public static int numberOfActiveBuffers() { return objectPool.getNumActive(); } public static int numberOfIdleBuffers() { return objectPool.getNumIdle(); } }