Java tutorial
/** * MemoryEvents.java * * Cytobank (TM) is server and client software for web-based management, analysis, * and sharing of flow cytometry data. * * Copyright (C) 2009 Cytobank, Inc. All rights reserved. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. * * Cytobank, Inc. * 659 Oak Grove Avenue #205 * Menlo Park, CA 94025 * * http://www.cytobank.org */ package org.cytobank.fcs_files.events; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.DoubleBuffer; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.nio.channels.FileChannel.MapMode; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Vector; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.pool.impl.GenericObjectPool; import org.cytobank.io.DoubleFile; public class MemoryEvents implements IEvents { public static final int BYTES_PER_DOUBLE = Double.SIZE / Byte.SIZE; public static final int THOUSAND = 1000; public static final int MILLION = THOUSAND * THOUSAND; /** Maximum wait time for a buffer before an exception */ public static final int MAX_WAIT_MS = 0; /** Maximum idle time for a buffer before eviction */ public static final int MAX_IDLE_MS = -1; public static final int MEDIUM_EVENT_ARRAY_SIZE = 220 * THOUSAND; public static final int LARGE_EVENT_ARRAY_SIZE = 8 * MILLION; public static final int MEDIUM_EVENT_ARRAY_BUFFERS = 11 * 17; // ( 187 buffers; 329 MB; ~76% of all experiments) public static final int LARGE_EVENT_ARRAY_BUFFERS = 20; // ( 20 buffers; 1.280 GB; ~TOP 24% of large experiments) protected static final Logger logger = Logger.getLogger(MemoryEvents.class.getPackage().getName()); protected File doubleFile; protected MemoryEventsArray memoryEventsArray; protected double[] events; protected int numberOfEvents = -1; protected String name; protected long openedAt = 0; protected boolean reusedEventArray = false; protected int position = 0; protected boolean closed = false; /** * How often to cleanup in ms */ public static final int CLEANUP_INTERVAL = 1000; protected static Thread cleanupThread; protected static Thread closeThread; protected static Vector<MemoryEventsArray> referencedMemoryEventsArrays = new Vector<MemoryEventsArray>(); protected static final GenericObjectPool mediumEventArrayPool, largeEventArrayPool; protected static final HashMap<String, GenericObjectPool> poolsByName = new HashMap<String, GenericObjectPool>(); protected static final HashSet<MemoryEvents> closedMemoryEvents = new HashSet<MemoryEvents>(); static { EventArrayPoolableObjectFactory mediumEventArrayPoolableObjectFactory = new EventArrayPoolableObjectFactory( MEDIUM_EVENT_ARRAY_SIZE); EventArrayPoolableObjectFactory largeEventArrayPoolableObjectFactory = new EventArrayPoolableObjectFactory( LARGE_EVENT_ARRAY_SIZE); //logger.info("Setting up MemoryEvents buffer pool with " + MAX_BUFFERS + " buffers of " + BUFFER_SIZE_IN_BYTES + " bytes (total memory size: " + MAX_BUFFERS * BUFFER_SIZE_IN_BYTES + " bytes), " + MAX_WAIT_MS + " ms max wait before exception, and " + MAX_IDLE_MS + " ms max idle."); // Setup the object pool to block when exhausted, never evict, and to not bother testing on borrow and return mediumEventArrayPool = new GenericObjectPool(mediumEventArrayPoolableObjectFactory, MEDIUM_EVENT_ARRAY_BUFFERS, GenericObjectPool.WHEN_EXHAUSTED_FAIL, MAX_WAIT_MS, MAX_IDLE_MS, true, false); largeEventArrayPool = new GenericObjectPool(largeEventArrayPoolableObjectFactory, LARGE_EVENT_ARRAY_BUFFERS, GenericObjectPool.WHEN_EXHAUSTED_FAIL, MAX_WAIT_MS, MAX_IDLE_MS, true, false); poolsByName.put("Medium Event Array Pool ( " + MEDIUM_EVENT_ARRAY_SIZE + " doubles)", mediumEventArrayPool); poolsByName.put("Large Event Array Pool ( " + LARGE_EVENT_ARRAY_SIZE + " doubles)", largeEventArrayPool); // Start the monitor startMonitor(); startCloseThread(); } public MemoryEvents(File doubleFile) throws IOException, MemoryEventsException { this.doubleFile = doubleFile; numberOfEvents = (int) doubleFile.length() / BYTES_PER_DOUBLE; setupMemoryEventsArray(numberOfEvents); try { // Read in the doubles file FileInputStream fileInputStream = new FileInputStream(doubleFile); FileChannel fileChannel = fileInputStream.getChannel(); MappedByteBuffer mappedByteBuffer = fileChannel.map(MapMode.READ_ONLY, 0, doubleFile.length()); DoubleBuffer doubleBuffer = mappedByteBuffer.asDoubleBuffer(); doubleBuffer.get(events, 0, numberOfEvents); fileChannel.close(); fileInputStream.close(); } catch (IOException ioe) { destroy(); throw ioe; } openedAt = System.currentTimeMillis(); } /** * Creates a new MemoryEvents with no values set * @param size the size to create * @throws MemoryEventsException */ public MemoryEvents(int size) throws MemoryEventsException { this.numberOfEvents = size; this.doubleFile = null; setupMemoryEventsArray(numberOfEvents); openedAt = System.currentTimeMillis(); } /** * Copy an IEvents instance * @param source * @throws IOException * @throws MemoryEventsException */ public MemoryEvents(IEvents source) throws IOException, MemoryEventsException { this.numberOfEvents = source.getNumberOfEvents(); // Get the events if (source instanceof MemoryEvents) { // Reuse the existing memory events array to save memory MemoryEvents memoryEventsSource = (MemoryEvents) source; // Copy the memoryEventsArray from source, and track that this new MemoryEvents // instance is referencing it memoryEventsArray = memoryEventsSource.memoryEventsArray; this.events = memoryEventsArray.getEvents(this); this.reusedEventArray = true; } else { setupMemoryEventsArray(numberOfEvents); source.setPosition(0); source.values(events); } this.doubleFile = source.getEventFile(); this.name = source.getName(); this.openedAt = System.currentTimeMillis(); } /** * Gets an event array from the pool, or throw an exception trying * @param size the size of the array to get. * @throws MemoryEventsException if something goes wrong * */ protected void setupMemoryEventsArray(int size) throws MemoryEventsException { // Get an event array from the pool, or throw an exception trying memoryEventsArray = fetchEventArray(numberOfEvents); // get the events array object and track that this instance of MemoryEvents references it events = memoryEventsArray.getEvents(this); } /* (non-Javadoc) * @see facs.IEvents#readOnly() */ @Override public IEvents readOnly() throws IOException { return openNew(); } /* (non-Javadoc) * @see facs.IEvents#getEventFile() */ @Override public File getEventFile() { return doubleFile; } /* (non-Javadoc) * @see facs.IEvents#getFile() */ @Override public File getFile() { // TODO Auto-generated method stub return doubleFile; } /* (non-Javadoc) * @see facs.IEvents#writeToDisk() */ @Override public File writeToDisk() throws IOException { throw new IOException("MemoryEvents cannot be written to disk"); } /* (non-Javadoc) * @see facs.IEvents#getName() */ @Override public String getName() { return name; } /* (non-Javadoc) * @see facs.IEvents#setName(java.lang.String) */ @Override public void setName(String name) { this.name = name; } /* (non-Javadoc) * @see facs.IEvents#get(int) */ @Override public double get(int eventNumber) throws IOException { return events[eventNumber]; } /* (non-Javadoc) * @see facs.IEvents#isReadOnly() */ @Override public boolean isReadOnly() { return true; } /* (non-Javadoc) * @see facs.IEvents#getNumberOfEvents() */ @Override public int getNumberOfEvents() { return numberOfEvents; } /* (non-Javadoc) * @see facs.IEvents#set(int, double) */ @Override public void set(int eventNumber, double value) throws IOException { throw new IOException("MemoryEvents cannot be set."); } /* (non-Javadoc) * @see facs.IEvents#get() */ @Override public double get() throws IOException { return events[position++]; } /* (non-Javadoc) * @see facs.IEvents#put(double) */ @Override public void put(double value) throws IOException { throw new IOException("MemoryEvents cannot be set."); } /* (non-Javadoc) * @see facs.IEvents#setPosition(int) */ @Override public void setPosition(int eventNumber) throws IOException { position = eventNumber; } /* (non-Javadoc) * @see facs.IEvents#values() */ @Override public double[] values() throws IOException { double[] result = new double[numberOfEvents]; values(result); return result; } public void values(double[] eventsArray) throws IOException { System.arraycopy(events, 0, eventsArray, 0, numberOfEvents); } public double[] getLiveEvents() { return events; } /* (non-Javadoc) * @see facs.IEvents#sortedValues() */ @Override public double[] sortedValues() throws IOException { double results[] = values(); sortedValues(results); return results; } /* (non-Javadoc) * @see facs.IEvents#sortedValues() */ @Override public void sortedValues(double[] eventsArray) throws IOException { values(eventsArray); Arrays.sort(eventsArray); } /* (non-Javadoc) * @see facs.IEvents#close() */ @Override public void close() throws IOException { closed = true; // Queue up this MemoryEvents to be written to disk if it hasn't been already // This process also releases the memory events array if (memoryEventsArray.getOnDisk() == null) synchronized (closedMemoryEvents) { closedMemoryEvents.add(this); } } public synchronized void destroy() { closed = true; events = null; if (memoryEventsArray != null) { memoryEventsArray.detrack(this); memoryEventsArray = null; } } /* (non-Javadoc) * @see facs.IEvents#flush() */ @Override public void flush() throws IOException { } /* (non-Javadoc) * @see facs.IEvents#openedFor() */ @Override public long openedFor() { return System.currentTimeMillis() - openedAt; } /* (non-Javadoc) * @see facs.IEvents#eventsFileSize() */ @Override public long eventsFileSize() { return 0; } /* (non-Javadoc) * @see facs.IEvents#memorySize() */ @Override public long memorySize() { return DoubleFile.BYTES_PER_DOUBLE * this.numberOfEvents; } /* (non-Javadoc) * @see facs.IEvents#eventsFilePath() */ @Override public String eventsFilePath() { if (doubleFile == null) return null; else return doubleFile.getPath(); } /* (non-Javadoc) * @see facs.IEvents#openNew() */ @Override public synchronized IEvents openNew() throws IOException, MemoryEventsException { if (this.events != null) return new MemoryEvents(this); else { try { return new MemoryEvents(memoryEventsArray.getOnDisk()); } catch (IOException ioe) { // If no memory events are available, fall back to traditional Events // TODO check endian compatibility return new Events(memoryEventsArray.getOnDisk()); } } } /* (non-Javadoc) * @see facs.IEvents#isClosed() */ @Override public synchronized boolean isClosed() { return closed; } /* (non-Javadoc) * @see facs.IEvents#isWrapped() */ @Override public boolean isWrapped() { return false; } public void finalize() { destroy(); } protected synchronized boolean writeOut() { if (memoryEventsArray == null) return false; File memoryEventsArrayOnDisk = null; synchronized (memoryEventsArray) { memoryEventsArrayOnDisk = memoryEventsArray.getOnDisk(); if (memoryEventsArrayOnDisk != null || events == null) return false; RandomAccessFile randomAccessFile = null; FileChannel fileChannel = null; try { memoryEventsArrayOnDisk = File.createTempFile("memoryEventsArrayOnDisk", "evt"); memoryEventsArrayOnDisk.deleteOnExit(); randomAccessFile = new RandomAccessFile(memoryEventsArrayOnDisk, "rw"); fileChannel = randomAccessFile.getChannel(); MappedByteBuffer mappedByteBuffer = fileChannel.map(MapMode.READ_WRITE, 0, numberOfEvents * BYTES_PER_DOUBLE); DoubleBuffer doubleBuffer = mappedByteBuffer.asDoubleBuffer(); doubleBuffer.put(events, 0, numberOfEvents); mappedByteBuffer.force(); fileChannel.close(); randomAccessFile.close(); memoryEventsArray.setOnDisk(memoryEventsArrayOnDisk); } catch (IOException ioe) { logger.log(Level.INFO, "::::: Failed to write out MemoryEventsArray :::::", ioe); if (memoryEventsArrayOnDisk != null) { memoryEventsArrayOnDisk.delete(); memoryEventsArrayOnDisk = null; } } finally { memoryEventsArrayOnDisk.delete(); } } return true; } public boolean isDependantOnOtherChannels() { return false; } public String toString() { StringBuffer sb = new StringBuffer(); sb.append("<MemoryEvents "); sb.append(name); sb.append(" numberOfEvents: "); sb.append(getNumberOfEvents()); sb.append(", size on disk: "); sb.append(eventsFileSize() / 1024.0); sb.append(" KB" + ", memory size: "); sb.append(memorySize()); sb.append(" KB, readOnly: "); sb.append(true); sb.append(", openedFor: "); // Convert ms to seconds sb.append(openedFor() / 1000); sb.append("s, path: "); sb.append(eventsFilePath()); sb.append(">"); return sb.toString(); } public static MemoryEvents[] getBlockOfMemoryEvents(int numberOfEvents, int numberOfMemoryEvents) throws IOException { MemoryEvents[] results = new MemoryEvents[numberOfMemoryEvents]; long totalNumberOfEvents = numberOfEvents * numberOfMemoryEvents; System.err.println("Creating a block of memory events. Number of memory events: " + numberOfMemoryEvents + ", events/MemoryEvents: " + numberOfEvents + ", total number of events: " + totalNumberOfEvents); try { // Instantiate the memory events and track their live event arrays for (int memoryEventsNumber = 0; memoryEventsNumber < numberOfMemoryEvents; memoryEventsNumber++) { MemoryEvents memoryEvents = new MemoryEvents(numberOfEvents); results[memoryEventsNumber] = memoryEvents; } } catch (IOException ioe) { for (MemoryEvents memoryEvents : results) { if (memoryEvents == null) continue; // Close would write the events out to disk for re-open. That isn't // needed or desired here. memoryEvents.destroy(); } throw ioe; } return results; } protected static MemoryEventsArray fetchEventArray(int size) throws MemoryEventsException { double[] result = null; GenericObjectPool owningPool = null; if (size <= MEDIUM_EVENT_ARRAY_SIZE) { owningPool = mediumEventArrayPool; } else if (size <= LARGE_EVENT_ARRAY_SIZE) { owningPool = largeEventArrayPool; } else { throw new MemoryEventsException("Event array requestd of " + size + " is bigger than the max available of " + LARGE_EVENT_ARRAY_SIZE); } System.err.println("Borrwing events arrays object pool for " + size + " events."); try { result = (double[]) owningPool.borrowObject(); System.err.println("Finished. size: " + result.length); } catch (Exception ignore) { } if (result == null) throw new MemoryEventsException("MemoryEvents not available."); MemoryEventsArray memoryEventsArray = new MemoryEventsArray(result, owningPool); referencedMemoryEventsArrays.add(memoryEventsArray); return memoryEventsArray; } /** * Start the the thread that writes out closed MemoryEvents so that their arrays can be returned to the pool. */ protected static void startCloseThread() { if (closeThread != null) return; closeThread = new Thread("MemoryEventsArray Closed events Writeout") { public void run() { logger.info(this.getName() + " Started"); while (true) { try { int numberOfCleanedUpMemoryEventsWrittenOut = 0; ////////////////////// Write closed memory events out to disk // Fetch the array of closedMemoryEvents events to write out then clear closedMemoryEvents MemoryEvents[] closedMemoryEventsToWriteOut = null; synchronized (closedMemoryEvents) { closedMemoryEventsToWriteOut = new MemoryEvents[closedMemoryEvents.size()]; closedMemoryEventsToWriteOut = closedMemoryEvents.toArray(closedMemoryEventsToWriteOut); closedMemoryEvents.clear(); } // Iterate the list of closedMemoryEvents and write them out for (MemoryEvents memoryEventToWriteOut : closedMemoryEventsToWriteOut) { if (memoryEventToWriteOut == null) continue; if (memoryEventToWriteOut.writeOut()) { memoryEventToWriteOut.closed = true; memoryEventToWriteOut.memoryEventsArray.detrack(memoryEventToWriteOut); numberOfCleanedUpMemoryEventsWrittenOut++; } // Be nice try { Thread.sleep(1); } catch (Throwable ignore) { } } if (numberOfCleanedUpMemoryEventsWrittenOut > 0) { logger.info("::::: Closed MemoryEvents writtent to disk: " + numberOfCleanedUpMemoryEventsWrittenOut + " :::::"); } } catch (Throwable t) { logger.log(Level.INFO, "Problem in MemoryEventsArray close Thread loop", t); } try { Thread.sleep(CLEANUP_INTERVAL); } catch (Throwable t) { logger.log(Level.INFO, this.getName() + " exiting. " + t.toString()); break; } } } }; closeThread.setDaemon(true); closeThread.start(); } /** * Start the tracking monitor to list out opened events */ protected static void startMonitor() { if (cleanupThread != null) return; cleanupThread = new Thread("MemoryEventsArray Cleanup") { public void run() { logger.info(this.getName() + " Started"); int lastNumberOfReferencedMemoryEventsArrays = -1; while (true) { try { int numberOfReferencedMemoryEventsArrays = 0; int numberOfCleanedUpMemoryEventsArrays = 0; ////////////////////// Release no longer used memory referencedMemoryEventsArrays MemoryEventsArray[] iHateThatConcurrentModificationExceptionsExist = new MemoryEventsArray[referencedMemoryEventsArrays .size()]; iHateThatConcurrentModificationExceptionsExist = referencedMemoryEventsArrays .toArray(iHateThatConcurrentModificationExceptionsExist); for (MemoryEventsArray referencedMemoryEventsArray : iHateThatConcurrentModificationExceptionsExist) { // Calling isReferenced returns the events array to their owning pools if (!referencedMemoryEventsArray.releaseIfNotReferenced()) { referencedMemoryEventsArrays.remove(referencedMemoryEventsArray); numberOfCleanedUpMemoryEventsArrays++; } else { numberOfReferencedMemoryEventsArrays++; } } if ((numberOfReferencedMemoryEventsArrays > 0 && numberOfReferencedMemoryEventsArrays != lastNumberOfReferencedMemoryEventsArrays) || numberOfCleanedUpMemoryEventsArrays > 0) { logger.info("::::: Number of active MemoryEventsArray: " + numberOfReferencedMemoryEventsArrays + " Cleaned up: " + numberOfCleanedUpMemoryEventsArrays + " :::::"); } lastNumberOfReferencedMemoryEventsArrays = numberOfReferencedMemoryEventsArrays; ////////////////////// Print pool status for (String poolName : poolsByName.keySet()) { GenericObjectPool pool = poolsByName.get(poolName); int numberOfIdleEvents = pool.getNumIdle(); int numberOfActiveEvents = pool.getNumActive() - numberOfIdleEvents; int numberOfMaxEvents = pool.getMaxActive(); if (numberOfActiveEvents > 0) logger.info("::::: " + poolName + " Number of active memory event arrays: " + numberOfActiveEvents + "/" + numberOfMaxEvents + " Idle: " + numberOfIdleEvents + " :::::"); } } catch (Throwable t) { logger.log(Level.INFO, "Problem in MemoryEventsArray Cleanup Thread loop", t); } try { Thread.sleep(CLEANUP_INTERVAL); } catch (Throwable t) { logger.log(Level.INFO, this.getName() + " exiting. " + t.toString()); break; } } } }; cleanupThread.setDaemon(true); cleanupThread.start(); } }