org.cytobank.fcs_files.events.MemoryEvents.java Source code

Java tutorial

Introduction

Here is the source code for org.cytobank.fcs_files.events.MemoryEvents.java

Source

/**
 * MemoryEvents.java
 *
 * Cytobank (TM) is server and client software for web-based management, analysis,
 * and sharing of flow cytometry data.
 *
 * Copyright (C) 2009 Cytobank, Inc.  All rights reserved.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * Cytobank, Inc.
 * 659 Oak Grove Avenue #205
 * Menlo Park, CA 94025
 *
 * http://www.cytobank.org
 */
package org.cytobank.fcs_files.events;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.DoubleBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Vector;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.apache.commons.pool.impl.GenericObjectPool;
import org.cytobank.io.DoubleFile;

public class MemoryEvents implements IEvents {

    public static final int BYTES_PER_DOUBLE = Double.SIZE / Byte.SIZE;

    public static final int THOUSAND = 1000;
    public static final int MILLION = THOUSAND * THOUSAND;

    /** Maximum wait time for a buffer before an exception */
    public static final int MAX_WAIT_MS = 0;

    /** Maximum idle time for a buffer before eviction */
    public static final int MAX_IDLE_MS = -1;

    public static final int MEDIUM_EVENT_ARRAY_SIZE = 220 * THOUSAND;
    public static final int LARGE_EVENT_ARRAY_SIZE = 8 * MILLION;

    public static final int MEDIUM_EVENT_ARRAY_BUFFERS = 11 * 17; // ( 187 buffers; 329 MB; ~76% of all experiments)
    public static final int LARGE_EVENT_ARRAY_BUFFERS = 20; // ( 20 buffers; 1.280 GB; ~TOP 24% of large experiments)

    protected static final Logger logger = Logger.getLogger(MemoryEvents.class.getPackage().getName());

    protected File doubleFile;
    protected MemoryEventsArray memoryEventsArray;
    protected double[] events;
    protected int numberOfEvents = -1;
    protected String name;
    protected long openedAt = 0;
    protected boolean reusedEventArray = false;

    protected int position = 0;

    protected boolean closed = false;

    /**
     * How often to cleanup in ms
     */
    public static final int CLEANUP_INTERVAL = 1000;

    protected static Thread cleanupThread;
    protected static Thread closeThread;

    protected static Vector<MemoryEventsArray> referencedMemoryEventsArrays = new Vector<MemoryEventsArray>();

    protected static final GenericObjectPool mediumEventArrayPool, largeEventArrayPool;
    protected static final HashMap<String, GenericObjectPool> poolsByName = new HashMap<String, GenericObjectPool>();
    protected static final HashSet<MemoryEvents> closedMemoryEvents = new HashSet<MemoryEvents>();

    static {
        EventArrayPoolableObjectFactory mediumEventArrayPoolableObjectFactory = new EventArrayPoolableObjectFactory(
                MEDIUM_EVENT_ARRAY_SIZE);
        EventArrayPoolableObjectFactory largeEventArrayPoolableObjectFactory = new EventArrayPoolableObjectFactory(
                LARGE_EVENT_ARRAY_SIZE);

        //logger.info("Setting up MemoryEvents buffer pool with " + MAX_BUFFERS + " buffers of " + BUFFER_SIZE_IN_BYTES + " bytes (total memory size: " + MAX_BUFFERS * BUFFER_SIZE_IN_BYTES + " bytes), " +  MAX_WAIT_MS + " ms max wait before exception, and " + MAX_IDLE_MS + " ms max idle.");
        // Setup the object pool to block when exhausted, never evict, and to not bother testing on borrow and return
        mediumEventArrayPool = new GenericObjectPool(mediumEventArrayPoolableObjectFactory,
                MEDIUM_EVENT_ARRAY_BUFFERS, GenericObjectPool.WHEN_EXHAUSTED_FAIL, MAX_WAIT_MS, MAX_IDLE_MS, true,
                false);
        largeEventArrayPool = new GenericObjectPool(largeEventArrayPoolableObjectFactory, LARGE_EVENT_ARRAY_BUFFERS,
                GenericObjectPool.WHEN_EXHAUSTED_FAIL, MAX_WAIT_MS, MAX_IDLE_MS, true, false);

        poolsByName.put("Medium Event Array Pool ( " + MEDIUM_EVENT_ARRAY_SIZE + " doubles)", mediumEventArrayPool);
        poolsByName.put("Large Event Array Pool ( " + LARGE_EVENT_ARRAY_SIZE + " doubles)", largeEventArrayPool);

        // Start the monitor
        startMonitor();
        startCloseThread();

    }

    public MemoryEvents(File doubleFile) throws IOException, MemoryEventsException {
        this.doubleFile = doubleFile;

        numberOfEvents = (int) doubleFile.length() / BYTES_PER_DOUBLE;

        setupMemoryEventsArray(numberOfEvents);

        try {
            // Read in the doubles file
            FileInputStream fileInputStream = new FileInputStream(doubleFile);
            FileChannel fileChannel = fileInputStream.getChannel();
            MappedByteBuffer mappedByteBuffer = fileChannel.map(MapMode.READ_ONLY, 0, doubleFile.length());
            DoubleBuffer doubleBuffer = mappedByteBuffer.asDoubleBuffer();

            doubleBuffer.get(events, 0, numberOfEvents);

            fileChannel.close();
            fileInputStream.close();
        } catch (IOException ioe) {
            destroy();
            throw ioe;
        }

        openedAt = System.currentTimeMillis();
    }

    /**
     * Creates a new MemoryEvents with no values set
     * @param size the size to create
     * @throws MemoryEventsException
     */
    public MemoryEvents(int size) throws MemoryEventsException {
        this.numberOfEvents = size;
        this.doubleFile = null;

        setupMemoryEventsArray(numberOfEvents);

        openedAt = System.currentTimeMillis();
    }

    /**
     * Copy an IEvents instance
     * @param source
     * @throws IOException
     * @throws MemoryEventsException
     */
    public MemoryEvents(IEvents source) throws IOException, MemoryEventsException {
        this.numberOfEvents = source.getNumberOfEvents();

        // Get the events
        if (source instanceof MemoryEvents) {
            // Reuse the existing memory events array to save memory
            MemoryEvents memoryEventsSource = (MemoryEvents) source;

            // Copy the memoryEventsArray from source, and track that this new MemoryEvents
            // instance is referencing it
            memoryEventsArray = memoryEventsSource.memoryEventsArray;
            this.events = memoryEventsArray.getEvents(this);

            this.reusedEventArray = true;
        } else {
            setupMemoryEventsArray(numberOfEvents);
            source.setPosition(0);
            source.values(events);
        }

        this.doubleFile = source.getEventFile();
        this.name = source.getName();

        this.openedAt = System.currentTimeMillis();
    }

    /**
     * Gets an event array from the pool, or throw an exception trying
     * @param size the size of the array to get.
     * @throws MemoryEventsException if something goes wrong
     *
     */
    protected void setupMemoryEventsArray(int size) throws MemoryEventsException {
        // Get an event array from the pool, or throw an exception trying
        memoryEventsArray = fetchEventArray(numberOfEvents);

        // get the events array object and track that this instance of MemoryEvents references it
        events = memoryEventsArray.getEvents(this);
    }

    /* (non-Javadoc)
     * @see facs.IEvents#readOnly()
     */
    @Override
    public IEvents readOnly() throws IOException {
        return openNew();
    }

    /* (non-Javadoc)
     * @see facs.IEvents#getEventFile()
     */
    @Override
    public File getEventFile() {
        return doubleFile;
    }

    /* (non-Javadoc)
     * @see facs.IEvents#getFile()
     */
    @Override
    public File getFile() {
        // TODO Auto-generated method stub
        return doubleFile;
    }

    /* (non-Javadoc)
     * @see facs.IEvents#writeToDisk()
     */
    @Override
    public File writeToDisk() throws IOException {
        throw new IOException("MemoryEvents cannot be written to disk");
    }

    /* (non-Javadoc)
     * @see facs.IEvents#getName()
     */
    @Override
    public String getName() {
        return name;
    }

    /* (non-Javadoc)
     * @see facs.IEvents#setName(java.lang.String)
     */
    @Override
    public void setName(String name) {
        this.name = name;
    }

    /* (non-Javadoc)
     * @see facs.IEvents#get(int)
     */
    @Override
    public double get(int eventNumber) throws IOException {
        return events[eventNumber];
    }

    /* (non-Javadoc)
     * @see facs.IEvents#isReadOnly()
     */
    @Override
    public boolean isReadOnly() {
        return true;
    }

    /* (non-Javadoc)
     * @see facs.IEvents#getNumberOfEvents()
     */
    @Override
    public int getNumberOfEvents() {
        return numberOfEvents;
    }

    /* (non-Javadoc)
     * @see facs.IEvents#set(int, double)
     */
    @Override
    public void set(int eventNumber, double value) throws IOException {
        throw new IOException("MemoryEvents cannot be set.");
    }

    /* (non-Javadoc)
     * @see facs.IEvents#get()
     */
    @Override
    public double get() throws IOException {
        return events[position++];
    }

    /* (non-Javadoc)
     * @see facs.IEvents#put(double)
     */
    @Override
    public void put(double value) throws IOException {
        throw new IOException("MemoryEvents cannot be set.");
    }

    /* (non-Javadoc)
     * @see facs.IEvents#setPosition(int)
     */
    @Override
    public void setPosition(int eventNumber) throws IOException {
        position = eventNumber;
    }

    /* (non-Javadoc)
     * @see facs.IEvents#values()
     */
    @Override
    public double[] values() throws IOException {
        double[] result = new double[numberOfEvents];

        values(result);

        return result;
    }

    public void values(double[] eventsArray) throws IOException {
        System.arraycopy(events, 0, eventsArray, 0, numberOfEvents);
    }

    public double[] getLiveEvents() {
        return events;
    }

    /* (non-Javadoc)
     * @see facs.IEvents#sortedValues()
     */
    @Override
    public double[] sortedValues() throws IOException {
        double results[] = values();
        sortedValues(results);
        return results;
    }

    /* (non-Javadoc)
     * @see facs.IEvents#sortedValues()
     */
    @Override
    public void sortedValues(double[] eventsArray) throws IOException {
        values(eventsArray);
        Arrays.sort(eventsArray);
    }

    /* (non-Javadoc)
     * @see facs.IEvents#close()
     */
    @Override
    public void close() throws IOException {
        closed = true;

        // Queue up this MemoryEvents to be written to disk if it hasn't been already
        // This process also releases the memory events array
        if (memoryEventsArray.getOnDisk() == null)
            synchronized (closedMemoryEvents) {
                closedMemoryEvents.add(this);
            }
    }

    public synchronized void destroy() {
        closed = true;
        events = null;

        if (memoryEventsArray != null) {
            memoryEventsArray.detrack(this);
            memoryEventsArray = null;
        }
    }

    /* (non-Javadoc)
     * @see facs.IEvents#flush()
     */
    @Override
    public void flush() throws IOException {
    }

    /* (non-Javadoc)
     * @see facs.IEvents#openedFor()
     */
    @Override
    public long openedFor() {
        return System.currentTimeMillis() - openedAt;
    }

    /* (non-Javadoc)
     * @see facs.IEvents#eventsFileSize()
     */
    @Override
    public long eventsFileSize() {
        return 0;
    }

    /* (non-Javadoc)
     * @see facs.IEvents#memorySize()
     */
    @Override
    public long memorySize() {
        return DoubleFile.BYTES_PER_DOUBLE * this.numberOfEvents;
    }

    /* (non-Javadoc)
     * @see facs.IEvents#eventsFilePath()
     */
    @Override
    public String eventsFilePath() {
        if (doubleFile == null)
            return null;
        else
            return doubleFile.getPath();
    }

    /* (non-Javadoc)
     * @see facs.IEvents#openNew()
     */
    @Override
    public synchronized IEvents openNew() throws IOException, MemoryEventsException {
        if (this.events != null)
            return new MemoryEvents(this);
        else {
            try {
                return new MemoryEvents(memoryEventsArray.getOnDisk());
            } catch (IOException ioe) {
                // If no memory events are available, fall back to traditional Events
                // TODO check endian compatibility
                return new Events(memoryEventsArray.getOnDisk());
            }
        }
    }

    /* (non-Javadoc)
     * @see facs.IEvents#isClosed()
     */
    @Override
    public synchronized boolean isClosed() {
        return closed;
    }

    /* (non-Javadoc)
     * @see facs.IEvents#isWrapped()
     */
    @Override
    public boolean isWrapped() {
        return false;
    }

    public void finalize() {
        destroy();
    }

    protected synchronized boolean writeOut() {
        if (memoryEventsArray == null)
            return false;

        File memoryEventsArrayOnDisk = null;

        synchronized (memoryEventsArray) {
            memoryEventsArrayOnDisk = memoryEventsArray.getOnDisk();

            if (memoryEventsArrayOnDisk != null || events == null)
                return false;

            RandomAccessFile randomAccessFile = null;
            FileChannel fileChannel = null;

            try {
                memoryEventsArrayOnDisk = File.createTempFile("memoryEventsArrayOnDisk", "evt");
                memoryEventsArrayOnDisk.deleteOnExit();
                randomAccessFile = new RandomAccessFile(memoryEventsArrayOnDisk, "rw");
                fileChannel = randomAccessFile.getChannel();

                MappedByteBuffer mappedByteBuffer = fileChannel.map(MapMode.READ_WRITE, 0,
                        numberOfEvents * BYTES_PER_DOUBLE);
                DoubleBuffer doubleBuffer = mappedByteBuffer.asDoubleBuffer();
                doubleBuffer.put(events, 0, numberOfEvents);
                mappedByteBuffer.force();
                fileChannel.close();
                randomAccessFile.close();

                memoryEventsArray.setOnDisk(memoryEventsArrayOnDisk);

            } catch (IOException ioe) {
                logger.log(Level.INFO, "::::: Failed to write out MemoryEventsArray :::::", ioe);
                if (memoryEventsArrayOnDisk != null) {
                    memoryEventsArrayOnDisk.delete();
                    memoryEventsArrayOnDisk = null;
                }
            } finally {
                memoryEventsArrayOnDisk.delete();
            }
        }

        return true;

    }

    public boolean isDependantOnOtherChannels() {
        return false;
    }

    public String toString() {
        StringBuffer sb = new StringBuffer();

        sb.append("<MemoryEvents ");
        sb.append(name);
        sb.append(" numberOfEvents: ");
        sb.append(getNumberOfEvents());
        sb.append(", size on disk: ");
        sb.append(eventsFileSize() / 1024.0);
        sb.append(" KB" + ", memory size: ");
        sb.append(memorySize());
        sb.append(" KB, readOnly: ");
        sb.append(true);
        sb.append(", openedFor: ");
        // Convert ms to seconds
        sb.append(openedFor() / 1000);
        sb.append("s, path: ");
        sb.append(eventsFilePath());

        sb.append(">");
        return sb.toString();
    }

    public static MemoryEvents[] getBlockOfMemoryEvents(int numberOfEvents, int numberOfMemoryEvents)
            throws IOException {
        MemoryEvents[] results = new MemoryEvents[numberOfMemoryEvents];

        long totalNumberOfEvents = numberOfEvents * numberOfMemoryEvents;
        System.err.println("Creating a block of memory events.  Number of memory events: " + numberOfMemoryEvents
                + ", events/MemoryEvents: " + numberOfEvents + ", total number of events: " + totalNumberOfEvents);

        try {
            // Instantiate the memory events and track their live event arrays
            for (int memoryEventsNumber = 0; memoryEventsNumber < numberOfMemoryEvents; memoryEventsNumber++) {
                MemoryEvents memoryEvents = new MemoryEvents(numberOfEvents);
                results[memoryEventsNumber] = memoryEvents;
            }
        } catch (IOException ioe) {
            for (MemoryEvents memoryEvents : results) {
                if (memoryEvents == null)
                    continue;

                // Close would write the events out to disk for re-open.  That isn't
                // needed or desired here.
                memoryEvents.destroy();
            }

            throw ioe;
        }

        return results;
    }

    protected static MemoryEventsArray fetchEventArray(int size) throws MemoryEventsException {
        double[] result = null;
        GenericObjectPool owningPool = null;
        if (size <= MEDIUM_EVENT_ARRAY_SIZE) {
            owningPool = mediumEventArrayPool;
        } else if (size <= LARGE_EVENT_ARRAY_SIZE) {
            owningPool = largeEventArrayPool;
        } else {
            throw new MemoryEventsException("Event array requestd of " + size
                    + " is bigger than the max available of " + LARGE_EVENT_ARRAY_SIZE);
        }

        System.err.println("Borrwing events arrays object pool for " + size + " events.");
        try {
            result = (double[]) owningPool.borrowObject();
            System.err.println("Finished. size: " + result.length);
        } catch (Exception ignore) {
        }

        if (result == null)
            throw new MemoryEventsException("MemoryEvents not available.");

        MemoryEventsArray memoryEventsArray = new MemoryEventsArray(result, owningPool);

        referencedMemoryEventsArrays.add(memoryEventsArray);

        return memoryEventsArray;
    }

    /**
     * Start the the thread that writes out closed MemoryEvents so that their arrays can be returned to the pool.
     */
    protected static void startCloseThread() {
        if (closeThread != null)
            return;

        closeThread = new Thread("MemoryEventsArray Closed events Writeout") {
            public void run() {
                logger.info(this.getName() + " Started");

                while (true) {
                    try {
                        int numberOfCleanedUpMemoryEventsWrittenOut = 0;

                        ////////////////////// Write closed memory events out to disk

                        // Fetch the array of closedMemoryEvents events to write out then clear closedMemoryEvents
                        MemoryEvents[] closedMemoryEventsToWriteOut = null;
                        synchronized (closedMemoryEvents) {
                            closedMemoryEventsToWriteOut = new MemoryEvents[closedMemoryEvents.size()];
                            closedMemoryEventsToWriteOut = closedMemoryEvents.toArray(closedMemoryEventsToWriteOut);
                            closedMemoryEvents.clear();
                        }

                        // Iterate the list of closedMemoryEvents and write them out
                        for (MemoryEvents memoryEventToWriteOut : closedMemoryEventsToWriteOut) {
                            if (memoryEventToWriteOut == null)
                                continue;
                            if (memoryEventToWriteOut.writeOut()) {
                                memoryEventToWriteOut.closed = true;
                                memoryEventToWriteOut.memoryEventsArray.detrack(memoryEventToWriteOut);
                                numberOfCleanedUpMemoryEventsWrittenOut++;
                            }

                            // Be nice
                            try {
                                Thread.sleep(1);
                            } catch (Throwable ignore) {
                            }
                        }

                        if (numberOfCleanedUpMemoryEventsWrittenOut > 0) {
                            logger.info("::::: Closed MemoryEvents writtent to disk: "
                                    + numberOfCleanedUpMemoryEventsWrittenOut + " :::::");
                        }

                    } catch (Throwable t) {
                        logger.log(Level.INFO, "Problem in MemoryEventsArray close Thread loop", t);
                    }

                    try {
                        Thread.sleep(CLEANUP_INTERVAL);
                    } catch (Throwable t) {
                        logger.log(Level.INFO, this.getName() + " exiting. " + t.toString());
                        break;
                    }
                }
            }
        };

        closeThread.setDaemon(true);
        closeThread.start();
    }

    /**
     * Start the tracking monitor to list out opened events
     */
    protected static void startMonitor() {
        if (cleanupThread != null)
            return;

        cleanupThread = new Thread("MemoryEventsArray Cleanup") {
            public void run() {
                logger.info(this.getName() + " Started");

                int lastNumberOfReferencedMemoryEventsArrays = -1;
                while (true) {
                    try {
                        int numberOfReferencedMemoryEventsArrays = 0;
                        int numberOfCleanedUpMemoryEventsArrays = 0;

                        ////////////////////// Release no longer used memory referencedMemoryEventsArrays

                        MemoryEventsArray[] iHateThatConcurrentModificationExceptionsExist = new MemoryEventsArray[referencedMemoryEventsArrays
                                .size()];
                        iHateThatConcurrentModificationExceptionsExist = referencedMemoryEventsArrays
                                .toArray(iHateThatConcurrentModificationExceptionsExist);
                        for (MemoryEventsArray referencedMemoryEventsArray : iHateThatConcurrentModificationExceptionsExist) {

                            // Calling isReferenced returns the events array to their owning pools
                            if (!referencedMemoryEventsArray.releaseIfNotReferenced()) {
                                referencedMemoryEventsArrays.remove(referencedMemoryEventsArray);
                                numberOfCleanedUpMemoryEventsArrays++;
                            } else {
                                numberOfReferencedMemoryEventsArrays++;
                            }
                        }

                        if ((numberOfReferencedMemoryEventsArrays > 0
                                && numberOfReferencedMemoryEventsArrays != lastNumberOfReferencedMemoryEventsArrays)
                                || numberOfCleanedUpMemoryEventsArrays > 0) {
                            logger.info("::::: Number of active MemoryEventsArray: "
                                    + numberOfReferencedMemoryEventsArrays + " Cleaned up: "
                                    + numberOfCleanedUpMemoryEventsArrays + " :::::");
                        }

                        lastNumberOfReferencedMemoryEventsArrays = numberOfReferencedMemoryEventsArrays;

                        ////////////////////// Print pool status

                        for (String poolName : poolsByName.keySet()) {
                            GenericObjectPool pool = poolsByName.get(poolName);
                            int numberOfIdleEvents = pool.getNumIdle();
                            int numberOfActiveEvents = pool.getNumActive() - numberOfIdleEvents;
                            int numberOfMaxEvents = pool.getMaxActive();

                            if (numberOfActiveEvents > 0)
                                logger.info("::::: " + poolName + " Number of active memory event arrays: "
                                        + numberOfActiveEvents + "/" + numberOfMaxEvents + " Idle: "
                                        + numberOfIdleEvents + " :::::");
                        }
                    } catch (Throwable t) {
                        logger.log(Level.INFO, "Problem in MemoryEventsArray Cleanup Thread loop", t);
                    }

                    try {
                        Thread.sleep(CLEANUP_INTERVAL);
                    } catch (Throwable t) {
                        logger.log(Level.INFO, this.getName() + " exiting. " + t.toString());
                        break;
                    }
                }
            }
        };

        cleanupThread.setDaemon(true);
        cleanupThread.start();
    }
}