com.cyberway.issue.io.WriterPool.java Source code

Java tutorial

Introduction

Here is the source code for com.cyberway.issue.io.WriterPool.java

Source

/* WriterPool
 *
 * $Id: WriterPool.java 5479 2007-09-20 01:33:53Z gojomo $
 *
 * Created July 19th, 2006.
 *
 * Copyright (C) 2006 Internet Archive.
 *
 * This file is part of the Heritrix web crawler (crawler.archive.org).
 *
 * Heritrix is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * any later version.
 *
 * Heritrix is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser Public License for more details.
 *
 * You should have received a copy of the GNU Lesser Public License
 * along with Heritrix; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
package com.cyberway.issue.io;

import java.io.File;
import java.io.IOException;
import java.util.NoSuchElementException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.apache.commons.pool.BasePoolableObjectFactory;
import org.apache.commons.pool.impl.FairGenericObjectPool;
import org.apache.commons.pool.impl.GenericObjectPool;

/**
 * Pool of Writers.
 * 
 * Abstract. Override and pass in the Constructor a factory that creates
 * {@link WriterPoolMember} implementations.
 * 
 * @author stack
 */
public abstract class WriterPool {
    final Logger logger = Logger.getLogger(this.getClass().getName());

    /**
     * Used to generate unique filename sequences.
     */
    final private AtomicInteger serialNo;

    /**
     * Don't enforce a maximum number of idle instances in pool.
     * To do so means GenericObjectPool will close files prematurely.
     */
    protected static final int NO_MAX_IDLE = -1;

    /**
     * Retry getting a file on fail the below arbitrary amount of times.
     * This facility is not configurable.  If we fail this many times
     * getting a file, something is seriously wrong.
     */
    private final int arbitraryRetryMax = 10;

    /**
     * Default maximum active number of files in the pool.
     */
    public static final int DEFAULT_MAX_ACTIVE = 5;

    /**
     * Maximum time to wait on a free file..
     */
    public static final int DEFAULT_MAXIMUM_WAIT = 1000 * 60 * 5;

    /**
     * Pool instance.
     */
    private GenericObjectPool pool = null;

    /**
     * File settings.
     * Keep in data structure rather than as individual values.
     */
    private final WriterPoolSettings settings;

    /**
     * Shutdown default constructor.
     */
    private WriterPool() {
        this(null, null, null, -1, -1);
    }

    /**
     * Constructor
     * @param serial  Used to generate unique filename sequences
     * @param factory Factory that knows how to make a {@link WriterPoolMember}.
     * @param settings Settings for this pool.
     * @param poolMaximumActive
     * @param poolMaximumWait
     */
    public WriterPool(final AtomicInteger serial, final BasePoolableObjectFactory factory,
            final WriterPoolSettings settings, final int poolMaximumActive, final int poolMaximumWait) {
        logger.info("Initial configuration:" + " prefix=" + settings.getPrefix() + ", suffix="
                + settings.getSuffix() + ", compress=" + settings.isCompressed() + ", maxSize="
                + settings.getMaxSize() + ", maxActive=" + poolMaximumActive + ", maxWait=" + poolMaximumWait);
        this.settings = settings;
        this.pool = new FairGenericObjectPool(factory, poolMaximumActive, GenericObjectPool.WHEN_EXHAUSTED_BLOCK,
                poolMaximumWait, NO_MAX_IDLE);
        this.serialNo = serial;
    }

    /**
     * Check out a {@link WriterPoolMember}.
     * 
     * This method must be answered by a call to
     * {@link #returnFile(WriterPoolMember)} else pool starts leaking.
     * 
     * @return Writer checked out of a pool of files.
     * @throws IOException Problem getting Writer from pool (Converted
     * from Exception to IOException so this pool can live as a good citizen
     * down in depths of ARCSocketFactory).
     * @throws NoSuchElementException If we time out waiting on a pool member.
     */
    public WriterPoolMember borrowFile() throws IOException {
        WriterPoolMember f = null;
        for (int i = 0; f == null; i++) {
            long waitStart = System.currentTimeMillis();
            try {
                f = (WriterPoolMember) this.pool.borrowObject();
                if (logger.getLevel() == Level.FINE) {
                    logger.fine("Borrowed " + f + " (Pool State: " + getPoolState(waitStart) + ").");
                }
            } catch (NoSuchElementException e) {
                // Let this exception out. Unit test at least depends on it.
                // Log current state of the pool.
                logger.warning(e.getMessage() + ": Retry #" + i + " of " + " max of " + arbitraryRetryMax
                        + ": NSEE Pool State: " + getPoolState(waitStart));
                if (i >= arbitraryRetryMax) {
                    logger.log(Level.SEVERE, "maximum retries exceeded; rethrowing", e);
                    throw e;
                }
            } catch (Exception e) {
                // Convert.
                logger.log(Level.SEVERE, "E Pool State: " + getPoolState(waitStart), e);
                throw new IOException("Failed getting writer from pool: " + e.getMessage());
            }
        }
        return f;
    }

    /**
     * @param writer Writer to return to the pool.
     * @throws IOException Problem returning File to pool.
     */
    public void returnFile(WriterPoolMember writer) throws IOException {
        try {
            if (logger.getLevel() == Level.FINE) {
                logger.fine("Returned " + writer);
            }
            this.pool.returnObject(writer);
        } catch (Exception e) {
            throw new IOException("Failed restoring writer to pool: " + e.getMessage());
        }
    }

    public void invalidateFile(WriterPoolMember f) throws IOException {
        try {
            this.pool.invalidateObject(f);
        } catch (Exception e) {
            // Convert exception.
            throw new IOException(e.getMessage());
        }
        // It'll have been closed.  Rename with an '.invalid' suffix so it
        // gets attention.
        File file = f.getFile();
        file.renameTo(new File(file.getAbsoluteFile() + WriterPoolMember.INVALID_SUFFIX));
    }

    /**
     * @return Number of {@link WriterPoolMember}s checked out of pool.
     * @throws java.lang.UnsupportedOperationException
     */
    public int getNumActive() throws UnsupportedOperationException {
        return this.pool.getNumActive();
    }

    /**
     * @return Number of {@link WriterPoolMember} instances still in the pool.
     * @throws java.lang.UnsupportedOperationException
     */
    public int getNumIdle() throws UnsupportedOperationException {
        return this.pool.getNumIdle();
    }

    /**
     * Close all {@link WriterPoolMember}s in pool.
     */
    public void close() {
        this.pool.clear();
    }

    /**
     * @return Returns settings.
     */
    public WriterPoolSettings getSettings() {
        return this.settings;
    }

    /**
     * @return State of the pool string
     */
    protected String getPoolState() {
        return getPoolState(-1);
    }

    /**
     * @param startTime If we are passed a start time, we'll add difference
     * between it and now to end of string.  Pass -1 if don't want this
     * added to end of state string.
     * @return State of the pool string
     */
    protected String getPoolState(long startTime) {
        StringBuffer buffer = new StringBuffer("Active ");
        buffer.append(getNumActive());
        buffer.append(" of max ");
        buffer.append(this.pool.getMaxActive());
        buffer.append(", idle ");
        buffer.append(this.pool.getNumIdle());
        if (startTime != -1) {
            buffer.append(", time ");
            buffer.append(System.currentTimeMillis() - startTime);
            buffer.append("ms of max ");
            buffer.append(this.pool.getMaxWait());
            buffer.append("ms");
        }
        return buffer.toString();
    }

    /**
     * Returns the atomic integer used to generate serial numbers
     * for files.
     * 
     * @return  the serial number generator
     */
    public AtomicInteger getSerialNo() {
        return serialNo;
    }
}