Java tutorial
/* * This file is part of the Heritrix web crawler (crawler.archive.org). * * Licensed to the Internet Archive (IA) by one or more individual * contributors. * * The IA licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.archive.io; import java.io.File; import java.io.IOException; import java.util.Collection; import java.util.LinkedList; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; /** * Pool of Writers. * * Abstract. Override and pass in the Constructor a factory that creates * {@link WriterPoolMember} implementations. * * @author stack */ public abstract class WriterPool { private final Logger logger = Logger.getLogger(this.getClass().getName()); /** * Used to generate unique filename sequences. */ final protected AtomicInteger serialNo; /** * Default maximum active number of files in the pool. */ public static final int DEFAULT_MAX_ACTIVE = 1; /** Assumed largest possible value of maxActive; pool will have this * maximum capacity, so dynamic changes beyond this number won't work. */ protected static final int LARGEST_MAX_ACTIVE = 255; /** * Maximum time to wait on a free file before considering * making a new one (if not already at max) */ public static final int DEFAULT_MAX_WAIT_FOR_IDLE = 500; /** * File settings. * Keep in data structure rather than as individual values. */ protected final WriterPoolSettings settings; /** maximum number of writers to create at a time*/ protected int maxActive; /** maximum ms to wait before considering creation of a writer */ protected int maxWait; /** current count of active writers; only read/mutated in synchronized blocks */ protected int currentActive = 0; /** round-robin queue of available writers */ protected BlockingQueue<WriterPoolMember> availableWriters; /** system time when writer was last wanted (because one was not ready in time) */ protected long lastWriterNeededTime; /** system time when writer was last 'rolled over' (imminent creation of new file) */ protected long lastWriterRolloverTime; /** * Constructor * @param serial Used to generate unique filename sequences * @param factory Factory that knows how to make a {@link WriterPoolMember}. * @param settings Settings for this pool. * @param poolMaximumActive * @param poolMaximumWait */ public WriterPool(final AtomicInteger serial, final WriterPoolSettings settings, final int poolMaximumActive, final int poolMaximumWait) { logger.info("Initial configuration:" + " prefix=" + settings.getPrefix() + ", template=" + settings.getTemplate() + ", compress=" + settings.getCompress() + ", maxSize=" + settings.getMaxFileSizeBytes() + ", maxActive=" + poolMaximumActive + ", maxWait=" + poolMaximumWait); this.settings = settings; this.maxActive = poolMaximumActive; this.maxWait = poolMaximumWait; availableWriters = new ArrayBlockingQueue<WriterPoolMember>(LARGEST_MAX_ACTIVE, true); this.serialNo = serial; } /** * Check out a {@link WriterPoolMember}. * * This method should be followed by a call to * {@link #returnFile(WriterPoolMember)} or * {@link #invalidateFile(WriterPoolMember)} else pool starts leaking. * * @return Writer checked out of a pool of files or created * @throws IOException Problem getting Writer from pool (Converted * from Exception to IOException so this pool can live as a good citizen * down in depths of ARCSocketFactory). */ public WriterPoolMember borrowFile() throws IOException { WriterPoolMember writer = null; while (writer == null) { try { writer = availableWriters.poll(maxWait, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { // nothing to do but proceed } if (writer == null) { writer = makeNewWriterIfAppropriate(); } } return writer; } /** * Create a new writer instance, if still below maxActive count. * Remember times to help make later decision when writer should * be discarded. * * @return WriterPoolMember or null if already at max */ protected synchronized WriterPoolMember makeNewWriterIfAppropriate() { long now = System.currentTimeMillis(); lastWriterNeededTime = now; if (currentActive < maxActive) { currentActive++; lastWriterRolloverTime = now; return makeWriter(); } return null; } /** * @return new WriterPoolMember of appropriate type */ protected abstract WriterPoolMember makeWriter(); /** * Discard a previously-used writer, cleanly closing it and leaving it out * of the pool. * @param writer * @throws IOException */ public synchronized void destroyWriter(WriterPoolMember writer) throws IOException { currentActive--; writer.close(); } /** * Return a writer, for likely reuse unless (1) writer's current file has * reached its target size; and (2) there's been no demand for additional * writers since the last time a new writer-file was rolled-over. In that * case, the possibly-superfluous writer instance is discarded. * @param writer Writer to return to the pool. * @throws IOException Problem returning File to pool. */ public void returnFile(WriterPoolMember writer) throws IOException { synchronized (this) { if (writer.isOversize()) { // maybe retire writer rather than recycle if (lastWriterNeededTime <= lastWriterRolloverTime) { // no timeouts waiting for recycled writer since last writer rollover destroyWriter(writer); return; } else { // reuse writer instance, causing new file to be created lastWriterRolloverTime = System.currentTimeMillis(); } } } if (!availableWriters.offer(writer)) { logger.log(Level.WARNING, "writer unreturnable to available pool; closing early"); destroyWriter(writer); } } /** * Close and discard a writer that experienced a potentially-corrupting * error. * @param f writer with problem * @throws IOException */ public synchronized void invalidateFile(WriterPoolMember f) throws IOException { try { destroyWriter(f); } catch (Exception e) { // Convert exception. throw new IOException(e.getMessage()); } // It'll have been closed. Rename with an '.invalid' suffix so it // gets attention. File file = f.getFile(); file.renameTo(new File(file.getAbsoluteFile() + WriterPoolMember.INVALID_SUFFIX)); } /** * @return Number of {@link WriterPoolMember}s checked out of pool. * @throws java.lang.UnsupportedOperationException */ public synchronized int getNumActive() throws UnsupportedOperationException { return currentActive - getNumIdle(); } /** * @return Number of {@link WriterPoolMember} instances still in the pool. * @throws java.lang.UnsupportedOperationException */ public int getNumIdle() throws UnsupportedOperationException { return availableWriters.size(); } /** * Close all {@link WriterPoolMember}s in pool. */ public void close() { Collection<WriterPoolMember> writers = drainAllWriters(); for (WriterPoolMember writer : writers) { try { destroyWriter(writer); } catch (IOException e) { logger.log(Level.WARNING, "problem closing writer", e); } } } /** * @return Returns settings. */ public WriterPoolSettings getSettings() { return this.settings; } /** * @return State of the pool string */ protected String getPoolState() { StringBuffer buffer = new StringBuffer("Active "); buffer.append(getNumActive()); buffer.append(" of max "); buffer.append(maxActive); buffer.append(", idle "); buffer.append(getNumIdle()); return buffer.toString(); } /** * Returns the atomic integer used to generate serial numbers * for files. * * @return the serial number generator */ public AtomicInteger getSerialNo() { return serialNo; } /** * Drains all the active writers from {@link #availableWriters}, blocking to * wait for any writers currently in use to become available. * * <p> * When finished with writers, call availableWriters.addAll(...) to put them * back into the rotation. * * @return all the active writers */ protected synchronized Collection<WriterPoolMember> drainAllWriters() { LinkedList<WriterPoolMember> writers = new LinkedList<WriterPoolMember>(); availableWriters.drainTo(writers); while (writers.size() < currentActive) { try { WriterPoolMember w = availableWriters.take(); writers.add(w); } catch (InterruptedException e) { logger.severe("caught " + e + " while waiting for writers to free up; returning only " + writers.size() + " of " + currentActive + " active writers"); break; } } return writers; } public void flush() { Collection<WriterPoolMember> writers = drainAllWriters(); for (WriterPoolMember writer : writers) { try { writer.flush(); } catch (IOException e) { logger.log(Level.WARNING, "problem flushing writer " + writer, e); } } availableWriters.addAll(writers); } public JSONArray jsonStatus() throws JSONException { Collection<WriterPoolMember> writers = drainAllWriters(); JSONArray ja = new JSONArray(); for (WriterPoolMember w : writers) { JSONObject jo = new JSONObject(); jo.put("file", w.getFile()); jo.put("position", w.getPosition()); ja.put(jo); } availableWriters.addAll(writers); return ja; } }