Java tutorial
/* * Copyright (c) 2012. The Genome Analysis Centre, Norwich, UK * MISO project contacts: Robert Davey, Mario Caccamo @ TGAC * ********************************************************************* * * This file is part of MISO. * * MISO is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * MISO is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with MISO. If not, see <http://www.gnu.org/licenses/>. * * ********************************************************************* */ package uk.ac.bbsrc.tgac.miso.tools.run; import java.io.File; import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.PriorityBlockingQueue; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.integration.Message; import org.springframework.integration.MessagingException; import org.springframework.integration.aggregator.ResequencingMessageGroupProcessor; import org.springframework.integration.context.IntegrationObjectSupport; import org.springframework.integration.core.MessageSource; import org.springframework.integration.file.DefaultDirectoryScanner; import org.springframework.integration.file.DirectoryScanner; import org.springframework.integration.file.FileLocker; import org.springframework.integration.file.HeadDirectoryScanner; import org.springframework.integration.file.filters.AcceptOnceFileListFilter; import org.springframework.integration.file.filters.FileListFilter; import org.springframework.integration.support.MessageBuilder; import org.springframework.util.Assert; /** * uk.ac.bbsrc.tgac.miso.notification.core * <p/> * Modified from the source below to provide a list of files as a result, rather than * polling single files sequentially from the queue * * @author Rob Davey * @date 08-Dec-2010 * @since 0.0.2 * * -------------------------------- * * {@link MessageSource} that creates messages from a file system directory. To * prevent messages for certain files, you may supply a * {@link FileListFilter}. By * default, an * {@link AcceptOnceFileListFilter} * is used. It ensures files are picked up only once from the directory. * <p/> * A common problem with reading files is that a file may be detected before it * is ready. The default {@link AcceptOnceFileListFilter} * does not prevent this. In most cases, this can be prevented if the * file-writing process renames each file as soon as it is ready for reading. A * pattern-matching filter that accepts only files that are ready (e.g. based on * a known suffix), composed with the default {@link AcceptOnceFileListFilter} * would allow for this. * <p/> * A {@link Comparator} can be used to ensure internal ordering of the Files in * a {@link PriorityBlockingQueue}. This does not provide the same guarantees as * a {@link ResequencingMessageGroupProcessor}, but in cases where writing files * and failure downstream are rare it might be sufficient. * <p/> * FileReadingMessageSource is fully thread-safe under concurrent * <code>receive()</code> invocations and message delivery callbacks. * * @author Iwein Fuld * @author Mark Fisher * @author Oleg Zhurakousky */ public class FileQueueMessageSource extends IntegrationObjectSupport implements MessageSource<Set<File>> { private static final int DEFAULT_INTERNAL_QUEUE_CAPACITY = 5; private static final Log logger = LogFactory.getLog(FileQueueMessageSource.class); private volatile File directory; private volatile DirectoryScanner scanner = new DefaultDirectoryScanner(); private volatile boolean autoCreateDirectory = true; /* * {@link PriorityBlockingQueue#iterator()} throws * {@link java.util.ConcurrentModificationException} in Java 5. * There is no locking around the queue, so there is also no iteration. */ private final PriorityBlockingQueue<File> toBeReceived; private volatile boolean scanEachPoll = false; /** * Creates a FileReadingMessageSource with a naturally ordered queue of unbounded capacity. */ public FileQueueMessageSource() { this(null); } /** * Creates a FileReadingMessageSource with a bounded queue of the given * capacity. This can be used to reduce the memory footprint of this * component when reading from a large directory. * * @param internalQueueCapacity the size of the queue used to cache files to be received * internally. This queue can be made larger to optimize the * directory scanning. With scanEachPoll set to false and the * queue to a large size, it will be filled once and then * completely emptied before a new directory listing is done. * This is particularly useful to reduce scans of large numbers * of files in a directory. */ public FileQueueMessageSource(int internalQueueCapacity) { this(null); Assert.isTrue(internalQueueCapacity > 0, "Cannot create a queue with non positive capacity"); this.setScanner(new HeadDirectoryScanner(internalQueueCapacity)); } /** * Creates a FileReadingMessageSource with a {@link PriorityBlockingQueue} * ordered with the passed in {@link Comparator} * <p/> * The size of the queue used should be large enough to hold all the files * in the input directory in order to sort all of them, so restricting the * size of the queue is mutually exclusive with ordering. No guarantees * about file delivery order can be made under concurrent access. * <p/> * * @param receptionOrderComparator the comparator to be used to order the files in the internal * queue */ public FileQueueMessageSource(Comparator<File> receptionOrderComparator) { this.toBeReceived = new PriorityBlockingQueue<File>(DEFAULT_INTERNAL_QUEUE_CAPACITY, receptionOrderComparator); } /** * Specify the input directory. * * @param directory to monitor */ public void setDirectory(File directory) { Assert.notNull(directory, "directory must not be null"); this.directory = directory; } /** * Optionally specify a custom scanner, for example the * {@link org.springframework.integration.file.RecursiveLeafOnlyDirectoryScanner} * * @param scanner scanner implementation */ public void setScanner(DirectoryScanner scanner) { this.scanner = scanner; } /** * Specify whether to create the source directory automatically if it does * not yet exist upon initialization. By default, this value is * <emphasis>true</emphasis>. If set to <emphasis>false</emphasis> and the * source directory does not exist, an Exception will be thrown upon * initialization. * * @param autoCreateDirectory should the directory to be monitored be created when this * component starts up? */ public void setAutoCreateDirectory(boolean autoCreateDirectory) { this.autoCreateDirectory = autoCreateDirectory; } /** * Sets a {@link FileListFilter}. By default a * {@link org.springframework.integration.file.filters.AbstractFileListFilter} * with no bounds is used. In most cases a customized {@link FileListFilter} will * be needed to deal with modification and duplication concerns. If multiple * filters are required a * {@link org.springframework.integration.file.filters.CompositeFileListFilter} * can be used to group them together. * <p/> * <b>The supplied filter must be thread safe.</b>. * * @param filter a filter */ public void setFilter(FileListFilter<File> filter) { Assert.notNull(filter, "'filter' must not be null"); this.scanner.setFilter(filter); } /** * Optional. Sets a {@link FileLocker} to be used to guard files against * duplicate processing. * <p/> * <b>The supplied FileLocker must be thread safe</b> * * @param locker a locker */ public void setLocker(FileLocker locker) { Assert.notNull(locker, "'fileLocker' must not be null."); this.scanner.setLocker(locker); } /** * Optional. Set this flag if you want to make sure the internal queue is * refreshed with the latest content of the input directory on each poll. * <p/> * By default this implementation will empty its queue before looking at the * directory again. In cases where order is relevant it is important to * consider the effects of setting this flag. The internal * {@link java.util.concurrent.BlockingQueue} that this class is keeping * will more likely be out of sync with the file system if this flag is set * to <code>false</code>, but it will change more often (causing expensive * reordering) if it is set to <code>true</code>. * * @param scanEachPoll whether or not the component should re-scan (as opposed to not * rescanning until the entire backlog has been delivered) */ public void setScanEachPoll(boolean scanEachPoll) { this.scanEachPoll = scanEachPoll; } public String getComponentType() { return "file:inbound-channel-adapter"; } protected void onInit() { Assert.notNull(directory, "'directory' must not be null"); if (!this.directory.exists() && this.autoCreateDirectory) { this.directory.mkdirs(); } Assert.isTrue(this.directory.exists(), "Source directory [" + directory + "] does not exist."); Assert.isTrue(this.directory.isDirectory(), "Source path [" + this.directory + "] does not point to a directory."); Assert.isTrue(this.directory.canRead(), "Source directory [" + this.directory + "] is not readable."); } public Message<Set<File>> receive() throws MessagingException { Message<Set<File>> message = null; // rescan only if needed or explicitly configured if (scanEachPoll || toBeReceived.isEmpty()) { scanInputDirectory(); } //instead of doing a poll() for a single files, drain the whole queue into a set Set<File> files = new HashSet<File>(); toBeReceived.drainTo(files); for (File file : files) { while ((file != null) && !scanner.tryClaim(file)) { files.remove(file); } } if (!files.isEmpty()) { message = MessageBuilder.withPayload(files).build(); if (logger.isDebugEnabled()) { logger.debug("Created message: [" + message + "]"); } } return message; } private void scanInputDirectory() { List<File> filteredFiles = scanner.listFiles(directory); Set<File> freshFiles = new HashSet<File>(filteredFiles); if (!freshFiles.isEmpty()) { toBeReceived.addAll(freshFiles); if (logger.isDebugEnabled()) { logger.debug("Added to queue: " + freshFiles); } } } /** * Adds the failed message back to the 'toBeReceived' queue if there is room. * * @param failedMessage the {@link org.springframework.integration.Message} that failed */ public void onFailure(Message<File> failedMessage) { if (logger.isWarnEnabled()) { logger.warn("Failed to send: " + failedMessage); } toBeReceived.offer(failedMessage.getPayload()); } /** * The message is just logged. It was already removed from the queue during * the call to <code>receive()</code> * * @param sentMessage the message that was successfully delivered */ public void onSend(Message<File> sentMessage) { if (logger.isDebugEnabled()) { logger.debug("Sent: " + sentMessage); } } }