com.adaptris.core.fs.NonDeletingFsConsumerTest.java Source code

Java tutorial

Introduction

Here is the source code for com.adaptris.core.fs.NonDeletingFsConsumerTest.java

Source

/*
 * Copyright 2015 Adaptris Ltd.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/

package com.adaptris.core.fs;

import java.io.File;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.PrintStream;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

import org.apache.commons.io.FileUtils;
import org.apache.oro.io.Perl5FilenameFilter;

import com.adaptris.core.AdaptrisMessage;
import com.adaptris.core.ConfiguredConsumeDestination;
import com.adaptris.core.CoreConstants;
import com.adaptris.core.CoreException;
import com.adaptris.core.FixedIntervalPoller;
import com.adaptris.core.PollerImp;
import com.adaptris.core.StandaloneConsumer;
import com.adaptris.core.stubs.MockMessageListener;
import com.adaptris.core.stubs.TempFileUtils;
import com.adaptris.core.util.LifecycleHelper;
import com.adaptris.util.GuidGenerator;
import com.adaptris.util.TimeInterval;

@SuppressWarnings("deprecation")
public class NonDeletingFsConsumerTest extends FsConsumerCase {
    private static final String PANGRAM_1 = "the quick brown fox jumps over the lazy dog";
    /**
     * Key in unit-test.properties that defines where example goes unless overriden {@link #setBaseDir(String)}.
     * 
     */
    public static final String EXAMPLE_BASEDIR = "NonDeletingFsConsumerExample.baseDir";

    public NonDeletingFsConsumerTest(java.lang.String testName) {
        super(testName);
    }

    @Override
    protected void setUp() throws Exception {
    }

    @Override
    protected void tearDown() throws Exception {
    }

    @Override
    protected void configureExampleConfigBaseDir() {
        if (PROPERTIES.getProperty(EXAMPLE_BASEDIR) != null) {
            setBaseDir(PROPERTIES.getProperty(EXAMPLE_BASEDIR));
        }
    }

    public void testConsume() throws Exception {
        String subDir = new GuidGenerator().safeUUID();
        MockMessageListener stub = new MockMessageListener(10);
        NonDeletingFsConsumer fs = createConsumer(subDir, "testConsume");
        fs.setPoller(new FixedIntervalPoller(new TimeInterval(300L, TimeUnit.MILLISECONDS)));
        StandaloneConsumer sc = new StandaloneConsumer(fs);
        sc.registerAdaptrisMessageListener(stub);
        int count = 10;
        File parentDir = FsHelper
                .createFileReference(FsHelper.createUrlFromString(PROPERTIES.getProperty(BASE_KEY), true));
        try {
            File baseDir = new File(parentDir, subDir);
            LifecycleHelper.init(sc);
            createFiles(baseDir, ".xml", count);
            LifecycleHelper.start(sc);
            waitForMessages(stub, count);
            assertMessages(stub.getMessages(), count,
                    baseDir.listFiles((FilenameFilter) new Perl5FilenameFilter(".*\\.xml")));
        } catch (Exception e) {
            log.warn(e.getMessage(), e);
        } finally {
            stop(sc);
            FileUtils.deleteQuietly(new File(parentDir, subDir));
        }
    }

    public void testBug2100_OriginalNameContainsWip() throws Exception {
        String subDir = new GuidGenerator().safeUUID();
        MockMessageListener stub = new MockMessageListener(10);
        NonDeletingFsConsumer fs = createConsumer(subDir, "testBug2100_OriginalNameContainsWip");
        fs.setReacquireLockBetweenMessages(true);
        fs.setPoller(new FixedIntervalPoller(new TimeInterval(300L, TimeUnit.MILLISECONDS)));
        StandaloneConsumer sc = new StandaloneConsumer(fs);
        sc.registerAdaptrisMessageListener(stub);
        int count = 10;
        File parentDir = FsHelper
                .createFileReference(FsHelper.createUrlFromString(PROPERTIES.getProperty(BASE_KEY), true));
        try {
            File baseDir = new File(parentDir, subDir);
            LifecycleHelper.init(sc);
            createFiles(baseDir, ".xml", count);
            LifecycleHelper.start(sc);
            waitForMessages(stub, count);
            assertMessages(stub.getMessages(), count,
                    baseDir.listFiles((FilenameFilter) new Perl5FilenameFilter(".*\\.xml")));
            for (AdaptrisMessage msg : stub.getMessages()) {
                assertFalse("original name should not contain '.wip'",
                        msg.getMetadataValue(CoreConstants.ORIGINAL_NAME_KEY).endsWith(".wip"));
            }
        } finally {
            stop(sc);
            FileUtils.deleteQuietly(new File(parentDir, subDir));
        }
    }

    public void testConsumeNotReprocessed() throws Exception {
        String subDir = new GuidGenerator().safeUUID();
        MockMessageListener stub = new MockMessageListener(10);
        NonDeletingFsConsumer fs = createConsumer(subDir, "testConsume");
        AtomicBoolean pollFired = new AtomicBoolean(false);
        fs.setPoller(
                new FixedIntervalPoller(new TimeInterval(500L, TimeUnit.MILLISECONDS)).withPollerCallback(e -> {
                    if (e == 0) {
                        pollFired.set(true);
                    }
                }));
        StandaloneConsumer sc = new StandaloneConsumer(fs);
        sc.registerAdaptrisMessageListener(stub);
        int count = 10;
        File parentDir = FsHelper
                .createFileReference(FsHelper.createUrlFromString(PROPERTIES.getProperty(BASE_KEY), true));
        try {
            File baseDir = new File(parentDir, subDir);
            LifecycleHelper.init(sc);
            createFiles(baseDir, ".xml", count);
            LifecycleHelper.start(sc);
            waitForMessages(stub, count);
            // The next call back should be on the next poll, when messages == 0;
            waitForPollCallback(pollFired);
            // that we don't reprocess them.
            assertMessages(stub.getMessages(), count,
                    baseDir.listFiles((FilenameFilter) new Perl5FilenameFilter(".*\\.xml")));
        } catch (Exception e) {
            log.warn(e.getMessage(), e);
        } finally {
            stop(sc);
            FileUtils.deleteQuietly(new File(parentDir, subDir));
        }
    }

    public void testConsumeWithQuietPeriod() throws Exception {
        String subDir = new GuidGenerator().safeUUID();
        MockMessageListener stub = new MockMessageListener(10);
        NonDeletingFsConsumer fs = createConsumer(subDir, "testConsumeWithQuietPeriod");
        fs.setQuietInterval(new TimeInterval(300L, TimeUnit.MILLISECONDS));
        fs.setPoller(new FixedIntervalPoller(new TimeInterval(300L, TimeUnit.MILLISECONDS)));
        StandaloneConsumer sc = new StandaloneConsumer(fs);
        sc.registerAdaptrisMessageListener(stub);
        int count = 10;
        File parentDir = FsHelper
                .createFileReference(FsHelper.createUrlFromString(PROPERTIES.getProperty(BASE_KEY), true));
        try {
            File baseDir = new File(parentDir, subDir);
            LifecycleHelper.init(sc);
            createFiles(baseDir, ".xml", count);
            LifecycleHelper.start(sc);
            waitForMessages(stub, count);
            assertMessages(stub.getMessages(), count,
                    baseDir.listFiles((FilenameFilter) new Perl5FilenameFilter(".*\\.xml")));
        } finally {
            stop(sc);
            FileUtils.deleteQuietly(new File(parentDir, subDir));

        }
    }

    public void testBug1675ConsumeWithSpacesInDir() throws Exception {
        String uniqueName = new GuidGenerator().safeUUID();
        String subDir = uniqueName + "Directory WithSpaces";
        String subDirEncoded = uniqueName + "Directory%20WithSpaces";
        MockMessageListener stub = new MockMessageListener(10);
        NonDeletingFsConsumer fs = createConsumer(subDirEncoded, "testBug1675ConsumeWithSpacesInDir");
        fs.setPoller(new FixedIntervalPoller(new TimeInterval(300L, TimeUnit.MILLISECONDS)));
        StandaloneConsumer sc = new StandaloneConsumer(fs);
        sc.registerAdaptrisMessageListener(stub);
        int count = 10;
        File parentDir = FsHelper
                .createFileReference(FsHelper.createUrlFromString(PROPERTIES.getProperty(BASE_KEY), true));
        try {
            File baseDir = new File(parentDir, subDir);
            LifecycleHelper.init(sc);
            createFiles(baseDir, ".xml", count);
            LifecycleHelper.start(sc);
            waitForMessages(stub, count);
            assertMessages(stub.getMessages(), count,
                    baseDir.listFiles((FilenameFilter) new Perl5FilenameFilter(".*\\.xml")));
        } catch (Exception e) {
            log.debug(e.getMessage(), e);
            throw e;
        } finally {
            stop(sc);
            FileUtils.deleteQuietly(new File(parentDir, subDir));

        }
    }

    public void testConsumeWithFilter() throws Exception {
        String subDir = new GuidGenerator().safeUUID();
        MockMessageListener stub = new MockMessageListener(10);
        NonDeletingFsConsumer fs = createConsumer(subDir, "testConsumeWithFilter");
        ((ConfiguredConsumeDestination) fs.getDestination()).setFilterExpression(".*\\.xml");
        fs.setFileFilterImp(Perl5FilenameFilter.class.getName());
        fs.setPoller(new FixedIntervalPoller(new TimeInterval(300L, TimeUnit.MILLISECONDS)));
        StandaloneConsumer sc = new StandaloneConsumer(fs);
        sc.registerAdaptrisMessageListener(stub);
        int count = 10;
        File parentDir = FsHelper
                .createFileReference(FsHelper.createUrlFromString(PROPERTIES.getProperty(BASE_KEY), true));
        try {
            File baseDir = new File(parentDir, subDir);
            LifecycleHelper.init(sc);
            createFiles(baseDir, ".xml", count);
            createFiles(baseDir, ".tmp", count);
            LifecycleHelper.start(sc);
            waitForMessages(stub, count);

            Perl5FilenameFilter wip = new Perl5FilenameFilter(".*\\.tmp");
            assertEquals("TMP Files remain", count, baseDir.listFiles((FilenameFilter) wip).length);
            assertMessages(stub.getMessages(), count,
                    baseDir.listFiles((FilenameFilter) new Perl5FilenameFilter(".*\\.xml")));
        } finally {
            stop(sc);
            FileUtils.deleteQuietly(new File(parentDir, subDir));

        }
    }

    public void testHasChanged_NotInCache() throws Exception {
        String subDir = new GuidGenerator().safeUUID();
        NonDeletingFsConsumer fs = createConsumer(subDir, "testConsume");
        File f = createFile(TempFileUtils.createTrackedFile(getName(), "", fs), PANGRAM_1);
        ProcessedItem item = new ProcessedItem(f.getAbsolutePath(), 0, f.length());
        ProcessedItemCache cache = new InlineItemCache();
        fs.setPoller(new Never());
        fs.setProcessedItemCache(cache);

        StandaloneConsumer consumer = new StandaloneConsumer(fs);
        try {
            start(consumer);
            assertTrue(fs.hasChanged(item));
        } finally {
            stop(consumer);
        }
    }

    public void testHasChanged_LastModified() throws Exception {
        String subDir = new GuidGenerator().safeUUID();
        NonDeletingFsConsumer fs = createConsumer(subDir, "testConsume");
        File f = createFile(TempFileUtils.createTrackedFile(getName(), "", fs), PANGRAM_1);
        ProcessedItem item = new ProcessedItem(f.getAbsolutePath(), 0, f.length());
        ProcessedItemCache cache = new InlineItemCache();
        fs.setPoller(new Never());
        fs.setProcessedItemCache(cache);

        StandaloneConsumer consumer = new StandaloneConsumer(fs);
        try {
            start(consumer);
            cache.update(item);
            ProcessedItem changed = new ProcessedItem(f.getAbsolutePath(), f.lastModified(), f.length());
            assertTrue(fs.hasChanged(changed));
            cache.update(changed);
            assertFalse(fs.hasChanged(changed));
        } finally {
            stop(consumer);
        }
    }

    public void testHasChanged_Size() throws Exception {
        String subDir = new GuidGenerator().safeUUID();
        NonDeletingFsConsumer fs = createConsumer(subDir, "testConsume");
        File f = createFile(TempFileUtils.createTrackedFile(getName(), "", fs), PANGRAM_1);
        ProcessedItem item = new ProcessedItem(f.getAbsolutePath(), f.lastModified(), 0);
        ProcessedItemCache cache = new InlineItemCache();
        fs.setPoller(new Never());
        fs.setProcessedItemCache(cache);

        StandaloneConsumer consumer = new StandaloneConsumer(fs);
        try {
            start(consumer);
            cache.update(item);
            ProcessedItem changed = new ProcessedItem(f.getAbsolutePath(), f.lastModified(), f.length());
            assertTrue(fs.hasChanged(changed));
            cache.update(changed);
            assertFalse(fs.hasChanged(changed));
        } finally {
            stop(consumer);
        }
    }

    public void testRedmine481_SubDirInConsumeDirectory() throws Exception {
        String consumeDir = new GuidGenerator().safeUUID();
        File parentDir = FsHelper
                .createFileReference(FsHelper.createUrlFromString(PROPERTIES.getProperty(BASE_KEY), true));
        File subDirectory = FsHelper.createFileReference(FsHelper.createUrlFromString(
                PROPERTIES.getProperty(BASE_KEY) + "/" + consumeDir + "/" + new GuidGenerator().safeUUID(), true));
        subDirectory.mkdirs();
        NonDeletingFsConsumer fs = createConsumer(consumeDir, "testRedmine481_SubDirInConsumeDirectory");
        fs.setReacquireLockBetweenMessages(true);
        AtomicBoolean pollFired = new AtomicBoolean(false);
        fs.setPoller(
                new FixedIntervalPoller(new TimeInterval(300L, TimeUnit.MILLISECONDS)).withPollerCallback(e -> {
                    pollFired.set(true);
                }));
        MockMessageListener stub = new MockMessageListener(0);
        StandaloneConsumer sc = new StandaloneConsumer(fs);
        sc.registerAdaptrisMessageListener(stub);
        try {
            start(sc);
            waitForPollCallback(pollFired);
            assertEquals(true, subDirectory.exists());
            assertEquals(true, subDirectory.isDirectory());
        } finally {
            stop(sc);
            FileUtils.deleteQuietly(new File(parentDir, consumeDir));
        }
    }

    @Override
    protected void assertMessages(List<AdaptrisMessage> list, int count, File[] remaining) {
        assertEquals("All files produced", count, list.size());
        assertEquals("All files left in dir", count, remaining.length);
        for (AdaptrisMessage m : list) {
            assertTrue(m.containsKey(CoreConstants.ORIGINAL_NAME_KEY));
            assertTrue(m.containsKey(CoreConstants.FILE_LAST_MODIFIED_KEY));
        }

    }

    protected NonDeletingFsConsumer createConsumer(String subDir, String threadname) {
        String destinationName = subDir == null ? PROPERTIES.getProperty(BASE_KEY)
                : PROPERTIES.getProperty(BASE_KEY) + "/" + subDir;
        ConfiguredConsumeDestination dest = threadname != null
                ? new ConfiguredConsumeDestination(destinationName, null, threadname)
                : new ConfiguredConsumeDestination(destinationName);

        NonDeletingFsConsumer fs = new NonDeletingFsConsumer(dest);
        fs.setReacquireLockBetweenMessages(true);
        fs.setCreateDirs(true);
        return fs;
    }

    @Override
    protected NonDeletingFsConsumer createConsumer(String subDir) {
        return createConsumer(subDir, null);
    }

    @Override
    protected NonDeletingFsConsumer createConsumer() {
        return new NonDeletingFsConsumer();
    }

    private void touch(List<File> files) throws IOException {
        for (File f : files) {
            f.setLastModified(System.currentTimeMillis());
        }
    }

    private void resize(List<File> files) throws IOException {
        for (File f : files) {
            long currentSize = f.length();
            RandomAccessFile raf = new RandomAccessFile(f, "rwd");
            try {
                raf.setLength(currentSize + 1);
            } finally {
                raf.close();
            }
        }
    }

    private List<File> createFiles(File baseDir, String ext, int count) throws IOException {
        baseDir.mkdirs();
        List<File> result = new ArrayList<File>();
        for (int i = 0; i < count; i++) {
            result.add(File.createTempFile("FSC", ext, baseDir));
        }
        return result;
    }

    private File createFile(File file, String content) throws IOException {
        try (PrintStream out = new PrintStream(new FileOutputStream(file), true)) {
            out.print(PANGRAM_1);
        }
        return file;
    }

    private class Never extends PollerImp {

        @Override
        public void init() throws CoreException {
        }

        @Override
        public void start() throws CoreException {
        }

        @Override
        public void stop() {
        }

        @Override
        public void close() {
        }

        @Override
        public void prepare() throws CoreException {
        }

    }
}