org.apache.flume.client.avro.TestReliableSpoolingFileEventReader.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.flume.client.avro.TestReliableSpoolingFileEventReader.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.flume.client.avro;

import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.SystemUtils;
import org.apache.flume.Event;
import org.apache.flume.client.avro.ReliableSpoolingFileEventReader.DeletePolicy;
import org.apache.flume.source.SpoolDirectorySourceConfigurationConstants;
import org.apache.flume.source.SpoolDirectorySourceConfigurationConstants.ConsumeOrder;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.*;

public class TestReliableSpoolingFileEventReader {

    private static final Logger logger = LoggerFactory.getLogger(TestReliableSpoolingFileEventReader.class);

    private static final File WORK_DIR = new File(
            "target/test/work/" + TestReliableSpoolingFileEventReader.class.getSimpleName());

    @Before
    public void setup() throws IOException, InterruptedException {
        if (!WORK_DIR.isDirectory()) {
            Files.createParentDirs(new File(WORK_DIR, "dummy"));
        }

        // write out a few files
        for (int i = 0; i < 4; i++) {
            File fileName = new File(WORK_DIR, "file" + i);
            StringBuilder sb = new StringBuilder();

            // write as many lines as the index of the file
            for (int j = 0; j < i; j++) {
                sb.append("file" + i + "line" + j + "\n");
            }
            Files.write(sb.toString(), fileName, Charsets.UTF_8);
        }
        Thread.sleep(1500L); // make sure timestamp is later
        Files.write("\n", new File(WORK_DIR, "emptylineFile"), Charsets.UTF_8);
    }

    @After
    public void tearDown() {
        deleteDir(WORK_DIR);
    }

    private void deleteDir(File dir) {
        // delete all the files & dirs we created
        File[] files = dir.listFiles();
        for (File f : files) {
            if (f.isDirectory()) {
                File[] subDirFiles = f.listFiles();
                for (File sdf : subDirFiles) {
                    if (!sdf.delete()) {
                        logger.warn("Cannot delete file {}", sdf.getAbsolutePath());
                    }
                }
                if (!f.delete()) {
                    logger.warn("Cannot delete directory {}", f.getAbsolutePath());
                }
            } else {
                if (!f.delete()) {
                    logger.warn("Cannot delete file {}", f.getAbsolutePath());
                }
            }
        }
        if (!dir.delete()) {
            logger.warn("Cannot delete work directory {}", dir.getAbsolutePath());
        }
    }

    @Test
    public void testIgnorePattern() throws IOException {
        ReliableEventReader reader = new ReliableSpoolingFileEventReader.Builder().spoolDirectory(WORK_DIR)
                .ignorePattern("^file2$").deletePolicy(DeletePolicy.IMMEDIATE.toString()).build();

        List<File> before = listFiles(WORK_DIR);
        Assert.assertEquals("Expected 5, not: " + before, 5, before.size());

        List<Event> events;
        do {
            events = reader.readEvents(10);
            reader.commit();
        } while (!events.isEmpty());

        List<File> after = listFiles(WORK_DIR);
        Assert.assertEquals("Expected 1, not: " + after, 1, after.size());
        Assert.assertEquals("file2", after.get(0).getName());
        List<File> trackerFiles = listFiles(
                new File(WORK_DIR, SpoolDirectorySourceConfigurationConstants.DEFAULT_TRACKER_DIR));
        Assert.assertEquals("Expected 0, not: " + trackerFiles, 0, trackerFiles.size());
    }

    @Test
    public void testRepeatedCallsWithCommitAlways() throws IOException {
        ReliableEventReader reader = new ReliableSpoolingFileEventReader.Builder().spoolDirectory(WORK_DIR).build();

        final int expectedLines = 0 + 1 + 2 + 3 + 1;
        int seenLines = 0;
        for (int i = 0; i < 10; i++) {
            List<Event> events = reader.readEvents(10);
            seenLines += events.size();
            reader.commit();
        }

        Assert.assertEquals(expectedLines, seenLines);
    }

    @Test
    public void testRepeatedCallsWithCommitOnSuccess() throws IOException {
        String trackerDirPath = SpoolDirectorySourceConfigurationConstants.DEFAULT_TRACKER_DIR;
        File trackerDir = new File(WORK_DIR, trackerDirPath);

        ReliableEventReader reader = new ReliableSpoolingFileEventReader.Builder().spoolDirectory(WORK_DIR)
                .trackerDirPath(trackerDirPath).build();

        final int expectedLines = 0 + 1 + 2 + 3 + 1;
        int seenLines = 0;
        for (int i = 0; i < 10; i++) {
            List<Event> events = reader.readEvents(10);
            int numEvents = events.size();
            if (numEvents > 0) {
                seenLines += numEvents;
                reader.commit();

                // ensure that there are files in the trackerDir
                File[] files = trackerDir.listFiles();
                Assert.assertNotNull(files);
                Assert.assertTrue("Expected tracker files in tracker dir " + trackerDir.getAbsolutePath(),
                        files.length > 0);
            }
        }

        Assert.assertEquals(expectedLines, seenLines);
    }

    @Test
    public void testFileDeletion() throws IOException {
        ReliableEventReader reader = new ReliableSpoolingFileEventReader.Builder().spoolDirectory(WORK_DIR)
                .deletePolicy(DeletePolicy.IMMEDIATE.name()).build();

        List<File> before = listFiles(WORK_DIR);
        Assert.assertEquals("Expected 5, not: " + before, 5, before.size());

        List<Event> events;
        do {
            events = reader.readEvents(10);
            reader.commit();
        } while (!events.isEmpty());

        List<File> after = listFiles(WORK_DIR);
        Assert.assertEquals("Expected 0, not: " + after, 0, after.size());
        List<File> trackerFiles = listFiles(
                new File(WORK_DIR, SpoolDirectorySourceConfigurationConstants.DEFAULT_TRACKER_DIR));
        Assert.assertEquals("Expected 0, not: " + trackerFiles, 0, trackerFiles.size());
    }

    @Test(expected = NullPointerException.class)
    public void testNullConsumeOrder() throws IOException {
        new ReliableSpoolingFileEventReader.Builder().spoolDirectory(WORK_DIR).consumeOrder(null).build();
    }

    @Test
    public void testConsumeFileRandomly() throws IOException {
        ReliableEventReader reader = new ReliableSpoolingFileEventReader.Builder().spoolDirectory(WORK_DIR)
                .consumeOrder(ConsumeOrder.RANDOM).build();
        File fileName = new File(WORK_DIR, "new-file");
        FileUtils.write(fileName, "New file created in the end. Shoud be read randomly.\n");
        Set<String> actual = Sets.newHashSet();
        readEventsForFilesInDir(WORK_DIR, reader, actual);
        Set<String> expected = Sets.newHashSet();
        createExpectedFromFilesInSetup(expected);
        expected.add("");
        expected.add("New file created in the end. Shoud be read randomly.");
        Assert.assertEquals(expected, actual);
    }

    @Test
    public void testConsumeFileRandomlyNewFile() throws Exception {
        // Atomic moves are not supported in Windows.
        if (SystemUtils.IS_OS_WINDOWS) {
            return;
        }
        final ReliableEventReader reader = new ReliableSpoolingFileEventReader.Builder().spoolDirectory(WORK_DIR)
                .consumeOrder(ConsumeOrder.RANDOM).build();
        File fileName = new File(WORK_DIR, "new-file");
        FileUtils.write(fileName, "New file created in the end. Shoud be read randomly.\n");
        Set<String> expected = Sets.newHashSet();
        int totalFiles = WORK_DIR.listFiles().length;
        final Set<String> actual = Sets.newHashSet();
        ExecutorService executor = Executors.newSingleThreadExecutor();
        final Semaphore semaphore1 = new Semaphore(0);
        final Semaphore semaphore2 = new Semaphore(0);
        Future<Void> wait = executor.submit(new Callable<Void>() {
            @Override
            public Void call() throws Exception {
                readEventsForFilesInDir(WORK_DIR, reader, actual, semaphore1, semaphore2);
                return null;
            }
        });
        semaphore1.acquire();
        File finalFile = new File(WORK_DIR, "t-file");
        FileUtils.write(finalFile, "Last file");
        semaphore2.release();
        wait.get();
        int listFilesCount = ((ReliableSpoolingFileEventReader) reader).getListFilesCount();
        finalFile.delete();
        createExpectedFromFilesInSetup(expected);
        expected.add("");
        expected.add("New file created in the end. Shoud be read randomly.");
        expected.add("Last file");
        Assert.assertTrue(listFilesCount < (totalFiles + 2));
        Assert.assertEquals(expected, actual);
    }

    @Test
    public void testConsumeFileOldest() throws IOException, InterruptedException {
        ReliableEventReader reader = new ReliableSpoolingFileEventReader.Builder().spoolDirectory(WORK_DIR)
                .consumeOrder(ConsumeOrder.OLDEST).build();
        File file1 = new File(WORK_DIR, "new-file1");
        File file2 = new File(WORK_DIR, "new-file2");
        File file3 = new File(WORK_DIR, "new-file3");
        Thread.sleep(1000L);
        FileUtils.write(file2, "New file2 created.\n");
        Thread.sleep(1000L);
        FileUtils.write(file1, "New file1 created.\n");
        Thread.sleep(1000L);
        FileUtils.write(file3, "New file3 created.\n");
        // order of age oldest to youngest (file2, file1, file3)
        List<String> actual = Lists.newLinkedList();
        readEventsForFilesInDir(WORK_DIR, reader, actual);
        List<String> expected = Lists.newLinkedList();
        createExpectedFromFilesInSetup(expected);
        expected.add(""); // Empty file was added in the last in setup.
        expected.add("New file2 created.");
        expected.add("New file1 created.");
        expected.add("New file3 created.");
        Assert.assertEquals(expected, actual);
    }

    @Test
    public void testConsumeFileYoungest() throws IOException, InterruptedException {
        ReliableEventReader reader = new ReliableSpoolingFileEventReader.Builder().spoolDirectory(WORK_DIR)
                .consumeOrder(ConsumeOrder.YOUNGEST).build();
        File file1 = new File(WORK_DIR, "new-file1");
        File file2 = new File(WORK_DIR, "new-file2");
        File file3 = new File(WORK_DIR, "new-file3");
        Thread.sleep(1000L);
        FileUtils.write(file2, "New file2 created.\n");
        Thread.sleep(1000L);
        FileUtils.write(file3, "New file3 created.\n");
        Thread.sleep(1000L);
        FileUtils.write(file1, "New file1 created.\n");
        // order of age youngest to oldest (file2, file3, file1)
        List<String> actual = Lists.newLinkedList();
        readEventsForFilesInDir(WORK_DIR, reader, actual);
        List<String> expected = Lists.newLinkedList();
        createExpectedFromFilesInSetup(expected);
        Collections.sort(expected);
        // Empty Line file was added in the last in Setup.
        expected.add(0, "");
        expected.add(0, "New file2 created.");
        expected.add(0, "New file3 created.");
        expected.add(0, "New file1 created.");

        Assert.assertEquals(expected, actual);
    }

    @Test
    public void testConsumeFileOldestWithLexicographicalComparision() throws IOException, InterruptedException {
        ReliableEventReader reader = new ReliableSpoolingFileEventReader.Builder().spoolDirectory(WORK_DIR)
                .consumeOrder(ConsumeOrder.OLDEST).build();
        File file1 = new File(WORK_DIR, "new-file1");
        File file2 = new File(WORK_DIR, "new-file2");
        File file3 = new File(WORK_DIR, "new-file3");
        Thread.sleep(1000L);
        FileUtils.write(file3, "New file3 created.\n");
        FileUtils.write(file2, "New file2 created.\n");
        FileUtils.write(file1, "New file1 created.\n");
        file1.setLastModified(file3.lastModified());
        file1.setLastModified(file2.lastModified());
        // file ages are same now they need to be ordered
        // lexicographically (file1, file2, file3).
        List<String> actual = Lists.newLinkedList();
        readEventsForFilesInDir(WORK_DIR, reader, actual);
        List<String> expected = Lists.newLinkedList();
        createExpectedFromFilesInSetup(expected);
        expected.add(""); // Empty file was added in the last in setup.
        expected.add("New file1 created.");
        expected.add("New file2 created.");
        expected.add("New file3 created.");
        Assert.assertEquals(expected, actual);
    }

    @Test
    public void testConsumeFileYoungestWithLexicographicalComparision() throws IOException, InterruptedException {
        ReliableEventReader reader = new ReliableSpoolingFileEventReader.Builder().spoolDirectory(WORK_DIR)
                .consumeOrder(ConsumeOrder.YOUNGEST).build();
        File file1 = new File(WORK_DIR, "new-file1");
        File file2 = new File(WORK_DIR, "new-file2");
        File file3 = new File(WORK_DIR, "new-file3");
        Thread.sleep(1000L);
        FileUtils.write(file1, "New file1 created.\n");
        FileUtils.write(file2, "New file2 created.\n");
        FileUtils.write(file3, "New file3 created.\n");
        file1.setLastModified(file3.lastModified());
        file1.setLastModified(file2.lastModified());
        // file ages are same now they need to be ordered
        // lexicographically (file1, file2, file3).
        List<String> actual = Lists.newLinkedList();
        readEventsForFilesInDir(WORK_DIR, reader, actual);
        List<String> expected = Lists.newLinkedList();
        createExpectedFromFilesInSetup(expected);
        expected.add(0, ""); // Empty file was added in the last in setup.
        expected.add(0, "New file3 created.");
        expected.add(0, "New file2 created.");
        expected.add(0, "New file1 created.");
        Assert.assertEquals(expected, actual);
    }

    @Test
    public void testLargeNumberOfFilesOLDEST() throws IOException {
        templateTestForLargeNumberOfFiles(ConsumeOrder.OLDEST, null, 1000);
    }

    @Test
    public void testLargeNumberOfFilesYOUNGEST() throws IOException {
        templateTestForLargeNumberOfFiles(ConsumeOrder.YOUNGEST, new Comparator<Long>() {

            @Override
            public int compare(Long o1, Long o2) {
                return o2.compareTo(o1);
            }
        }, 1000);
    }

    @Test
    public void testLargeNumberOfFilesRANDOM() throws IOException {
        templateTestForLargeNumberOfFiles(ConsumeOrder.RANDOM, null, 1000);
    }

    @Test
    public void testZeroByteTrackerFile() throws IOException {
        String trackerDirPath = SpoolDirectorySourceConfigurationConstants.DEFAULT_TRACKER_DIR;
        File trackerDir = new File(WORK_DIR, trackerDirPath);
        if (!trackerDir.exists()) {
            trackerDir.mkdir();
        }
        File trackerFile = new File(trackerDir, ReliableSpoolingFileEventReader.metaFileName);
        if (trackerFile.exists()) {
            trackerFile.delete();
        }
        trackerFile.createNewFile();

        ReliableEventReader reader = new ReliableSpoolingFileEventReader.Builder().spoolDirectory(WORK_DIR)
                .trackerDirPath(trackerDirPath).build();
        final int expectedLines = 1;
        int seenLines = 0;
        List<Event> events = reader.readEvents(10);
        int numEvents = events.size();
        if (numEvents > 0) {
            seenLines += numEvents;
            reader.commit();
        }
        // This line will fail, if the zero-byte tracker file has not been handled
        Assert.assertEquals(expectedLines, seenLines);
    }

    private void templateTestForLargeNumberOfFiles(ConsumeOrder order, Comparator<Long> comparator, int N)
            throws IOException {
        File dir = null;
        try {
            dir = new File("target/test/work/" + this.getClass().getSimpleName() + "_large");
            Files.createParentDirs(new File(dir, "dummy"));
            ReliableEventReader reader = new ReliableSpoolingFileEventReader.Builder().spoolDirectory(dir)
                    .consumeOrder(order).build();
            Map<Long, List<String>> expected;
            if (comparator == null) {
                expected = new TreeMap<Long, List<String>>();
            } else {
                expected = new TreeMap<Long, List<String>>(comparator);
            }
            for (int i = 0; i < N; i++) {
                File f = new File(dir, "file-" + i);
                String data = "file-" + i;
                Files.write(data, f, Charsets.UTF_8);
                if (expected.containsKey(f.lastModified())) {
                    expected.get(f.lastModified()).add(data);
                } else {
                    expected.put(f.lastModified(), Lists.newArrayList(data));
                }
            }
            Collection<String> expectedList;
            if (order == ConsumeOrder.RANDOM) {
                expectedList = Sets.newHashSet();
            } else {
                expectedList = Lists.newArrayList();
            }
            for (Entry<Long, List<String>> entry : expected.entrySet()) {
                Collections.sort(entry.getValue());
                expectedList.addAll(entry.getValue());
            }
            for (int i = 0; i < N; i++) {
                List<Event> events;
                events = reader.readEvents(10);
                for (Event e : events) {
                    if (order == ConsumeOrder.RANDOM) {
                        Assert.assertTrue(expectedList.remove(new String(e.getBody())));
                    } else {
                        Assert.assertEquals(((ArrayList<String>) expectedList).get(0), new String(e.getBody()));
                        ((ArrayList<String>) expectedList).remove(0);
                    }
                }
                reader.commit();
            }
        } finally {
            deleteDir(dir);
        }
    }

    private void readEventsForFilesInDir(File dir, ReliableEventReader reader, Collection<String> actual)
            throws IOException {
        readEventsForFilesInDir(dir, reader, actual, null, null);
    }

    /* Read events, one for each file in the given directory. */
    private void readEventsForFilesInDir(File dir, ReliableEventReader reader, Collection<String> actual,
            Semaphore semaphore1, Semaphore semaphore2) throws IOException {
        List<Event> events;
        boolean executed = false;
        for (int i = 0; i < listFiles(dir).size(); i++) {
            events = reader.readEvents(10);
            for (Event e : events) {
                actual.add(new String(e.getBody()));
            }
            reader.commit();
            try {
                if (!executed) {
                    executed = true;
                    if (semaphore1 != null) {
                        semaphore1.release();
                    }
                    if (semaphore2 != null) {
                        semaphore2.acquire();
                    }
                }
            } catch (Exception ex) {
                throw new IOException(ex);
            }
        }
    }

    /* Create expected results out of the files created in the setup method. */
    private void createExpectedFromFilesInSetup(Collection<String> expected) {
        for (int i = 0; i < 4; i++) {
            for (int j = 0; j < i; j++) {
                expected.add("file" + i + "line" + j);
            }
        }
    }

    private static List<File> listFiles(File dir) {
        List<File> files = Lists.newArrayList(dir.listFiles(new FileFilter() {
            @Override
            public boolean accept(File pathname) {
                return !pathname.isDirectory();
            }
        }));
        return files;
    }
}