gobblin.util.FileListUtilsTest.java Source code

Introduction

Here is the source code for gobblin.util.FileListUtilsTest.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package gobblin.util;

import java.io.IOException;
import java.util.List;
import java.util.Set;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.testng.Assert;
import org.testng.annotations.Test;

import com.google.common.collect.Sets;

/**
 * Unit tests for the job configuration file monitor in {@link gobblin.util.FileListUtils}.
 */
@Test(groups = { "gobblin.util" })
public class FileListUtilsTest {

    private static final String FILE_UTILS_TEST_DIR = "gobblin-utility/src/test/resources/";
    private static final String TEST_FILE_NAME1 = "test1";
    private static final String TEST_FILE_NAME2 = "test2";

    @Test
    public void testListFilesRecursively() throws IOException {
        FileSystem localFs = FileSystem.getLocal(new Configuration());
        Path baseDir = new Path(FILE_UTILS_TEST_DIR, "fileListTestDir1");
        try {
            if (localFs.exists(baseDir)) {
                localFs.delete(baseDir, true);
            }
            localFs.mkdirs(baseDir);
            localFs.create(new Path(baseDir, TEST_FILE_NAME1));
            Path subDir = new Path(baseDir, "subDir");
            localFs.mkdirs(subDir);
            localFs.create(new Path(subDir, TEST_FILE_NAME2));
            List<FileStatus> testFiles = FileListUtils.listFilesRecursively(localFs, baseDir);

            Assert.assertEquals(2, testFiles.size());

            Set<String> fileNames = Sets.newHashSet();
            for (FileStatus testFileStatus : testFiles) {
                fileNames.add(testFileStatus.getPath().getName());
            }
            Assert.assertTrue(fileNames.contains(TEST_FILE_NAME1) && fileNames.contains(TEST_FILE_NAME2));
        } finally {
            localFs.delete(baseDir, true);
        }
    }

    @Test
    public void testListPathsRecursively() throws IOException {
        FileSystem localFs = FileSystem.getLocal(new Configuration());
        Path baseDir = new Path(FILE_UTILS_TEST_DIR, "fileListTestDir2");
        try {
            if (localFs.exists(baseDir)) {
                localFs.delete(baseDir, true);
            }
            localFs.mkdirs(baseDir);
            localFs.create(new Path(baseDir, TEST_FILE_NAME1));
            Path subDir = new Path(baseDir, "subDir");
            localFs.mkdirs(subDir);
            localFs.create(new Path(subDir, TEST_FILE_NAME2));
            List<FileStatus> testFiles = FileListUtils.listPathsRecursively(localFs, baseDir, new PathFilter() {
                @Override
                public boolean accept(Path path) {
                    return true;
                }
            });
            Assert.assertEquals(4, testFiles.size());

            Set<String> fileNames = Sets.newHashSet();
            for (FileStatus testFileStatus : testFiles) {
                fileNames.add(testFileStatus.getPath().getName());
            }

            Set<String> expectedFileNames = Sets.newHashSet();
            expectedFileNames.add(baseDir.getName());
            expectedFileNames.add(subDir.getName());
            expectedFileNames.add(TEST_FILE_NAME1);
            expectedFileNames.add(TEST_FILE_NAME2);

            Assert.assertEquals(fileNames, expectedFileNames);
        } finally {
            localFs.delete(baseDir, true);
        }
    }

    @Test
    public void testListMostNestedPathRecursively() throws IOException {
        FileSystem localFs = FileSystem.getLocal(new Configuration());
        Path baseDir = new Path(FILE_UTILS_TEST_DIR, "fileListTestDir3");
        String emptyDir1 = "emptyDir1";
        String emptyDir2 = "emptyDir2";
        try {
            if (localFs.exists(baseDir)) {
                localFs.delete(baseDir, true);
            }
            localFs.mkdirs(baseDir);
            localFs.create(new Path(baseDir, TEST_FILE_NAME1));
            localFs.mkdirs(new Path(baseDir, emptyDir1));
            Path subDir = new Path(baseDir, "subDir");
            localFs.mkdirs(subDir);
            localFs.create(new Path(subDir, TEST_FILE_NAME2));
            localFs.mkdirs(new Path(subDir, emptyDir2));

            List<FileStatus> testFiles = FileListUtils.listMostNestedPathRecursively(localFs, baseDir);

            Assert.assertEquals(4, testFiles.size());

            Set<String> fileNames = Sets.newHashSet();
            for (FileStatus testFileStatus : testFiles) {
                fileNames.add(testFileStatus.getPath().getName());
            }

            Set<String> expectedFileNames = Sets.newHashSet();
            expectedFileNames.add(emptyDir1);
            expectedFileNames.add(emptyDir2);
            expectedFileNames.add(TEST_FILE_NAME1);
            expectedFileNames.add(TEST_FILE_NAME2);

            Assert.assertEquals(fileNames, expectedFileNames);
        } finally {
            localFs.delete(baseDir, true);
        }
    }

}