com.asakusafw.operation.tools.hadoop.fs.CleanTest.java Source code

Java tutorial

Introduction

Here is the source code for com.asakusafw.operation.tools.hadoop.fs.CleanTest.java

Source

/**
 * Copyright 2011-2017 Asakusa Framework Team.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.asakusafw.operation.tools.hadoop.fs;

import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;

import org.apache.commons.lang.SystemUtils;
import org.apache.hadoop.conf.Configuration;
import org.junit.Assume;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

/**
 * Test for {@link Clean}.
 */
public class CleanTest {

    private static final String OPT_KEEP = "-keep-days";

    /**
     * Temporary folder.
     */
    @Rule
    public final TemporaryFolder folder = new TemporaryFolder();

    /**
     * simple test case.
     * @throws Exception if failed to execute
     */
    @Test
    public void simple() throws Exception {
        File file = touch("file", 50);

        Clean c = createService(100);
        int exit = c.run(args(1, path("file")));

        assertThat(exit, is(0));
        assertThat(file.toString(), file.exists(), is(false));
    }

    /**
     * simple test case for keep.
     * @throws Exception if failed to execute
     */
    @Test
    public void keep() throws Exception {
        File file = touch("file", 50);

        Clean c = createService(100);
        int exit = c.run(args(51, path("file")));

        assertThat(exit, is(0));
        assertThat(file.toString(), file.exists(), is(true));
    }

    /**
     * simple test case for folder.
     * @throws Exception if failed to execute
     */
    @Test
    public void folder() throws Exception {
        File file = touch("folder/file", 50);

        Clean c = createService(100);
        int exit = c.run(args(1, path("folder")));

        assertThat(exit, is(0));
        assertThat(file.toString(), file.exists(), is(true));
    }

    /**
     * simple test case for recursive delete.
     * @throws Exception if failed to execute
     */
    @Test
    public void recursive() throws Exception {
        File file = touch("folder/file", 50);

        Clean c = createService(100);
        int exit = c.run(args(1, "-r", path("folder")));

        assertThat(exit, is(0));
        assertThat(file.toString(), file.exists(), is(false));
    }

    /**
     * simple test case for dryrun.
     * @throws Exception if failed to execute
     */
    @Test
    public void dry_run() throws Exception {
        File file = touch("file", 50);

        Clean c = createService(100);
        int exit = c.run(args(1, "-s", path("file")));

        assertThat(exit, is(0));
        assertThat(file.toString(), file.exists(), is(true));
    }

    /**
     * test case using wildcard.
     * @throws Exception if failed to execute
     */
    @Test
    public void wildcard() throws Exception {
        File f1 = touch("file1.txt", 50);
        File f2 = touch("file2.csv", 50);
        File f3 = touch("file3.txt", 50);
        File f4 = touch("file4.csv", 50);
        File f5 = touch("file5.txt", 50);

        Clean c = createService(100);
        int exit = c.run(args(1, path("*.csv")));

        assertThat(exit, is(0));
        assertThat(f1.toString(), f1.exists(), is(true));
        assertThat(f2.toString(), f2.exists(), is(false));
        assertThat(f3.toString(), f3.exists(), is(true));
        assertThat(f4.toString(), f4.exists(), is(false));
        assertThat(f5.toString(), f5.exists(), is(true));
    }

    /**
     * deletes multiple files.
     * @throws Exception if failed to execute
     */
    @Test
    public void multiple() throws Exception {
        File f1 = touch("file1", 50);
        File f2 = touch("file2", 50);
        File f3 = touch("file3", 50);

        Clean c = createService(100);
        int exit = c.run(args(1, path("file1"), path("file3")));

        assertThat(exit, is(0));
        assertThat(f1.toString(), f1.exists(), is(false));
        assertThat(f2.toString(), f2.exists(), is(true));
        assertThat(f3.toString(), f3.exists(), is(false));
    }

    /**
     * test case using deep wildcard.
     * @throws Exception if failed to execute
     */
    @Test
    public void deep_wildcard() throws Exception {
        File f1 = touch("a/file1.txt", 50);
        File f2 = touch("a/file2.csv", 50);
        File f3 = touch("a/file3.txt", 50);
        File f4 = touch("b/file4.csv", 50);
        File f5 = touch("b/file5.txt", 50);

        Clean c = createService(100);
        int exit = c.run(args(1, path("*/*.csv")));

        assertThat(exit, is(0));
        assertThat(f1.toString(), f1.exists(), is(true));
        assertThat(f2.toString(), f2.exists(), is(false));
        assertThat(f3.toString(), f3.exists(), is(true));
        assertThat(f4.toString(), f4.exists(), is(false));
        assertThat(f5.toString(), f5.exists(), is(true));
    }

    /**
     * test case using deep wildcard.
     * @throws Exception if failed to execute
     */
    @Test
    public void skip_folder() throws Exception {
        File f1 = touch("a/file1", 50);
        File f2 = touch("a/file2", 100);
        File f3 = touch("a/file3", 50);
        File f4 = touch("b/file4", 50);
        File f5 = touch("b/file5", 50);
        touch("a", 50);
        touch("b", 50);

        Clean c = createService(100);
        int exit = c.run(args(30, "-r", path("*")));

        assertThat(exit, is(0));
        assertThat(f1.toString(), f1.exists(), is(false));
        assertThat(f2.toString(), f2.exists(), is(true));
        assertThat(f3.toString(), f3.exists(), is(false));
        assertThat(f4.toString(), f4.exists(), is(false));
        assertThat(f5.toString(), f5.exists(), is(false));

        assertThat("a", file("a").exists(), is(true));
        assertThat("a", file("b").exists(), is(false));
    }

    /**
     * test case using deep wildcard.
     * @throws Exception if failed to execute
     */
    @Test
    public void keep_folder() throws Exception {
        File f1 = touch("a/file1", 50);
        File f2 = touch("a/file2", 50);
        File f3 = touch("a/file3", 50);
        File f4 = touch("b/file4", 50);
        File f5 = touch("b/file5", 50);
        touch("a", 100);
        touch("b", 50);

        Clean c = createService(100);
        int exit = c.run(args(30, "-r", path("*")));

        assertThat(exit, is(0));
        assertThat(f1.toString(), f1.exists(), is(false));
        assertThat(f2.toString(), f2.exists(), is(false));
        assertThat(f3.toString(), f3.exists(), is(false));
        assertThat(f4.toString(), f4.exists(), is(false));
        assertThat(f5.toString(), f5.exists(), is(false));

        assertThat("a", file("a").exists(), is(true));
        assertThat("a", file("b").exists(), is(false));
    }

    /**
     * minus prefixed file.
     * @throws Exception if failed to execute
     */
    @Test
    public void minus_file() throws Exception {
        File file = touch("-r", 50);

        Clean c = createService(100);
        int exit = c.run(args(1, "--", path("-r")));

        assertThat(exit, is(0));
        assertThat(file.toString(), file.exists(), is(false));
    }

    /**
     * missing path.
     * @throws Exception if failed to execute
     */
    @Test
    public void missing_path() throws Exception {
        Clean c = createService(100);
        int exit = c.run(args(1, path("file")));

        assertThat(exit, is(not(0)));
    }

    /**
     * missing wildcard.
     * @throws Exception if failed to execute
     */
    @Test
    public void missing_wildcard() throws Exception {
        Clean c = createService(100);
        int exit = c.run(args(1, path("dir/*")));

        assertThat(exit, is(not(0)));
    }

    /**
     * no keep options.
     * @throws Exception if failed to execute
     */
    @Test
    public void missing_keep() throws Exception {
        touch("file", 50);

        Clean c = createService(100);
        int exit = c.run(new String[] { path("file") });

        assertThat(exit, is(not(0)));
    }

    /**
     * invalid keep options.
     * @throws Exception if failed to execute
     */
    @Test
    public void invalid_keep() throws Exception {
        touch("file", 50);

        Clean c = createService(100);
        int exit = c.run(new String[] { OPT_KEEP, "INVALID", path("file") });

        assertThat(exit, is(not(0)));
    }

    /**
     * unknown options.
     * @throws Exception if failed to execute
     */
    @Test
    public void unknown_opts() throws Exception {
        touch("file", 50);

        Clean c = createService(100);
        int exit = c.run(args(0, "-r", "-unknown", path("*")));

        assertThat(exit, is(not(0)));
    }

    /**
     * no targets.
     * @throws Exception if failed to execute
     */
    @Test
    public void empty_path() throws Exception {
        Clean c = createService(100);
        int exit = c.run(args(1));

        assertThat(exit, is(not(0)));
    }

    /**
     * invalid file system path.
     * @throws Exception if failed to execute
     */
    @Test
    public void malformed_path() throws Exception {
        Clean c = createService(100);
        int exit = c.run(args(1, ":MALFORMED:"));

        assertThat(exit, is(not(0)));
    }

    /**
     * invalid file system.
     * @throws Exception if failed to execute
     */
    @Test
    public void invalid_filesystem() throws Exception {
        Clean c = createService(100);
        int exit = c.run(args(1, "INVALID:///"));

        assertThat(exit, is(not(0)));
    }

    /**
     * test case for inaccessible folder.
     * @throws Exception if failed to execute
     */
    @Test
    public void inaccessible_folder() throws Exception {
        assumeAccessRestrictionAvailable();

        File f1 = touch("a/file1", 50);
        File f2 = touch("a/RESTRICTED/file", 50);
        File f3 = touch("a/file3", 50);

        file("a/RESTRICTED").setExecutable(false, false);
        int exit;
        try {
            Clean c = createService(100);
            exit = c.run(args(0, "-r", path("*")));
        } finally {
            file("a/RESTRICTED").setExecutable(true, false);
        }
        assertThat(exit, is(not(0)));
        assertThat(f1.toString(), f1.exists(), is(false));
        assertThat(f2.toString(), f2.exists(), is(true));
        assertThat(f3.toString(), f3.exists(), is(false));

        assertThat("a", file("a").exists(), is(true));
    }

    /**
     * test case for readonly file.
     * @throws Exception if failed to execute
     */
    @Test
    public void readonly_folder() throws Exception {
        assumeAccessRestrictionAvailable();

        File f1 = touch("a/file1", 50);
        File f2 = touch("a/RESTRICTED/file", 50);
        File f3 = touch("a/file3", 50);

        file("a/RESTRICTED").setWritable(false, false);
        int exit;
        try {
            Clean c = createService(100);
            exit = c.run(args(0, "-r", path("*")));
        } finally {
            file("a/RESTRICTED").setWritable(true, false);
        }
        assertThat(exit, is(not(0)));
        assertThat(f1.toString(), f1.exists(), is(false));
        assertThat(f2.toString(), f2.exists(), is(true));
        assertThat(f3.toString(), f3.exists(), is(false));

        assertThat("a", file("a").exists(), is(true));
    }

    /**
     * test case for symlink to file.
     * @throws Exception if failed to execute
     */
    @Test
    public void symlink_file() throws Exception {
        File f1 = touch("a/file1", 50);
        File f2 = touch("b/file2", 50);
        File f3 = link("b/link", f1, 50);

        Clean c = createService(100);
        int exit = c.run(args(0, "-r", path("b")));

        assertThat(exit, is(not(0)));
        assertThat(f1.toString(), f1.exists(), is(true));
        assertThat(f2.toString(), f2.exists(), is(false));
        assertThat(f3.toString(), f3.exists(), is(true));
    }

    /**
     * test case for symlink to dir.
     * @throws Exception if failed to execute
     */
    @Test
    public void symlink_dir() throws Exception {
        File f1 = touch("a/file1", 50);
        File f2 = touch("b/file2", 50);
        File f3 = link("b/link", file("a"), 50);

        Clean c = createService(100);
        int exit = c.run(args(0, "-r", path("b")));

        assertThat(exit, is(not(0)));
        assertThat(f1.toString(), f1.exists(), is(true));
        assertThat(f2.toString(), f2.exists(), is(false));
        assertThat(f3.toString(), f3.exists(), is(true));
    }

    /**
     * test case for symlink to file.
     * @throws Exception if failed to execute
     */
    @Test
    public void symlink_on_same_name() throws Exception {
        File f1 = touch("a/file", 50);
        File f2 = touch("b/file2", 50);
        File f3 = link("b/file", f1, 50);

        Clean c = createService(100);
        int exit = c.run(args(0, "-r", path("b")));

        assertThat(exit, is(not(0)));
        assertThat(f1.toString(), f1.exists(), is(true));
        assertThat(f2.toString(), f2.exists(), is(false));
        assertThat(f3.toString(), f3.exists(), is(true));
    }

    /**
     * test case for symlink to file.
     * @throws Exception if failed to execute
     */
    @Test
    public void symlink_on_same_dir() throws Exception {
        File f1 = touch("a/file1", 50);
        File f2 = touch("b/file2", 50);
        File f3 = link("b/file3.lnk", f2, 50);

        Clean c = createService(100);
        int exit = c.run(args(0, "-r", path("b/*.lnk")));

        assertThat(exit, is(not(0)));
        assertThat(f1.toString(), f1.exists(), is(true));
        assertThat(f2.toString(), f2.exists(), is(true));
        assertThat(f3.toString(), f3.exists(), is(true));
    }

    /**
     * test case for symlink to file.
     * @throws Exception if failed to execute
     */
    @Test
    public void symlink_lost() throws Exception {
        File f1 = touch("a/file1", 50);
        File f2 = touch("b/file2", 50);
        link("c/link", f1, 50);
        Assume.assumeThat(f1.delete(), is(true));

        Clean c = createService(100);
        c.run(args(0, "-r", path("*")));

        assertThat(f2.toString(), f2.exists(), is(false));
    }

    /**
     * test case for symlink to file.
     * @throws Exception if failed to execute
     */
    @Test
    public void symlink_self() throws Exception {
        File f1 = touch("a/file1", 50);
        File f2 = touch("b/file2", 50);
        File f3 = link("c/link", f1, 50);
        Assume.assumeThat(f1.delete(), is(true));
        f3.renameTo(f1);

        Clean c = createService(100);
        c.run(args(0, "-r", path("*")));

        assertThat(f2.toString(), f2.exists(), is(false));
    }

    private Clean createService(long days) {
        Clean service = new Clean(TimeUnit.DAYS.toMillis(days));
        service.setConf(new Configuration());
        return service;
    }

    private String[] args(int keep, String... args) {
        List<String> list = new ArrayList<>();
        Collections.addAll(list, OPT_KEEP, String.valueOf(keep));
        Collections.addAll(list, args);
        return list.toArray(new String[list.size()]);
    }

    private String path(String path) {
        String uri = folder.getRoot().toURI().toString();
        return uri + "/" + path;
    }

    private File link(String path, File target, int day) throws IOException {
        Assume.assumeFalse("In Windows, tests with symlink are skipped", SystemUtils.IS_OS_WINDOWS);
        File link = file(path);
        link.getParentFile().mkdirs();
        try {
            Process process = new ProcessBuilder()
                    .command("ln", "-s", target.getCanonicalPath(), link.getAbsolutePath())
                    .redirectErrorStream(true).start();
            try {
                int exit = process.waitFor();
                Assume.assumeThat(exit, is(0));
            } finally {
                process.destroy();
            }
        } catch (Exception e) {
            Assume.assumeNoException(e);
        }
        touch(path, day);
        return link;
    }

    private File touch(String path, double day) throws IOException {
        File file = file(path);
        if (file.exists() == false) {
            file.getParentFile().mkdirs();
            try {
                file.createNewFile();
            } catch (IOException e) {
                throw new AssertionError(e);
            }
        }
        File root = folder.getRoot().getCanonicalFile();
        File current = file;
        while (true) {
            current = current.getCanonicalFile();
            if (root.equals(current)) {
                break;
            }
            boolean succeed = current.setLastModified((long) (day * TimeUnit.DAYS.toMillis(1)));
            if (succeed == false) {
                break;
            }
            current = current.getParentFile();
            if (current == null) {
                break;
            }
        }
        return file;
    }

    private File file(String path) {
        File file = new File(folder.getRoot(), path);
        return file;
    }

    private void assumeAccessRestrictionAvailable() throws IOException {
        File f = File.createTempFile("access-restriction-check", ".dummy");
        f.setReadable(false, false);
        try {
            if (f.canRead()) {
                System.err.println("Current context does not support access restriction.");
                Assume.assumeTrue(false);
            }
        } finally {
            f.setReadable(true, true);
            if (f.delete() == false) {
                System.err.printf("Failed to delete a dummy file: %s%n", f);
            }
        }
    }
}