org.elasticsearch.xpack.qa.sql.security.SqlSecurityTestCase.java Source code

Java tutorial

Introduction

Here is the source code for org.elasticsearch.xpack.qa.sql.security.SqlSecurityTestCase.java

Source

/*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License;
 * you may not use this file except in compliance with the Elastic License.
 */
package org.elasticsearch.xpack.qa.sql.security;

import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.lucene.util.SuppressForbidden;
import org.elasticsearch.SpecialPermission;
import org.elasticsearch.action.admin.indices.get.GetIndexAction;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.hamcrest.Matcher;
import org.hamcrest.Matchers;
import org.junit.AfterClass;
import org.junit.Before;

import java.io.BufferedReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.function.Function;
import java.util.regex.Pattern;

import static java.util.Collections.singletonMap;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.hasItems;

public abstract class SqlSecurityTestCase extends ESRestTestCase {
    /**
     * Actions taken by this test.
     * <p>
     * For methods that take {@code user} a {@code null} user means "use the admin".
     */
    protected interface Actions {
        String minimalPermissionsForAllActions();

        void queryWorksAsAdmin() throws Exception;

        /**
         * Assert that running some sql as a user returns the same result as running it as
         * the administrator.
         */
        void expectMatchesAdmin(String adminSql, String user, String userSql) throws Exception;

        /**
         * Same as {@link #expectMatchesAdmin(String, String, String)} but sets the scroll size
         * to 1 and completely scrolls the results.
         */
        void expectScrollMatchesAdmin(String adminSql, String user, String userSql) throws Exception;

        void expectDescribe(Map<String, String> columns, String user) throws Exception;

        void expectShowTables(List<String> tables, String user) throws Exception;

        void expectForbidden(String user, String sql) throws Exception;

        void expectUnknownIndex(String user, String sql) throws Exception;

        void expectUnknownColumn(String user, String sql, String column) throws Exception;

        void checkNoMonitorMain(String user) throws Exception;
    }

    protected static final String SQL_ACTION_NAME = "indices:data/read/sql";
    /**
     * Location of the audit log file. We could technically figure this out by reading the admin
     * APIs but it isn't worth doing because we also have to give ourselves permission to read
     * the file and that must be done by setting a system property and reading it in
     * {@code plugin-security.policy}. So we may as well have gradle set the property.
     */
    private static final Path AUDIT_LOG_FILE = lookupAuditLog();

    @SuppressForbidden(reason = "security doesn't work with mock filesystem")
    private static Path lookupAuditLog() {
        String auditLogFileString = System.getProperty("tests.audit.logfile");
        if (null == auditLogFileString) {
            throw new IllegalStateException("tests.audit.logfile must be set to run this test. It is automatically "
                    + "set by gradle. If you must set it yourself then it should be the absolute path to the audit "
                    + "log file generated by running x-pack with audit logging enabled.");
        }
        return Paths.get(auditLogFileString);
    }

    private static boolean oneTimeSetup = false;
    private static boolean auditFailure = false;

    /**
     * The actions taken by this test.
     */
    private final Actions actions;

    /**
     * How much of the audit log was written before the test started.
     */
    private long auditLogWrittenBeforeTestStart;

    public SqlSecurityTestCase(Actions actions) {
        this.actions = actions;
    }

    /**
     * All tests run as a an administrative user but use
     * <code>es-security-runas-user</code> to become a less privileged user when needed.
     */
    @Override
    protected Settings restClientSettings() {
        return RestSqlIT.securitySettings();
    }

    @Override
    protected boolean preserveIndicesUponCompletion() {
        /* We can't wipe the cluster between tests because that nukes the audit
         * trail index which makes the auditing flaky. Instead we wipe all
         * indices after the entire class is finished. */
        return true;
    }

    @Before
    public void oneTimeSetup() throws Exception {
        if (oneTimeSetup) {
            /* Since we don't wipe the cluster between tests we only need to
             * write the test data once. */
            return;
        }
        Request request = new Request("PUT", "/_bulk");
        request.addParameter("refresh", "true");

        StringBuilder bulk = new StringBuilder();
        bulk.append("{\"index\":{\"_index\": \"test\", \"_type\": \"doc\", \"_id\":\"1\"}\n");
        bulk.append("{\"a\": 1, \"b\": 2, \"c\": 3}\n");
        bulk.append("{\"index\":{\"_index\": \"test\", \"_type\": \"doc\", \"_id\":\"2\"}\n");
        bulk.append("{\"a\": 4, \"b\": 5, \"c\": 6}\n");
        bulk.append("{\"index\":{\"_index\": \"bort\", \"_type\": \"doc\", \"_id\":\"1\"}\n");
        bulk.append("{\"a\": \"test\"}\n");
        request.setJsonEntity(bulk.toString());
        client().performRequest(request);
        oneTimeSetup = true;
    }

    @Before
    public void setInitialAuditLogOffset() {
        SecurityManager sm = System.getSecurityManager();
        if (sm != null) {
            sm.checkPermission(new SpecialPermission());
        }
        AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
            if (false == Files.exists(AUDIT_LOG_FILE)) {
                auditLogWrittenBeforeTestStart = 0;
                return null;
            }
            if (false == Files.isRegularFile(AUDIT_LOG_FILE)) {
                throw new IllegalStateException(
                        "expected tests.audit.logfile [" + AUDIT_LOG_FILE + "]to be a plain file but wasn't");
            }
            try {
                auditLogWrittenBeforeTestStart = Files.size(AUDIT_LOG_FILE);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            return null;
        });
    }

    @AfterClass
    public static void wipeIndicesAfterTests() throws IOException {
        try {
            adminClient().performRequest(new Request("DELETE", "*"));
        } catch (ResponseException e) {
            // 404 here just means we had no indexes
            if (e.getResponse().getStatusLine().getStatusCode() != 404) {
                throw e;
            }
        } finally {
            // Clear the static state so other subclasses can reuse it later
            oneTimeSetup = false;
            auditFailure = false;
        }
    }

    @Override
    protected String getProtocol() {
        return RestSqlIT.SSL_ENABLED ? "https" : "http";
    }

    public void testQueryWorksAsAdmin() throws Exception {
        actions.queryWorksAsAdmin();
        createAuditLogAsserter().expectSqlCompositeAction("test_admin", "test").assertLogs();
    }

    public void testQueryWithFullAccess() throws Exception {
        createUser("full_access", actions.minimalPermissionsForAllActions());

        actions.expectMatchesAdmin("SELECT * FROM test ORDER BY a", "full_access", "SELECT * FROM test ORDER BY a");
        createAuditLogAsserter().expectSqlCompositeAction("test_admin", "test")
                .expectSqlCompositeAction("full_access", "test").assertLogs();
    }

    public void testScrollWithFullAccess() throws Exception {
        createUser("full_access", actions.minimalPermissionsForAllActions());

        actions.expectScrollMatchesAdmin("SELECT * FROM test ORDER BY a", "full_access",
                "SELECT * FROM test ORDER BY a");
        createAuditLogAsserter().expectSqlCompositeAction("test_admin", "test")
                /* Scrolling doesn't have to access the index again, at least not through sql.
                 * If we asserted query and scroll logs then we would see the scroll. */
                .expect(true, SQL_ACTION_NAME, "test_admin", empty())
                .expect(true, SQL_ACTION_NAME, "test_admin", empty())
                .expectSqlCompositeAction("full_access", "test")
                .expect(true, SQL_ACTION_NAME, "full_access", empty())
                .expect(true, SQL_ACTION_NAME, "full_access", empty()).assertLogs();
    }

    public void testQueryNoAccess() throws Exception {
        createUser("no_access", "read_nothing");

        actions.expectForbidden("no_access", "SELECT * FROM test");
        createAuditLogAsserter().expect(false, SQL_ACTION_NAME, "no_access", empty()).assertLogs();
    }

    public void testQueryWrongAccess() throws Exception {
        createUser("wrong_access", "read_something_else");

        actions.expectUnknownIndex("wrong_access", "SELECT * FROM test");
        createAuditLogAsserter()
                //This user has permission to run sql queries so they are given preliminary authorization
                .expect(true, SQL_ACTION_NAME, "wrong_access", empty())
                //the following get index is granted too but against the no indices placeholder, as ignore_unavailable=true
                .expect(true, GetIndexAction.NAME, "wrong_access", hasItems("*", "-*")).assertLogs();
    }

    public void testQuerySingleFieldGranted() throws Exception {
        createUser("only_a", "read_test_a");

        actions.expectMatchesAdmin("SELECT a FROM test ORDER BY a", "only_a", "SELECT * FROM test ORDER BY a");
        createAuditLogAsserter().expectSqlCompositeAction("test_admin", "test")
                .expectSqlCompositeAction("only_a", "test").assertLogs();
    }

    public void testScrollWithSingleFieldGranted() throws Exception {
        createUser("only_a", "read_test_a");

        actions.expectScrollMatchesAdmin("SELECT a FROM test ORDER BY a", "only_a",
                "SELECT * FROM test ORDER BY a");
        createAuditLogAsserter().expectSqlCompositeAction("test_admin", "test")
                /* Scrolling doesn't have to access the index again, at least not through sql.
                 * If we asserted query and scroll logs then we would see the scoll. */
                .expect(true, SQL_ACTION_NAME, "test_admin", empty())
                .expect(true, SQL_ACTION_NAME, "test_admin", empty()).expectSqlCompositeAction("only_a", "test")
                .expect(true, SQL_ACTION_NAME, "only_a", empty()).expect(true, SQL_ACTION_NAME, "only_a", empty())
                .assertLogs();
    }

    public void testQueryStringSingeFieldGrantedWrongRequested() throws Exception {
        createUser("only_a", "read_test_a");

        actions.expectUnknownColumn("only_a", "SELECT c FROM test", "c");
        /* The user has permission to query the index but one of the
         * columns that they explicitly mention is hidden from them
         * by field level access control. This *looks* like a successful
         * query from the audit side because all the permissions checked
         * out but it failed in SQL because it couldn't compile the
         * query without the metadata for the missing field. */
        createAuditLogAsserter().expectSqlCompositeAction("only_a", "test").assertLogs();
    }

    public void testQuerySingleFieldExcepted() throws Exception {
        createUser("not_c", "read_test_a_and_b");

        actions.expectMatchesAdmin("SELECT a, b FROM test ORDER BY a", "not_c", "SELECT * FROM test ORDER BY a");
        createAuditLogAsserter().expectSqlCompositeAction("test_admin", "test")
                .expectSqlCompositeAction("not_c", "test").assertLogs();
    }

    public void testScrollWithSingleFieldExcepted() throws Exception {
        createUser("not_c", "read_test_a_and_b");

        actions.expectScrollMatchesAdmin("SELECT a, b FROM test ORDER BY a", "not_c",
                "SELECT * FROM test ORDER BY a");
        createAuditLogAsserter().expectSqlCompositeAction("test_admin", "test")
                /* Scrolling doesn't have to access the index again, at least not through sql.
                 * If we asserted query and scroll logs then we would see the scroll. */
                .expect(true, SQL_ACTION_NAME, "test_admin", empty())
                .expect(true, SQL_ACTION_NAME, "test_admin", empty()).expectSqlCompositeAction("not_c", "test")
                .expect(true, SQL_ACTION_NAME, "not_c", empty()).expect(true, SQL_ACTION_NAME, "not_c", empty())
                .assertLogs();
    }

    public void testQuerySingleFieldExceptionedWrongRequested() throws Exception {
        createUser("not_c", "read_test_a_and_b");

        actions.expectUnknownColumn("not_c", "SELECT c FROM test", "c");
        /* The user has permission to query the index but one of the
         * columns that they explicitly mention is hidden from them
         * by field level access control. This *looks* like a successful
         * query from the audit side because all the permissions checked
         * out but it failed in SQL because it couldn't compile the
         * query without the metadata for the missing field. */
        createAuditLogAsserter().expectSqlCompositeAction("not_c", "test").assertLogs();
    }

    public void testQueryDocumentExcluded() throws Exception {
        createUser("no_3s", "read_test_without_c_3");

        actions.expectMatchesAdmin("SELECT * FROM test WHERE c != 3 ORDER BY a", "no_3s",
                "SELECT * FROM test ORDER BY a");
        createAuditLogAsserter().expectSqlCompositeAction("test_admin", "test")
                .expectSqlCompositeAction("no_3s", "test").assertLogs();
    }

    public void testShowTablesWorksAsAdmin() throws Exception {
        actions.expectShowTables(Arrays.asList("bort", "test"), null);
        createAuditLogAsserter().expectSqlCompositeAction("test_admin", "bort", "test").assertLogs();
    }

    public void testShowTablesWorksAsFullAccess() throws Exception {
        createUser("full_access", actions.minimalPermissionsForAllActions());

        actions.expectMatchesAdmin("SHOW TABLES LIKE '%t'", "full_access", "SHOW TABLES");
        createAuditLogAsserter().expectSqlCompositeAction("test_admin", "bort", "test")
                .expectSqlCompositeAction("full_access", "bort", "test").assertLogs();
    }

    public void testShowTablesWithNoAccess() throws Exception {
        createUser("no_access", "read_nothing");

        actions.expectForbidden("no_access", "SHOW TABLES");
        createAuditLogAsserter().expect(false, SQL_ACTION_NAME, "no_access", empty()).assertLogs();
    }

    public void testShowTablesWithLimitedAccess() throws Exception {
        createUser("read_bort", "read_bort");

        actions.expectMatchesAdmin("SHOW TABLES LIKE 'bort'", "read_bort", "SHOW TABLES");
        createAuditLogAsserter().expectSqlCompositeAction("test_admin", "bort")
                .expectSqlCompositeAction("read_bort", "bort").assertLogs();
    }

    public void testShowTablesWithLimitedAccessUnaccessableIndex() throws Exception {
        createUser("read_bort", "read_bort");

        actions.expectMatchesAdmin("SHOW TABLES LIKE 'not-created'", "read_bort", "SHOW TABLES LIKE 'test'");
        createAuditLogAsserter().expect(true, SQL_ACTION_NAME, "test_admin", empty())
                .expect(true, GetIndexAction.NAME, "test_admin", contains("*", "-*"))
                .expect(true, SQL_ACTION_NAME, "read_bort", empty())
                .expect(true, GetIndexAction.NAME, "read_bort", contains("*", "-*")).assertLogs();
    }

    public void testDescribeWorksAsAdmin() throws Exception {
        Map<String, String> expected = new TreeMap<>();
        expected.put("a", "BIGINT");
        expected.put("b", "BIGINT");
        expected.put("c", "BIGINT");
        actions.expectDescribe(expected, null);
        createAuditLogAsserter().expectSqlCompositeAction("test_admin", "test").assertLogs();
    }

    public void testDescribeWorksAsFullAccess() throws Exception {
        createUser("full_access", actions.minimalPermissionsForAllActions());

        actions.expectMatchesAdmin("DESCRIBE test", "full_access", "DESCRIBE test");
        createAuditLogAsserter().expectSqlCompositeAction("test_admin", "test")
                .expectSqlCompositeAction("full_access", "test").assertLogs();
    }

    public void testDescribeWithNoAccess() throws Exception {
        createUser("no_access", "read_nothing");

        actions.expectForbidden("no_access", "DESCRIBE test");
        createAuditLogAsserter().expect(false, SQL_ACTION_NAME, "no_access", empty()).assertLogs();
    }

    public void testDescribeWithWrongAccess() throws Exception {
        createUser("wrong_access", "read_something_else");

        actions.expectDescribe(Collections.emptyMap(), "wrong_access");
        createAuditLogAsserter()
                //This user has permission to run sql queries so they are given preliminary authorization
                .expect(true, SQL_ACTION_NAME, "wrong_access", empty())
                //the following get index is granted too but against the no indices placeholder, as ignore_unavailable=true
                .expect(true, GetIndexAction.NAME, "wrong_access", hasItems("*", "-*")).assertLogs();
    }

    public void testDescribeSingleFieldGranted() throws Exception {
        createUser("only_a", "read_test_a");

        actions.expectDescribe(singletonMap("a", "BIGINT"), "only_a");
        createAuditLogAsserter().expectSqlCompositeAction("only_a", "test").assertLogs();
    }

    public void testDescribeSingleFieldExcepted() throws Exception {
        createUser("not_c", "read_test_a_and_b");

        Map<String, String> expected = new TreeMap<>();
        expected.put("a", "BIGINT");
        expected.put("b", "BIGINT");
        actions.expectDescribe(expected, "not_c");
        createAuditLogAsserter().expectSqlCompositeAction("not_c", "test").assertLogs();
    }

    public void testDescribeDocumentExcluded() throws Exception {
        createUser("no_3s", "read_test_without_c_3");

        actions.expectMatchesAdmin("DESCRIBE test", "no_3s", "DESCRIBE test");
        createAuditLogAsserter().expectSqlCompositeAction("test_admin", "test")
                .expectSqlCompositeAction("no_3s", "test").assertLogs();
    }

    public void testNoMonitorMain() throws Exception {
        createUser("no_monitor_main", "no_monitor_main");
        actions.checkNoMonitorMain("no_monitor_main");
    }

    public void testNoGetIndex() throws Exception {
        createUser("no_get_index", "no_get_index");

        actions.expectForbidden("no_get_index", "SELECT * FROM test");
        actions.expectForbidden("no_get_index", "SHOW TABLES LIKE 'test'");
        actions.expectForbidden("no_get_index", "DESCRIBE test");
    }

    protected static void createUser(String name, String role) throws IOException {
        Request request = new Request("PUT", "/_xpack/security/user/" + name);
        XContentBuilder user = JsonXContent.contentBuilder().prettyPrint();
        user.startObject();
        {
            user.field("password", "testpass");
            user.field("roles", role);
        }
        user.endObject();
        request.setJsonEntity(Strings.toString(user));
        client().performRequest(request);
    }

    protected AuditLogAsserter createAuditLogAsserter() {
        return new AuditLogAsserter();
    }

    /**
     * Used to assert audit logs. Logs are asserted to match in any order because
     * we don't always scroll in the same order but each log checker must match a
     * single log and all logs must be matched.
     */
    protected class AuditLogAsserter {
        protected final List<Function<Map<String, Object>, Boolean>> logCheckers = new ArrayList<>();

        public AuditLogAsserter expectSqlCompositeAction(String user, String... indices) {
            expect(true, SQL_ACTION_NAME, user, empty());
            expect(true, GetIndexAction.NAME, user, hasItems(indices));
            return this;
        }

        public AuditLogAsserter expect(boolean granted, String action, String principal,
                Matcher<? extends Iterable<? extends String>> indicesMatcher) {
            String request;
            switch (action) {
            case SQL_ACTION_NAME:
                request = "SqlQueryRequest";
                break;
            case GetIndexAction.NAME:
                request = GetIndexRequest.class.getSimpleName();
                break;
            default:
                throw new IllegalArgumentException("Unknown action [" + action + "]");
            }
            final String eventType = granted ? "access_granted" : "access_denied";
            final String realm = principal.equals("test_admin") ? "default_file" : "default_native";
            return expect(eventType, action, principal, realm, indicesMatcher, request);
        }

        public AuditLogAsserter expect(String eventType, String action, String principal, String realm,
                Matcher<? extends Iterable<? extends String>> indicesMatcher, String request) {
            logCheckers.add(m -> eventType.equals(m.get("event_type")) && action.equals(m.get("action"))
                    && principal.equals(m.get("principal")) && realm.equals(m.get("realm"))
                    && Matchers.nullValue(String.class).matches(m.get("run_by_principal"))
                    && Matchers.nullValue(String.class).matches(m.get("run_by_realm"))
                    && indicesMatcher.matches(m.get("indices")) && request.equals(m.get("request")));
            return this;
        }

        public void assertLogs() throws Exception {
            assertFalse(
                    "Previous test had an audit-related failure. All subsequent audit related assertions are bogus because we can't "
                            + "guarantee that we fully cleaned up after the last test.",
                    auditFailure);
            try {
                assertBusy(() -> {
                    SecurityManager sm = System.getSecurityManager();
                    if (sm != null) {
                        sm.checkPermission(new SpecialPermission());
                    }
                    BufferedReader logReader = AccessController
                            .doPrivileged((PrivilegedAction<BufferedReader>) () -> {
                                try {
                                    return Files.newBufferedReader(AUDIT_LOG_FILE, StandardCharsets.UTF_8);
                                } catch (IOException e) {
                                    throw new RuntimeException(e);
                                }
                            });
                    logReader.skip(auditLogWrittenBeforeTestStart);

                    List<Map<String, Object>> logs = new ArrayList<>();
                    String line;
                    Pattern logPattern = Pattern.compile(
                            ("PART PART PART PART origin_type=PART, origin_address=PART, principal=PART, realm=PART, "
                                    + "(?:run_as_principal=IGN, )?(?:run_as_realm=IGN, )?(?:run_by_principal=PART, )?(?:run_by_realm=PART, )?"
                                    + "roles=PART, action=\\[(.*?)\\], (?:indices=PART, )?request=PART")
                                            .replace(" ", "\\s+").replace("PART", "\\[([^\\]]*)\\]")
                                            .replace("IGN", "\\[[^\\]]*\\]"));
                    // fail(logPattern.toString());
                    while ((line = logReader.readLine()) != null) {
                        java.util.regex.Matcher m = logPattern.matcher(line);
                        if (false == m.matches()) {
                            throw new IllegalArgumentException("Unrecognized log: " + line);
                        }
                        int i = 1;
                        Map<String, Object> log = new HashMap<>();
                        /* We *could* parse the date but leaving it in the original format makes it
                        * easier to find the lines in the file that this log comes from. */
                        log.put("time", m.group(i++));
                        log.put("node", m.group(i++));
                        log.put("origin", m.group(i++));
                        String eventType = m.group(i++);
                        if (false == ("access_denied".equals(eventType) || "access_granted".equals(eventType))) {
                            continue;
                        }
                        log.put("event_type", eventType);
                        log.put("origin_type", m.group(i++));
                        log.put("origin_address", m.group(i++));
                        String principal = m.group(i++);
                        log.put("principal", principal);
                        log.put("realm", m.group(i++));
                        log.put("run_by_principal", m.group(i++));
                        log.put("run_by_realm", m.group(i++));
                        log.put("roles", m.group(i++));
                        String action = m.group(i++);
                        if (false == (SQL_ACTION_NAME.equals(action) || GetIndexAction.NAME.equals(action))) {
                            //TODO we may want to extend this and the assertions to SearchAction.NAME as well
                            continue;
                        }
                        log.put("action", action);
                        // Use a sorted list for indices for consistent error reporting
                        List<String> indices = new ArrayList<>(Strings.tokenizeByCommaToSet(m.group(i++)));
                        Collections.sort(indices);
                        if ("test_admin".equals(principal)) {
                            /* Sometimes we accidentally sneak access to the security tables. This is fine, SQL
                            * drops them from the interface. So we might have access to them, but we don't show
                            * them. */
                            indices.remove(".security");
                            indices.remove(".security-6");
                        }
                        log.put("indices", indices);
                        log.put("request", m.group(i));
                        logs.add(log);
                    }
                    List<Map<String, Object>> allLogs = new ArrayList<>(logs);
                    List<Integer> notMatching = new ArrayList<>();
                    checker: for (int c = 0; c < logCheckers.size(); c++) {
                        Function<Map<String, Object>, Boolean> logChecker = logCheckers.get(c);
                        for (Iterator<Map<String, Object>> logsItr = logs.iterator(); logsItr.hasNext();) {
                            Map<String, Object> log = logsItr.next();
                            if (logChecker.apply(log)) {
                                logsItr.remove();
                                continue checker;
                            }
                        }
                        notMatching.add(c);
                    }
                    if (false == notMatching.isEmpty()) {
                        fail("Some checkers " + notMatching + " didn't match any logs. All logs:"
                                + logsMessage(allLogs) + "\nRemaining logs:" + logsMessage(logs));
                    }
                    if (false == logs.isEmpty()) {
                        fail("Not all logs matched. Unmatched logs:" + logsMessage(logs));
                    }
                });
            } catch (AssertionError e) {
                auditFailure = true;
                logger.warn(
                        "Failed to find an audit log. Skipping remaining tests in this class after this the missing audit"
                                + "logs could turn up later.");
                throw e;
            }
        }

        private String logsMessage(List<Map<String, Object>> logs) {
            if (logs.isEmpty()) {
                return "  none!";
            }
            StringBuilder logsMessage = new StringBuilder();
            for (Map<String, Object> log : logs) {
                logsMessage.append('\n').append(log);
            }
            return logsMessage.toString();
        }
    }
}