com.ikanow.aleph2.logging.service.TestLoggingService.java Source code

Java tutorial

Introduction

Here is the source code for com.ikanow.aleph2.logging.service.TestLoggingService.java

Source

/*******************************************************************************
 * Copyright 2015, The IKANOW Open Source Project.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *******************************************************************************/
package com.ikanow.aleph2.logging.service;

import static org.junit.Assert.*;

import java.io.File;
import java.io.Serializable;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.stream.IntStream;

import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.Filter;
import org.apache.logging.log4j.core.Layout;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.appender.AbstractAppender;
import org.apache.logging.log4j.core.config.AbstractConfiguration;
import org.apache.logging.log4j.core.config.AppenderRef;
import org.apache.logging.log4j.core.config.LoggerConfig;
import org.apache.logging.log4j.core.layout.PatternLayout;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;

import scala.Tuple2;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.ikanow.aleph2.data_model.interfaces.data_services.ISearchIndexService;
import com.ikanow.aleph2.data_model.interfaces.shared_services.IBasicMessageBeanSupplier;
import com.ikanow.aleph2.data_model.interfaces.shared_services.IBucketLogger;
import com.ikanow.aleph2.data_model.interfaces.shared_services.IDataWriteService;
import com.ikanow.aleph2.data_model.interfaces.shared_services.IServiceContext;
import com.ikanow.aleph2.data_model.objects.data_import.DataBucketBean;
import com.ikanow.aleph2.data_model.objects.data_import.DataSchemaBean;
import com.ikanow.aleph2.data_model.objects.data_import.DataSchemaBean.SearchIndexSchemaBean;
import com.ikanow.aleph2.data_model.objects.shared.BasicMessageBean;
import com.ikanow.aleph2.data_model.objects.shared.BasicMessageBeanSupplier;
import com.ikanow.aleph2.data_model.objects.shared.ManagementSchemaBean;
import com.ikanow.aleph2.data_model.objects.shared.ManagementSchemaBean.LoggingSchemaBean;
import com.ikanow.aleph2.data_model.utils.BeanTemplateUtils;
import com.ikanow.aleph2.data_model.utils.BucketUtils;
import com.ikanow.aleph2.data_model.utils.ErrorUtils;
import com.ikanow.aleph2.data_model.utils.ModuleUtils;
import com.ikanow.aleph2.logging.data_model.LoggingServiceConfigBean;
import com.ikanow.aleph2.logging.module.LoggingServiceModule;
import com.ikanow.aleph2.logging.utils.LoggingMergeFunctions;
import com.ikanow.aleph2.logging.utils.LoggingRules;
import com.ikanow.aleph2.logging.utils.LoggingUtils;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;

public class TestLoggingService {
    private static final Logger _logger = LogManager.getLogger();
    private static ISearchIndexService search_index_service;
    private static LoggingService logging_service;
    private static NoLoggingService logging_service_no;
    private static Log4JLoggingService logging_service_log4j;
    protected ObjectMapper _mapper = BeanTemplateUtils.configureMapper(Optional.empty());
    protected Injector _app_injector;
    public static final String VALUE_FIELD = "merge_value";

    // All the services
    @Inject
    IServiceContext _service_context;
    @Inject
    LoggingServiceConfigBean _config;

    @BeforeClass
    public static void setUpBeforeClass() throws Exception {
    }

    public void getServices() {
        _logger.info("run injectModules");
        final File config_file = new File("./resources/context_local_test.properties");
        final Config config = ConfigFactory.parseFile(config_file);

        try {
            _app_injector = ModuleUtils.createTestInjector(Arrays.asList(new LoggingServiceModule()),
                    Optional.of(config));
        } catch (Exception e) {
            try {
                e.printStackTrace();
            } catch (Exception ee) {
                System.out.println(ErrorUtils.getLongForm("{0}", e));
            }
        }

        _app_injector.injectMembers(this);
        search_index_service = _service_context.getSearchIndexService().get();
    }

    @AfterClass
    public static void tearDownAfterClass() throws Exception {
    }

    @Before
    public void setUp() throws Exception {
        getServices();
        logging_service = new LoggingService(_config, _service_context);
        logging_service_no = new NoLoggingService();
        logging_service_log4j = new Log4JLoggingService();
    }

    @After
    public void tearDown() throws Exception {
    }

    @Test
    public void test_validateSchema() {
        final Map<String, String> override_good_single = ImmutableMap.of("a", Level.ALL.toString());
        final Map<String, String> override_bad_single = ImmutableMap.of("a", "fork");
        final Map<String, String> override_bad_with_good = ImmutableMap.of("a", Level.ALL.toString(), "b", "fork");

        //test with good log_level
        final DataBucketBean test_bucket1 = getTestBucket("test", Optional.of(Level.ALL.toString()),
                Optional.empty());
        assertTrue(
                logging_service.validateSchema(test_bucket1.management_schema().logging_schema(), test_bucket1)._2
                        .isEmpty());

        //test with good log_level_override      
        final DataBucketBean test_bucket2 = getTestBucket("test", Optional.empty(),
                Optional.of(override_good_single));
        assertTrue(
                logging_service.validateSchema(test_bucket2.management_schema().logging_schema(), test_bucket2)._2
                        .isEmpty());

        //test with good log_level and log_level override
        final DataBucketBean test_bucket3 = getTestBucket("test", Optional.of(Level.ALL.toString()),
                Optional.of(override_good_single));
        assertTrue(
                logging_service.validateSchema(test_bucket3.management_schema().logging_schema(), test_bucket3)._2
                        .isEmpty());

        //test with bad log_level
        final DataBucketBean test_bucket4 = getTestBucket("test", Optional.of("fork"), Optional.empty());
        assertFalse(
                logging_service.validateSchema(test_bucket4.management_schema().logging_schema(), test_bucket4)._2
                        .isEmpty());

        //test with bad log_level_override
        final DataBucketBean test_bucket5 = getTestBucket("test", Optional.empty(),
                Optional.of(override_bad_single));
        assertFalse(
                logging_service.validateSchema(test_bucket5.management_schema().logging_schema(), test_bucket5)._2
                        .isEmpty());

        //test with good + bad log_level override
        final DataBucketBean test_bucket6 = getTestBucket("test", Optional.empty(),
                Optional.of(override_bad_with_good));
        assertFalse(
                logging_service.validateSchema(test_bucket6.management_schema().logging_schema(), test_bucket6)._2
                        .isEmpty());

        //test with bad log level + bad log_level override
        final DataBucketBean test_bucket7 = getTestBucket("test", Optional.of("fork"),
                Optional.of(override_bad_single));
        assertFalse(
                logging_service.validateSchema(test_bucket7.management_schema().logging_schema(), test_bucket7)._2
                        .isEmpty());
    }

    /**
     * Tests writing messages as user, system, external and checks all the messages were stored.
     * 
     * @throws InterruptedException
     * @throws ExecutionException
     */
    @Test
    public void test_logBucket() throws InterruptedException, ExecutionException {
        final String subsystem_name = "logging_test1";
        final int num_messages_to_log = 50;
        final DataBucketBean test_bucket = getTestBucket("test1", Optional.of(Level.ALL.toString()),
                Optional.empty());
        final IBucketLogger user_logger = logging_service.getLogger(test_bucket);
        final IBucketLogger system_logger = logging_service.getSystemLogger(test_bucket);
        final IBucketLogger external_logger = logging_service.getExternalLogger(subsystem_name);
        //log a few messages
        IntStream.rangeClosed(1, num_messages_to_log).boxed().forEach(i -> {
            user_logger.log(Level.ERROR, ErrorUtils.lazyBuildMessage(true, () -> subsystem_name,
                    () -> "test_message " + i, () -> null, () -> "no error", () -> Collections.emptyMap()));
            system_logger.log(Level.ERROR, ErrorUtils.lazyBuildMessage(true, () -> subsystem_name,
                    () -> "test_message " + i, () -> null, () -> "no error", () -> Collections.emptyMap()));
            external_logger.log(Level.ERROR, ErrorUtils.lazyBuildMessage(true, () -> subsystem_name,
                    () -> "test_message " + i, () -> null, () -> "no error", () -> Collections.emptyMap()));
        });

        user_logger.flush();
        system_logger.flush();
        external_logger.flush();

        //check its in ES, wait 10s max for the index to refresh
        final DataBucketBean logging_test_bucket = BucketUtils.convertDataBucketBeanToLogging(test_bucket);
        final IDataWriteService<BasicMessageBean> logging_crud = search_index_service.getDataService().get()
                .getWritableDataService(BasicMessageBean.class, logging_test_bucket, Optional.empty(),
                        Optional.empty())
                .get();
        waitForResults(logging_crud, 10, num_messages_to_log * 2);
        assertEquals(num_messages_to_log * 2, logging_crud.countObjects().get().longValue());

        final DataBucketBean logging_external_test_bucket = BucketUtils
                .convertDataBucketBeanToLogging(BeanTemplateUtils.clone(test_bucket)
                        .with(DataBucketBean::full_name, "/external/" + subsystem_name + "/").done());
        final IDataWriteService<BasicMessageBean> logging_crud_external = search_index_service.getDataService()
                .get().getWritableDataService(BasicMessageBean.class, logging_external_test_bucket,
                        Optional.empty(), Optional.empty())
                .get();
        waitForResults(logging_crud_external, 10, num_messages_to_log);
        assertEquals(num_messages_to_log, logging_crud_external.countObjects().get().longValue());

        //cleanup
        logging_crud.deleteDatastore().get();
    }

    /**
     * Tests writing messages as user, system, external at 3 different log levels and verifies
     * the too low of level messages were filtered out (not written to storage).
     * 
     * @throws InterruptedException
     * @throws ExecutionException
     */
    @Test
    public void test_logFilter() throws InterruptedException, ExecutionException {
        final String subsystem_name = "logging_test2";
        final int num_messages_to_log_each_type = 5;
        final List<Level> levels = Arrays.asList(Level.DEBUG, Level.INFO, Level.ERROR);
        final DataBucketBean test_bucket = getTestBucket("test2", Optional.of(Level.ERROR.toString()),
                Optional.empty());
        final IBucketLogger user_logger = logging_service.getLogger(test_bucket);
        final IBucketLogger system_logger = logging_service.getSystemLogger(test_bucket);
        final IBucketLogger external_logger = logging_service.getExternalLogger(subsystem_name);
        //log a few messages
        IntStream.rangeClosed(1, num_messages_to_log_each_type).boxed().forEach(i -> {
            levels.stream().forEach(level -> {
                user_logger.log(level, ErrorUtils.lazyBuildMessage(true, () -> subsystem_name,
                        () -> "test_message " + i, () -> null, () -> "no error", () -> Collections.emptyMap()));
                system_logger.log(level, ErrorUtils.lazyBuildMessage(true, () -> subsystem_name,
                        () -> "test_message " + i, () -> null, () -> "no error", () -> Collections.emptyMap()));
                external_logger.log(level, ErrorUtils.lazyBuildMessage(true, () -> subsystem_name,
                        () -> "test_message " + i, () -> null, () -> "no error", () -> Collections.emptyMap()));
            });
        });

        user_logger.flush();
        system_logger.flush();
        external_logger.flush();

        //check its in ES, wait 10s max for the index to refresh
        final DataBucketBean logging_test_bucket = BucketUtils.convertDataBucketBeanToLogging(test_bucket);
        final IDataWriteService<BasicMessageBean> logging_crud = search_index_service.getDataService().get()
                .getWritableDataService(BasicMessageBean.class, logging_test_bucket, Optional.empty(),
                        Optional.empty())
                .get();
        waitForResults(logging_crud, 10, 10);
        assertEquals(10, logging_crud.countObjects().get().longValue()); //should only have logged ERROR messages

        final DataBucketBean logging_external_test_bucket = BucketUtils
                .convertDataBucketBeanToLogging(BeanTemplateUtils.clone(test_bucket)
                        .with(DataBucketBean::full_name, "/external/" + subsystem_name + "/").done());
        final IDataWriteService<BasicMessageBean> logging_crud_external = search_index_service.getDataService()
                .get().getWritableDataService(BasicMessageBean.class, logging_external_test_bucket,
                        Optional.empty(), Optional.empty())
                .get();
        waitForResults(logging_crud_external, 10, 15);
        assertEquals(15, logging_crud_external.countObjects().get().longValue());

        //cleanup
        logging_crud.deleteDatastore().get();
    }

    /**
     * Tests and empty management schema falls back to defaults and filters items by the
     * defaults used in the config file.
     * @throws InterruptedException
     * @throws ExecutionException
     */
    @Test
    public void test_logEmptyManagement() throws InterruptedException, ExecutionException {
        //if no logging schema is supplied, falls back to defaults in config file (if any)
        //config file is set to:
        //SYSTEM: DEBUG
        //USER: ERROR
        //therefore we should see 
        //5 messages of each type to make it through via the system calls (15)
        //5 messages of DEBUG making it through via user calls (5)
        //5 messages of each type to make it through via the external calls (15)
        final String subsystem_name = "logging_test3";
        final int num_messages_to_log_each_type = 5;
        final List<Level> levels = Arrays.asList(Level.DEBUG, Level.INFO, Level.ERROR);
        final DataBucketBean test_bucket = getEmptyTestBucket("test3");
        final IBucketLogger user_logger = logging_service.getLogger(test_bucket);
        final IBucketLogger system_logger = logging_service.getSystemLogger(test_bucket);
        final IBucketLogger external_logger = logging_service.getExternalLogger(subsystem_name);
        //log a few messages
        IntStream.rangeClosed(1, num_messages_to_log_each_type).boxed().forEach(i -> {
            levels.stream().forEach(level -> {
                user_logger.log(level, ErrorUtils.lazyBuildMessage(true, () -> subsystem_name,
                        () -> "test_message " + i, () -> null, () -> "no error", () -> Collections.emptyMap()));
                system_logger.log(level, ErrorUtils.lazyBuildMessage(true, () -> subsystem_name,
                        () -> "test_message " + i, () -> null, () -> "no error", () -> Collections.emptyMap()));
                external_logger.log(level, ErrorUtils.lazyBuildMessage(true, () -> subsystem_name,
                        () -> "test_message " + i, () -> null, () -> "no error", () -> Collections.emptyMap()));
            });
        });

        user_logger.flush();
        system_logger.flush();
        external_logger.flush();

        //check its in ES, wait 10s max for the index to refresh
        //USER + SYSTEM
        final DataBucketBean logging_test_bucket = BucketUtils.convertDataBucketBeanToLogging(test_bucket);
        final IDataWriteService<BasicMessageBean> logging_crud = search_index_service.getDataService().get()
                .getWritableDataService(BasicMessageBean.class, logging_test_bucket, Optional.empty(),
                        Optional.empty())
                .get();
        waitForResults(logging_crud, 10, 20);
        assertEquals(20, logging_crud.countObjects().get().longValue()); //should only have logged ERROR messages

        //EXTERNAL
        final DataBucketBean logging_external_test_bucket = BucketUtils
                .convertDataBucketBeanToLogging(BeanTemplateUtils.clone(test_bucket)
                        .with(DataBucketBean::full_name, "/external/" + subsystem_name + "/").done());
        final IDataWriteService<BasicMessageBean> logging_crud_external = search_index_service.getDataService()
                .get().getWritableDataService(BasicMessageBean.class, logging_external_test_bucket,
                        Optional.empty(), Optional.empty())
                .get();
        waitForResults(logging_crud_external, 10, 15);
        assertEquals(15, logging_crud_external.countObjects().get().longValue());

        //cleanup
        logging_crud.deleteDatastore().get();
    }

    @Test
    public void test_NoLoggingService() throws InterruptedException, ExecutionException {
        final String subsystem_name = "logging_test4";
        final int num_messages_to_log = 50;
        final DataBucketBean test_bucket = getTestBucket("test4", Optional.of(Level.ALL.toString()),
                Optional.empty());
        final IBucketLogger user_logger = logging_service_no.getLogger(test_bucket);
        final IBucketLogger system_logger = logging_service_no.getSystemLogger(test_bucket);
        final IBucketLogger external_logger = logging_service_no.getExternalLogger(subsystem_name);
        //log a few messages
        IntStream.rangeClosed(1, num_messages_to_log).boxed().forEach(i -> {
            user_logger.log(Level.ERROR, ErrorUtils.lazyBuildMessage(true, () -> subsystem_name,
                    () -> "test_message " + i, () -> null, () -> "no error", () -> Collections.emptyMap()));
            system_logger.log(Level.ERROR, ErrorUtils.lazyBuildMessage(true, () -> subsystem_name,
                    () -> "test_message " + i, () -> null, () -> "no error", () -> Collections.emptyMap()));
            external_logger.log(Level.ERROR, ErrorUtils.lazyBuildMessage(true, () -> subsystem_name,
                    () -> "test_message " + i, () -> null, () -> "no error", () -> Collections.emptyMap()));
        });

        user_logger.flush();
        system_logger.flush();
        external_logger.flush();

        //check its in ES, wait 10s max for the index to refresh
        final DataBucketBean logging_test_bucket = BucketUtils.convertDataBucketBeanToLogging(test_bucket);
        final IDataWriteService<BasicMessageBean> logging_crud = search_index_service.getDataService().get()
                .getWritableDataService(BasicMessageBean.class, logging_test_bucket, Optional.empty(),
                        Optional.empty())
                .get();
        assertEquals(0, logging_crud.countObjects().get().longValue());

        final DataBucketBean logging_external_test_bucket = BucketUtils
                .convertDataBucketBeanToLogging(BeanTemplateUtils.clone(test_bucket)
                        .with(DataBucketBean::full_name, "/external/" + subsystem_name + "/").done());
        final IDataWriteService<BasicMessageBean> logging_crud_external = search_index_service.getDataService()
                .get().getWritableDataService(BasicMessageBean.class, logging_external_test_bucket,
                        Optional.empty(), Optional.empty())
                .get();
        assertEquals(0, logging_crud_external.countObjects().get().longValue());

        //cleanup
        logging_crud.deleteDatastore().get();
    }

    @Test
    public void test_Log4JLoggingService() throws InterruptedException, ExecutionException {
        //add in our appender so we can count messages
        final TestAppender appender = new TestAppender("test", null, PatternLayout.createDefaultLayout());
        appender.start();
        final LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
        final AbstractConfiguration config = (AbstractConfiguration) ctx.getConfiguration();
        config.addAppender(appender);
        AppenderRef ref = AppenderRef.createAppenderRef("test", null, null);
        AppenderRef[] refs = new AppenderRef[] { ref };
        LoggerConfig loggerConfig = LoggerConfig.createLogger("false", Level.ALL, "test", "true", refs, null,
                config, null);
        loggerConfig.addAppender(appender, null, null);
        config.addLogger("test", loggerConfig);
        ctx.updateLoggers();
        config.getRootLogger().addAppender(appender, Level.ALL, null);

        //run a normal test
        final String subsystem_name = "logging_test5";
        final int num_messages_to_log = 50;
        final DataBucketBean test_bucket = getTestBucket("test5", Optional.of(Level.ALL.toString()),
                Optional.empty());
        final IBucketLogger user_logger = logging_service_log4j.getLogger(test_bucket);
        final IBucketLogger system_logger = logging_service_log4j.getSystemLogger(test_bucket);
        final IBucketLogger external_logger = logging_service_log4j.getExternalLogger(subsystem_name);
        //log a few messages
        IntStream.rangeClosed(1, num_messages_to_log).boxed().forEach(i -> {
            user_logger.log(Level.ERROR, ErrorUtils.lazyBuildMessage(true, () -> subsystem_name,
                    () -> "test_message " + i, () -> null, () -> "no error", () -> Collections.emptyMap()));
            system_logger.log(Level.ERROR, ErrorUtils.lazyBuildMessage(true, () -> subsystem_name,
                    () -> "test_message " + i, () -> null, () -> "no error", () -> Collections.emptyMap()));
            external_logger.log(Level.ERROR, ErrorUtils.lazyBuildMessage(true, () -> subsystem_name,
                    () -> "test_message " + i, () -> null, () -> "no error", () -> Collections.emptyMap()));
        });

        user_logger.flush();
        system_logger.flush();
        external_logger.flush();

        //check in our appender for how many messages we received      
        assertEquals(num_messages_to_log * 3, appender.message_count);

        //cleanup
        config.removeAppender("test");
        ctx.updateLoggers();
    }

    @SuppressWarnings("unchecked")
    @Test
    public void test_logFunctions() throws InterruptedException, ExecutionException {
        final String subsystem_name = "logging_test6";
        final int num_messages_to_log_each_type = 5;
        final List<Level> levels = Arrays.asList(Level.DEBUG, Level.INFO, Level.ERROR);
        final DataBucketBean test_bucket = getTestBucket("test6", Optional.of(Level.ERROR.toString()),
                Optional.empty());
        final IBucketLogger user_logger = logging_service.getLogger(test_bucket);
        final IBucketLogger system_logger = logging_service.getSystemLogger(test_bucket);
        final IBucketLogger external_logger = logging_service.getExternalLogger(subsystem_name);

        //log a few messages
        IntStream.rangeClosed(1, num_messages_to_log_each_type).boxed().forEach(i -> {
            levels.stream().forEach(level -> {
                //append
                user_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> Collections.emptyMap()),
                        "key1", Collections.emptyList(), Optional.empty(), LoggingMergeFunctions.appendMessage());
                system_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> Collections.emptyMap()),
                        "key1", Collections.emptyList(), Optional.empty(), LoggingMergeFunctions.appendMessage());
                external_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> Collections.emptyMap()),
                        "key1", Collections.emptyList(), Optional.empty(), LoggingMergeFunctions.appendMessage());

                //count
                user_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> Collections.emptyMap()),
                        "key1", Collections.emptyList(), Optional.empty(), LoggingMergeFunctions.countMessages());
                system_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> Collections.emptyMap()),
                        "key1", Collections.emptyList(), Optional.empty(), LoggingMergeFunctions.countMessages());
                external_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> Collections.emptyMap()),
                        "key1", Collections.emptyList(), Optional.empty(), LoggingMergeFunctions.countMessages());

                //sum
                user_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                        "key1", Collections.emptyList(), Optional.empty(),
                        LoggingMergeFunctions.sumField(VALUE_FIELD));
                system_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                        "key1", Collections.emptyList(), Optional.empty(),
                        LoggingMergeFunctions.sumField(VALUE_FIELD));
                external_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                        "key1", Collections.emptyList(), Optional.empty(),
                        LoggingMergeFunctions.sumField(VALUE_FIELD));

                //min
                user_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                        "key1", Collections.emptyList(), Optional.empty(),
                        LoggingMergeFunctions.minField(VALUE_FIELD));
                system_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                        "key1", Collections.emptyList(), Optional.empty(),
                        LoggingMergeFunctions.minField(VALUE_FIELD));
                external_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                        "key1", Collections.emptyList(), Optional.empty(),
                        LoggingMergeFunctions.minField(VALUE_FIELD));
                //max
                user_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                        "key1", Collections.emptyList(), Optional.empty(),
                        LoggingMergeFunctions.maxField(VALUE_FIELD));
                system_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                        "key1", Collections.emptyList(), Optional.empty(),
                        LoggingMergeFunctions.maxField(VALUE_FIELD));
                external_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                        "key1", Collections.emptyList(), Optional.empty(),
                        LoggingMergeFunctions.maxField(VALUE_FIELD));

                //minmax
                user_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                        "key1", Collections.emptyList(), Optional.empty(),
                        LoggingMergeFunctions.minMaxField(VALUE_FIELD));
                system_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                        "key1", Collections.emptyList(), Optional.empty(),
                        LoggingMergeFunctions.minMaxField(VALUE_FIELD));
                external_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                        "key1", Collections.emptyList(), Optional.empty(),
                        LoggingMergeFunctions.minMaxField(VALUE_FIELD));

                //chaining test
                user_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                        "key1", Collections.emptyList(), Optional.empty(),
                        LoggingMergeFunctions.sumField(VALUE_FIELD, "out1", false),
                        LoggingMergeFunctions.sumField(VALUE_FIELD, "out2", false));
                system_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                        "key1", Collections.emptyList(), Optional.empty(),
                        LoggingMergeFunctions.sumField(VALUE_FIELD, "out1", false),
                        LoggingMergeFunctions.sumField(VALUE_FIELD, "out2", false));
                external_logger.log(level,
                        ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message " + i,
                                () -> null, () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                        "key1", Collections.emptyList(), Optional.empty(),
                        LoggingMergeFunctions.sumField(VALUE_FIELD, "out1", false),
                        LoggingMergeFunctions.sumField(VALUE_FIELD, "out2", false));
            });
        });

        user_logger.flush();
        system_logger.flush();
        external_logger.flush();

        //check its in ES, wait 10s max for the index to refresh
        final DataBucketBean logging_test_bucket = BucketUtils.convertDataBucketBeanToLogging(test_bucket);
        final IDataWriteService<BasicMessageBean> logging_crud = search_index_service.getDataService().get()
                .getWritableDataService(BasicMessageBean.class, logging_test_bucket, Optional.empty(),
                        Optional.empty())
                .get();
        waitForResults(logging_crud, 10, 70);
        assertEquals(70, logging_crud.countObjects().get().longValue()); //should only have logged ERROR messages

        final DataBucketBean logging_external_test_bucket = BucketUtils
                .convertDataBucketBeanToLogging(BeanTemplateUtils.clone(test_bucket)
                        .with(DataBucketBean::full_name, "/external/" + subsystem_name + "/").done());
        final IDataWriteService<BasicMessageBean> logging_crud_external = search_index_service.getDataService()
                .get().getWritableDataService(BasicMessageBean.class, logging_external_test_bucket,
                        Optional.empty(), Optional.empty())
                .get();
        waitForResults(logging_crud_external, 10, 105);
        assertEquals(105, logging_crud_external.countObjects().get().longValue());

        //cleanup
        logging_crud.deleteDatastore().get();
    }

    @SuppressWarnings("unchecked")
    @Test
    public void test_logRules() throws InterruptedException, ExecutionException {
        final String subsystem_name = "logging_test7";
        final int num_messages_to_log = 50;
        final DataBucketBean test_bucket = getTestBucket("test7", Optional.of(Level.ALL.toString()),
                Optional.empty());
        final IBucketLogger user_logger = logging_service.getLogger(test_bucket);
        final IBucketLogger system_logger = logging_service.getSystemLogger(test_bucket);
        final IBucketLogger external_logger = logging_service.getExternalLogger(subsystem_name);
        //log a few messages
        IntStream.rangeClosed(1, num_messages_to_log).boxed().forEach(i -> {
            //NOTE HAVE TO DO TIME RULE FIRST, BECAUSE IT WILL GET UPDATED EVERY OTHER SUCCESSFUL LOG MESSAGE
            //rule: to log every 30s, should only log the first time, then test should finish before 2nd one is allowed
            //should result in 1 message each
            user_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message1 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key1", Arrays.asList(LoggingRules.logEveryMilliseconds(500000)), Optional.empty(),
                    LoggingMergeFunctions.replaceMessage());
            system_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message1 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key1", Arrays.asList(LoggingRules.logEveryMilliseconds(500000)), Optional.empty(),
                    LoggingMergeFunctions.replaceMessage());
            external_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message1 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key1", Arrays.asList(LoggingRules.logEveryMilliseconds(500000)), Optional.empty(),
                    LoggingMergeFunctions.replaceMessage());

            //rule: log every 5 messages
            //should result in num_messages_to_log/5 each aka 10 each
            //NOTE count field has to go on its own key, because count is being increased for every successful message in any of the tests
            user_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message2 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key2", Arrays.asList(LoggingRules.logEveryCount(5)), Optional.empty(),
                    LoggingMergeFunctions.replaceMessage());
            system_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message2 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key2", Arrays.asList(LoggingRules.logEveryCount(5)), Optional.empty(),
                    LoggingMergeFunctions.replaceMessage());
            external_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message2 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key2", Arrays.asList(LoggingRules.logEveryCount(5)), Optional.empty(),
                    LoggingMergeFunctions.replaceMessage());

            //rule: log if max over threshold
            //should result in 44 message over each
            user_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message3 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key1",
                    Arrays.asList(
                            LoggingRules.logOutsideThreshold(VALUE_FIELD, Optional.empty(), Optional.of(6.0))),
                    Optional.empty(), LoggingMergeFunctions.replaceMessage());
            system_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message3 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key1",
                    Arrays.asList(
                            LoggingRules.logOutsideThreshold(VALUE_FIELD, Optional.empty(), Optional.of(6.0))),
                    Optional.empty(), LoggingMergeFunctions.replaceMessage());
            external_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message3 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key1",
                    Arrays.asList(
                            LoggingRules.logOutsideThreshold(VALUE_FIELD, Optional.empty(), Optional.of(6.0))),
                    Optional.empty(), LoggingMergeFunctions.replaceMessage());

            //rule: log if min under threshold
            //should result in 1 under each
            user_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message4 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key1",
                    Arrays.asList(
                            LoggingRules.logOutsideThreshold(VALUE_FIELD, Optional.of(2.0), Optional.empty())),
                    Optional.empty(), LoggingMergeFunctions.replaceMessage());
            system_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message4 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key1",
                    Arrays.asList(
                            LoggingRules.logOutsideThreshold(VALUE_FIELD, Optional.of(2.0), Optional.empty())),
                    Optional.empty(), LoggingMergeFunctions.replaceMessage());
            external_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message4 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key1",
                    Arrays.asList(
                            LoggingRules.logOutsideThreshold(VALUE_FIELD, Optional.of(2.0), Optional.empty())),
                    Optional.empty(), LoggingMergeFunctions.replaceMessage());

            //rule: log if min/max outside thresholds
            //should result in 1 message under, 44 over each (45 each)
            user_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message5 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key1",
                    Arrays.asList(
                            LoggingRules.logOutsideThreshold(VALUE_FIELD, Optional.of(2.0), Optional.of(6.0))),
                    Optional.empty(), LoggingMergeFunctions.replaceMessage());
            system_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message5 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key1",
                    Arrays.asList(
                            LoggingRules.logOutsideThreshold(VALUE_FIELD, Optional.of(2.0), Optional.of(6.0))),
                    Optional.empty(), LoggingMergeFunctions.replaceMessage());
            external_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message5 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key1",
                    Arrays.asList(
                            LoggingRules.logOutsideThreshold(VALUE_FIELD, Optional.of(2.0), Optional.of(6.0))),
                    Optional.empty(), LoggingMergeFunctions.replaceMessage());

            //test multi rules rule: log every 2 messages, or if max over threshold of 45
            //should result in 22 messages + 5 messages (27 each) was 202 got 302
            user_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message6 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key1",
                    Arrays.asList(LoggingRules.logEveryCount(2),
                            LoggingRules.logOutsideThreshold(VALUE_FIELD, Optional.empty(), Optional.of(45.0))),
                    Optional.empty(), LoggingMergeFunctions.replaceMessage());
            system_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message6 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key1",
                    Arrays.asList(LoggingRules.logEveryCount(2),
                            LoggingRules.logOutsideThreshold(VALUE_FIELD, Optional.empty(), Optional.of(45.0))),
                    Optional.empty(), LoggingMergeFunctions.replaceMessage());
            external_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message6 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key1",
                    Arrays.asList(LoggingRules.logEveryCount(2),
                            LoggingRules.logOutsideThreshold(VALUE_FIELD, Optional.empty(), Optional.of(45.0))),
                    Optional.empty(), LoggingMergeFunctions.replaceMessage());

            //test output formatter, doesn't do anything special, adds 44 messages each
            user_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message7 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key3",
                    Arrays.asList(
                            LoggingRules.logOutsideThreshold(VALUE_FIELD, Optional.empty(), Optional.of(6.0))),
                    Optional.of((b) -> {
                        return BeanTemplateUtils.clone(b).with(BasicMessageBean::message, "gestapo!").done();
                    }), LoggingMergeFunctions.replaceMessage());
            system_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message7 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key3",
                    Arrays.asList(
                            LoggingRules.logOutsideThreshold(VALUE_FIELD, Optional.empty(), Optional.of(6.0))),
                    Optional.of((b) -> {
                        return BeanTemplateUtils.clone(b).with(BasicMessageBean::message, "gestapo!").done();
                    }), LoggingMergeFunctions.replaceMessage());
            external_logger.log(Level.ERROR,
                    ErrorUtils.lazyBuildMessage(true, () -> subsystem_name, () -> "test_message7 " + i, () -> null,
                            () -> "no error", () -> ImmutableMap.of(VALUE_FIELD, (double) i)),
                    "key3",
                    Arrays.asList(
                            LoggingRules.logOutsideThreshold(VALUE_FIELD, Optional.empty(), Optional.of(6.0))),
                    Optional.of((b) -> {
                        return BeanTemplateUtils.clone(b).with(BasicMessageBean::message, "gestapo!").done();
                    }), LoggingMergeFunctions.replaceMessage());
        });

        user_logger.flush();
        system_logger.flush();
        external_logger.flush();

        //check its in ES, wait 10s max for the index to refresh
        final DataBucketBean logging_test_bucket = BucketUtils.convertDataBucketBeanToLogging(test_bucket);
        final IDataWriteService<JsonNode> logging_crud = search_index_service.getDataService().get()
                .getWritableDataService(JsonNode.class, logging_test_bucket, Optional.empty(), Optional.empty())
                .get();
        waitForResults(logging_crud, 10, (1 + 10 + 44 + 1 + 45 + 27 + 44) * 2);
        assertEquals((1 + 10 + 44 + 1 + 45 + 27 + 44) * 2, logging_crud.countObjects().get().longValue());

        final DataBucketBean logging_external_test_bucket = BucketUtils
                .convertDataBucketBeanToLogging(BeanTemplateUtils.clone(test_bucket)
                        .with(DataBucketBean::full_name, "/external/" + subsystem_name + "/").done());
        final IDataWriteService<BasicMessageBean> logging_crud_external = search_index_service.getDataService()
                .get().getWritableDataService(BasicMessageBean.class, logging_external_test_bucket,
                        Optional.empty(), Optional.empty())
                .get();
        waitForResults(logging_crud_external, 10, 1 + (num_messages_to_log / 5) + 44 + 1 + 45 + 27 + 44);
        assertEquals(1 + (num_messages_to_log / 5) + 44 + 1 + 45 + 27 + 44,
                logging_crud_external.countObjects().get().longValue());

        //cleanup
        logging_crud.deleteDatastore().get();
    }

    @SuppressWarnings("serial")
    @Test
    public void test_LoggingMergeFunctions() {
        //testing LoggingMergeFunctions.getDetailsMapValue
        assertNull(LoggingUtils.getDetailsMapValue(BeanTemplateUtils.build(BasicMessageBean.class).done().get(),
                "field1", String.class));
        assertNull(LoggingUtils.getDetailsMapValue(
                BeanTemplateUtils.build(BasicMessageBean.class).with(BasicMessageBean::details, null).done().get(),
                "field1", String.class));
        assertNull(LoggingUtils.getDetailsMapValue(BeanTemplateUtils.build(BasicMessageBean.class)
                .with(BasicMessageBean::details, ImmutableMap.of()).done().get(), "field1", String.class));
        assertTrue(LoggingUtils.getDetailsMapValue(
                BeanTemplateUtils.build(BasicMessageBean.class)
                        .with(BasicMessageBean::details, ImmutableMap.of("field1", "value1")).done().get(),
                "field1", String.class).equals("value1"));

        //test LoggingMergeFunctions.copyDetailsPutValue
        assertEquals(1,
                LoggingUtils
                        .mergeDetailsAddValue(BeanTemplateUtils.build(BasicMessageBean.class).done().get(),
                                BeanTemplateUtils.build(BasicMessageBean.class).done().get(), "field1", "value1")
                        .size());
        assertEquals(1,
                LoggingUtils
                        .mergeDetailsAddValue(
                                BeanTemplateUtils.build(BasicMessageBean.class)
                                        .with(BasicMessageBean::details, null).done().get(),
                                BeanTemplateUtils.build(BasicMessageBean.class).done().get(), "field1", "value1")
                        .size());
        assertEquals(1,
                LoggingUtils.mergeDetailsAddValue(BeanTemplateUtils.build(BasicMessageBean.class).done().get(),
                        BeanTemplateUtils.build(BasicMessageBean.class)
                                .with(BasicMessageBean::details, ImmutableMap.of()).done().get(),
                        "field1", "value1").size());
        assertEquals(1,
                LoggingUtils.mergeDetailsAddValue(BeanTemplateUtils.build(BasicMessageBean.class).done().get(),
                        BeanTemplateUtils.build(BasicMessageBean.class)
                                .with(BasicMessageBean::details, ImmutableMap.of("field1", "value1")).done().get(),
                        "field1", "value1").size());
        assertEquals(2,
                LoggingUtils.mergeDetailsAddValue(BeanTemplateUtils.build(BasicMessageBean.class).done().get(),
                        BeanTemplateUtils.build(BasicMessageBean.class)
                                .with(BasicMessageBean::details, ImmutableMap.of("field2", "value2")).done().get(),
                        "field1", "value1").size());
        assertEquals(3,
                LoggingUtils.mergeDetailsAddValue(
                        BeanTemplateUtils.build(BasicMessageBean.class)
                                .with(BasicMessageBean::details, ImmutableMap.of("field3", "value3")).done().get(),
                        BeanTemplateUtils.build(BasicMessageBean.class)
                                .with(BasicMessageBean::details, ImmutableMap.of("field2", "value2")).done().get(),
                        "field1", "value1").size());

        //test LoggingMergeFunctions.getMinOrNull
        assertNull(LoggingMergeFunctions.getMinMaxOrNull(null, null, true));
        assertEquals(1D, LoggingMergeFunctions.getMinMaxOrNull(null, 1D, true), .0001D);
        assertEquals(1D, LoggingMergeFunctions.getMinMaxOrNull(1D, null, true), .0001D);
        assertEquals(1D, LoggingMergeFunctions.getMinMaxOrNull(2.1D, 1D, true), .0001D);
        assertEquals(1D, LoggingMergeFunctions.getMinMaxOrNull(1D, 2.1D, true), .0001D);
        assertEquals(1D, LoggingMergeFunctions.getMinMaxOrNull(null, 1D, false), .0001D);
        assertEquals(1D, LoggingMergeFunctions.getMinMaxOrNull(1D, null, false), .0001D);
        assertEquals(2.1D, LoggingMergeFunctions.getMinMaxOrNull(2.1D, 1D, false), .0001D);
        assertEquals(2.1D, LoggingMergeFunctions.getMinMaxOrNull(1D, 2.1D, false), .0001D);

        //test LoggingUtils.updateInfo
        assertEquals(1L,
                LoggingUtils.updateInfo(new Tuple2<BasicMessageBean, Map<String, Object>>(
                        BeanTemplateUtils.build(BasicMessageBean.class).done().get(),
                        new HashMap<String, Object>()), Optional.empty())._2.get(LoggingUtils.LOG_COUNT_FIELD));
        assertEquals(1L,
                LoggingUtils.updateInfo(new Tuple2<BasicMessageBean, Map<String, Object>>(
                        BeanTemplateUtils.build(BasicMessageBean.class).done().get(),
                        new HashMap<String, Object>()), Optional.of(new Date().getTime()))._2
                                .get(LoggingUtils.LOG_COUNT_FIELD));
        assertEquals(5L,
                LoggingUtils.updateInfo(new Tuple2<BasicMessageBean, Map<String, Object>>(
                        BeanTemplateUtils.build(BasicMessageBean.class).done().get(),
                        new HashMap<String, Object>() {
                            {
                                put(LoggingUtils.LOG_COUNT_FIELD, 4L);
                            }
                        }), Optional.empty())._2.get(LoggingUtils.LOG_COUNT_FIELD));
        assertNotNull(LoggingUtils.updateInfo(new Tuple2<BasicMessageBean, Map<String, Object>>(
                BeanTemplateUtils.build(BasicMessageBean.class).done().get(), new HashMap<String, Object>() {
                    {
                        put(LoggingUtils.LAST_LOG_TIMESTAMP_FIELD, 0L);
                    }
                }), Optional.of(new Date().getTime()))._2.get(LoggingUtils.LAST_LOG_TIMESTAMP_FIELD));

        //test appender
        assertTrue(
                LoggingMergeFunctions.appendMessage()
                        .apply(BeanTemplateUtils.build(BasicMessageBean.class)
                                .with(BasicMessageBean::message, "msg1").done().get(), null)
                        .message().equals("msg1"));
        assertTrue(LoggingMergeFunctions.appendMessage()
                .apply(BeanTemplateUtils.build(BasicMessageBean.class).with(BasicMessageBean::message, "msg1")
                        .done().get(),
                        BeanTemplateUtils.build(BasicMessageBean.class).with(BasicMessageBean::message, "msg2")
                                .done().get())
                .message().equals("msg1 msg2"));

        //test counter
        assertEquals((long) LoggingMergeFunctions
                .countMessages().apply(BeanTemplateUtils.build(BasicMessageBean.class)
                        .with(BasicMessageBean::message, "msg1").done().get(), null)
                .details().get(LoggingMergeFunctions.COUNT_FIELD), 1L);
        assertEquals((long) LoggingMergeFunctions.countMessages()
                .apply(BeanTemplateUtils.build(BasicMessageBean.class).with(BasicMessageBean::message, "msg1")
                        .done().get(),
                        BeanTemplateUtils.build(BasicMessageBean.class)
                                .with(BasicMessageBean::details,
                                        ImmutableMap.of(LoggingMergeFunctions.COUNT_FIELD, 4L))
                                .done().get())
                .details().get(LoggingMergeFunctions.COUNT_FIELD), 5L);

        //test sum
        assertEquals((double) LoggingMergeFunctions
                .sumField(VALUE_FIELD).apply(BeanTemplateUtils.build(BasicMessageBean.class)
                        .with(BasicMessageBean::message, "msg1").done().get(), null)
                .details().get(LoggingMergeFunctions.SUM_FIELD), 0D, 0.0001D);
        assertEquals((double) LoggingMergeFunctions.sumField(VALUE_FIELD)
                .apply(BeanTemplateUtils.build(BasicMessageBean.class)
                        .with(BasicMessageBean::details, ImmutableMap.of(VALUE_FIELD, 5.2)).done().get(),
                        BeanTemplateUtils.build(BasicMessageBean.class)
                                .with(BasicMessageBean::details,
                                        ImmutableMap.of(LoggingMergeFunctions.SUM_FIELD, 4.66))
                                .done().get())
                .details().get(LoggingMergeFunctions.SUM_FIELD), 9.86D, 0.0001D);

        //test min
        assertNull(LoggingMergeFunctions
                .minField(VALUE_FIELD).apply(BeanTemplateUtils.build(BasicMessageBean.class)
                        .with(BasicMessageBean::message, "msg1").done().get(), null)
                .details().get(LoggingMergeFunctions.MIN_FIELD));
        assertEquals((double) LoggingMergeFunctions.minField(VALUE_FIELD)
                .apply(BeanTemplateUtils.build(BasicMessageBean.class)
                        .with(BasicMessageBean::details, ImmutableMap.of(VALUE_FIELD, 5.2)).done().get(), null)
                .details().get(LoggingMergeFunctions.MIN_FIELD), 5.2D, 0.0001D);
        assertEquals((double) LoggingMergeFunctions.minField(VALUE_FIELD)
                .apply(BeanTemplateUtils.build(BasicMessageBean.class)
                        .with(BasicMessageBean::details, ImmutableMap.of(VALUE_FIELD, 5.2)).done().get(),
                        BeanTemplateUtils.build(BasicMessageBean.class)
                                .with(BasicMessageBean::details, ImmutableMap.of(VALUE_FIELD, 4.66)).done().get())
                .details().get(LoggingMergeFunctions.MIN_FIELD), 4.66D, 0.0001D);

        //test max
        assertNull(LoggingMergeFunctions
                .minField(VALUE_FIELD).apply(BeanTemplateUtils.build(BasicMessageBean.class)
                        .with(BasicMessageBean::message, "msg1").done().get(), null)
                .details().get(LoggingMergeFunctions.MAX_FIELD));
        assertEquals((double) LoggingMergeFunctions.maxField(VALUE_FIELD)
                .apply(BeanTemplateUtils.build(BasicMessageBean.class)
                        .with(BasicMessageBean::details, ImmutableMap.of(VALUE_FIELD, 5.2)).done().get(), null)
                .details().get(LoggingMergeFunctions.MAX_FIELD), 5.2D, 0.0001D);
        assertEquals((double) LoggingMergeFunctions.maxField(VALUE_FIELD)
                .apply(BeanTemplateUtils.build(BasicMessageBean.class)
                        .with(BasicMessageBean::details, ImmutableMap.of(VALUE_FIELD, 5.2)).done().get(),
                        BeanTemplateUtils.build(BasicMessageBean.class)
                                .with(BasicMessageBean::details, ImmutableMap.of(VALUE_FIELD, 4.66)).done().get())
                .details().get(LoggingMergeFunctions.MAX_FIELD), 5.2D, 0.0001D);

        //test minmax
        final BasicMessageBean mm1 = LoggingMergeFunctions.minField(VALUE_FIELD).apply(BeanTemplateUtils
                .build(BasicMessageBean.class).with(BasicMessageBean::message, "msg1").done().get(), null);
        assertNull(mm1.details().get(LoggingMergeFunctions.MIN_FIELD));
        assertNull(mm1.details().get(LoggingMergeFunctions.MAX_FIELD));
        final BasicMessageBean mm2 = LoggingMergeFunctions
                .minMaxField(VALUE_FIELD).apply(
                        BeanTemplateUtils.build(BasicMessageBean.class)
                                .with(BasicMessageBean::details, ImmutableMap.of(VALUE_FIELD, 5.2)).done().get(),
                        null);
        assertEquals((double) mm2.details().get(LoggingMergeFunctions.MIN_FIELD), 5.2, 0.0001D);
        assertEquals((double) mm2.details().get(LoggingMergeFunctions.MAX_FIELD), 5.2, 0.0001D);
        final BasicMessageBean mm3 = LoggingMergeFunctions.minMaxField(VALUE_FIELD)
                .apply(BeanTemplateUtils.build(BasicMessageBean.class)
                        .with(BasicMessageBean::details, ImmutableMap.of(VALUE_FIELD, 5.2)).done().get(),
                        BeanTemplateUtils.build(BasicMessageBean.class)
                                .with(BasicMessageBean::details, ImmutableMap.of(VALUE_FIELD, 4.66)).done().get());
        assertEquals((double) mm3.details().get(LoggingMergeFunctions.MIN_FIELD), 4.66, 0.0001D);
        assertEquals((double) mm3.details().get(LoggingMergeFunctions.MAX_FIELD), 5.2, 0.0001D);
    }

    @SuppressWarnings("unchecked")
    @Test
    public void test_simpleLog() throws InterruptedException, ExecutionException {
        final String subsystem_name = "logging_test8";
        final int num_messages_to_log = 50;
        final DataBucketBean test_bucket = getTestBucket("test8", Optional.of(Level.ALL.toString()),
                Optional.empty());
        final IBucketLogger user_logger = logging_service.getLogger(test_bucket);
        final IBucketLogger system_logger = logging_service.getSystemLogger(test_bucket);
        final IBucketLogger external_logger = logging_service.getExternalLogger(subsystem_name);
        //log a few messages
        IntStream.rangeClosed(1, num_messages_to_log).boxed().forEach(i -> {
            user_logger.log(Level.ERROR, true, () -> "test message " + i, () -> subsystem_name);
            system_logger.log(Level.ERROR, true, () -> "test message " + i, () -> subsystem_name);
            external_logger.log(Level.ERROR, true, () -> "test message " + i, () -> subsystem_name);

            user_logger.log(Level.ERROR, true, () -> "test message " + i, () -> subsystem_name, () -> "command");
            system_logger.log(Level.ERROR, true, () -> "test message " + i, () -> subsystem_name, () -> "command");
            external_logger.log(Level.ERROR, true, () -> "test message " + i, () -> subsystem_name,
                    () -> "command");

            user_logger.log(Level.ERROR, true, () -> "test message " + i, () -> subsystem_name, () -> "command",
                    () -> 32);
            system_logger.log(Level.ERROR, true, () -> "test message " + i, () -> subsystem_name, () -> "command",
                    () -> 32);
            external_logger.log(Level.ERROR, true, () -> "test message " + i, () -> subsystem_name, () -> "command",
                    () -> 32);

            user_logger.log(Level.ERROR, true, () -> "test message " + i, () -> subsystem_name, () -> "command",
                    () -> 32, () -> ImmutableMap.of());
            system_logger.log(Level.ERROR, true, () -> "test message " + i, () -> subsystem_name, () -> "command",
                    () -> 32, () -> ImmutableMap.of());
            external_logger.log(Level.ERROR, true, () -> "test message " + i, () -> subsystem_name, () -> "command",
                    () -> 32, () -> ImmutableMap.of());

            //merge shorthand rules
            IBasicMessageBeanSupplier message = new BasicMessageBeanSupplier(true, () -> "test_message",
                    () -> "command", () -> null, () -> "test_message " + i, () -> ImmutableMap.of());

            user_logger.log(Level.ERROR, message, "key1", LoggingMergeFunctions.replaceMessage());
            system_logger.log(Level.ERROR, message, "key1", LoggingMergeFunctions.replaceMessage());
            external_logger.log(Level.ERROR, message, "key1", LoggingMergeFunctions.replaceMessage());

            user_logger.log(Level.ERROR, message, "key1", (b) -> b, LoggingMergeFunctions.replaceMessage());
            system_logger.log(Level.ERROR, message, "key1", (b) -> b, LoggingMergeFunctions.replaceMessage());
            external_logger.log(Level.ERROR, message, "key1", (b) -> b, LoggingMergeFunctions.replaceMessage());

            user_logger.log(Level.ERROR, message, "key1", Arrays.asList(LoggingRules.logEveryCount(1)),
                    LoggingMergeFunctions.replaceMessage());
            system_logger.log(Level.ERROR, message, "key1", Arrays.asList(LoggingRules.logEveryCount(1)),
                    LoggingMergeFunctions.replaceMessage());
            external_logger.log(Level.ERROR, message, "key1", Arrays.asList(LoggingRules.logEveryCount(1)),
                    LoggingMergeFunctions.replaceMessage());
        });

        user_logger.flush();
        system_logger.flush();
        external_logger.flush();

        //check its in ES, wait 10s max for the index to refresh
        final DataBucketBean logging_test_bucket = BucketUtils.convertDataBucketBeanToLogging(test_bucket);
        final IDataWriteService<BasicMessageBean> logging_crud = search_index_service.getDataService().get()
                .getWritableDataService(BasicMessageBean.class, logging_test_bucket, Optional.empty(),
                        Optional.empty())
                .get();
        waitForResults(logging_crud, 10, num_messages_to_log * 14);
        assertEquals(num_messages_to_log * 14, logging_crud.countObjects().get().longValue());

        final DataBucketBean logging_external_test_bucket = BucketUtils
                .convertDataBucketBeanToLogging(BeanTemplateUtils.clone(test_bucket)
                        .with(DataBucketBean::full_name, "/external/" + subsystem_name + "/").done());
        final IDataWriteService<BasicMessageBean> logging_crud_external = search_index_service.getDataService()
                .get().getWritableDataService(BasicMessageBean.class, logging_external_test_bucket,
                        Optional.empty(), Optional.empty())
                .get();
        waitForResults(logging_crud_external, 10, num_messages_to_log * 7);
        assertEquals(num_messages_to_log * 7, logging_crud_external.countObjects().get().longValue());

        //cleanup
        logging_crud.deleteDatastore().get();
    }

    private class TestAppender extends AbstractAppender {
        static final long serialVersionUID = 3427100436050801263L;
        public long message_count = 0;

        /**
         * @param name
         * @param filter
         * @param layout
         */
        protected TestAppender(String name, Filter filter, Layout<? extends Serializable> layout) {
            super(name, filter, layout);
        }

        @Override
        public void append(LogEvent arg0) {
            message_count++;
        }

    }

    /**
     * Waits for the crud service count objects to return some amount of objects w/in the given
     * timeframe, returns as soon as we find any results.  Useful for waiting for ES to flush/update the index. 
     * 
     * @param crud_service
     * @param max_wait_time_s
     */
    private static void waitForResults(final IDataWriteService<?> crud_service, final long max_wait_time_s,
            final long expected_num_results) {
        for (int ii = 0; ii < max_wait_time_s; ++ii) {
            try {
                Thread.sleep(1000L);
            } catch (Exception e) {
            }
            if (crud_service.countObjects().join().intValue() >= expected_num_results)
                break;
        }
    }

    /**
     * @param name
     * @param override_good_single
     * @return
     */
    private DataBucketBean getTestBucket(final String name, final Optional<String> min_log_level,
            final Optional<Map<String, String>> overrides) {
        return BeanTemplateUtils.build(DataBucketBean.class)
                .with(DataBucketBean::full_name, "/test/logtest/" + name + "/")
                .with(DataBucketBean::data_schema, BeanTemplateUtils.build(DataSchemaBean.class)
                        .with(DataSchemaBean::search_index_schema,
                                BeanTemplateUtils.build(SearchIndexSchemaBean.class)
                                        .with(SearchIndexSchemaBean::enabled, true).done().get())
                        .done().get())
                .with(DataBucketBean::management_schema, BeanTemplateUtils.build(ManagementSchemaBean.class)
                        .with(ManagementSchemaBean::logging_schema, BeanTemplateUtils.build(LoggingSchemaBean.class)
                                .with(LoggingSchemaBean::log_level, min_log_level.orElse(Level.OFF.toString()))
                                .with(LoggingSchemaBean::log_level_overrides, overrides.orElse(ImmutableMap.of()))
                                .done().get())
                        .done().get())
                .done().get();
    }

    /**
     * Creates a sample bucket without a management schema.
     * 
     * @param name
     * @return
     */
    private DataBucketBean getEmptyTestBucket(final String name) {
        return BeanTemplateUtils.build(DataBucketBean.class)
                .with(DataBucketBean::full_name, "/test/logtest/" + name + "/")
                .with(DataBucketBean::data_schema, BeanTemplateUtils.build(DataSchemaBean.class)
                        .with(DataSchemaBean::search_index_schema,
                                BeanTemplateUtils.build(SearchIndexSchemaBean.class)
                                        .with(SearchIndexSchemaBean::enabled, true).done().get())
                        .done().get())
                .done().get();
    }
}