com.auditbucket.test.functional.TestAuditIntegration.java Source code

Java tutorial

Introduction

Here is the source code for com.auditbucket.test.functional.TestAuditIntegration.java

Source

/*
 * Copyright (c) 2012-2013 "Monowai Developments Limited"
 *
 * This file is part of AuditBucket.
 *
 * AuditBucket is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * AuditBucket is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with AuditBucket.  If not, see <http://www.gnu.org/licenses/>.
 */

package com.auditbucket.test.functional;

import com.auditbucket.client.AbRestClient;
import com.auditbucket.engine.endpoint.TrackEP;
import com.auditbucket.engine.service.MediationFacade;
import com.auditbucket.engine.service.TrackService;
import com.auditbucket.engine.service.WhatService;
import com.auditbucket.helper.DatagioException;
import com.auditbucket.registration.bean.FortressInputBean;
import com.auditbucket.registration.bean.RegistrationBean;
import com.auditbucket.registration.bean.TagInputBean;
import com.auditbucket.registration.endpoint.TagEP;
import com.auditbucket.registration.model.Fortress;
import com.auditbucket.registration.model.SystemUser;
import com.auditbucket.registration.service.FortressService;
import com.auditbucket.registration.service.RegistrationService;
import com.auditbucket.search.model.MetaSearchSchema;
import com.auditbucket.search.model.QueryParams;
import com.auditbucket.track.bean.*;
import com.auditbucket.track.model.MetaHeader;
import com.auditbucket.track.model.TrackLog;
import com.auditbucket.track.model.TrackTag;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.JestResult;
import io.searchbox.client.config.HttpClientConfig;
import io.searchbox.core.Search;
import io.searchbox.indices.DeleteIndex;
import org.apache.commons.lang.time.StopWatch;
import org.joda.time.DateTime;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.neo4j.support.Neo4jTemplate;
import org.springframework.data.neo4j.support.node.Neo4jHelper;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.converter.StringHttpMessageConverter;
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.test.annotation.Rollback;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.transaction.BeforeTransaction;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.HttpServerErrorException;
import org.springframework.web.client.RestTemplate;

import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Set;

import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertSame;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assume.assumeTrue;
import static org.springframework.test.util.AssertionErrors.assertTrue;

/**
 * User: nabil
 * Date: 16/07/13
 * Time: 22:51
 * To change this template use File | Settings | File Templates.
 */
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration("classpath:root-context.xml")
public class TestAuditIntegration {
    private boolean ignoreMe = false;
    private static int fortressMax = 1;
    private static JestClient esClient;

    @Autowired
    TrackService trackService;
    @Autowired
    TrackEP trackEP;

    @Autowired
    RegistrationService regService;
    @Autowired
    FortressService fortressService;
    @Autowired
    MediationFacade mediationFacade;

    @Autowired
    WhatService whatService;

    @Autowired
    private Neo4jTemplate template;

    private Logger logger = LoggerFactory.getLogger(TestAuditIntegration.class);
    private String email = "test@ab.com";
    private Authentication authA = new UsernamePasswordAuthenticationToken(email, "user1");

    @BeforeClass
    @Rollback(false)
    public static void cleanupElasticSearch() throws Exception {
        HttpClientConfig clientConfig = new HttpClientConfig.Builder("http://localhost:9201").multiThreaded(false)
                .build();
        // Construct a new Jest client according to configuration via factory
        JestClientFactory factory = new JestClientFactory();
        factory.setHttpClientConfig(clientConfig);
        //factory.setClientConfig(clientConfig);
        esClient = factory.getObject();

        esClient.execute(new DeleteIndex.Builder("ab.testaudit.suppress").build());
        esClient.execute(new DeleteIndex.Builder("ab.testaudit.ngram").build());
        esClient.execute(new DeleteIndex.Builder("ab.companywithspace.audittest").build());
        esClient.execute(new DeleteIndex.Builder("ab.monowai.trackgraph").build());
        esClient.execute(new DeleteIndex.Builder("ab.monowai.audittest").build());

        for (int i = 1; i < fortressMax + 1; i++) {
            esClient.execute(new DeleteIndex.Builder("ab.testaudit.bulkloada" + i).build());
        }

    }

    @AfterClass
    public static void shutDownElasticSearch() throws Exception {
        esClient.shutdownClient();
    }

    @Rollback(false)
    @BeforeTransaction
    public void cleanUpGraph() throws Exception {
        // This will fail if running over REST. Haven't figured out how to use a view to look at the embedded db
        // See: https://github.com/SpringSource/spring-data-neo4j/blob/master/spring-data-neo4j-examples/todos/src/main/resources/META-INF/spring/applicationContext-graph.xml

        SecurityContextHolder.getContext().setAuthentication(authA);
        if ("rest".equals(System.getProperty("neo4j")))
            return;
        Neo4jHelper.cleanDb(template);

    }

    @Test
    public void companyAndFortressWithSpaces() throws Exception {
        assumeTrue(!ignoreMe);
        SecurityContextHolder.getContext().setAuthentication(authA);
        regService.registerSystemUser(new RegistrationBean("Company With Space", email, "bah"));
        Thread.sleep(1000);
        Fortress fortressA = fortressService.registerFortress(new FortressInputBean("Audit Test", false));
        String docType = "TestAuditX";
        String callerRef = "ABC123X";
        MetaInputBean inputBean = new MetaInputBean(fortressA.getName(), "wally", docType, new DateTime(),
                callerRef);

        String ahKey = mediationFacade.createHeader(inputBean, null).getMetaKey();
        assertNotNull(ahKey);
        MetaHeader header = trackService.getHeader(ahKey);
        mediationFacade.processLog(new LogInputBean(ahKey, "wally", new DateTime(), "{\"blah\":" + 1 + "}"));
        Thread.sleep(2000);
        doEsQuery(header.getIndexName(), header.getMetaKey());
    }

    @Test
    public void headerWithTagsProcess() throws Exception {
        assumeTrue(!ignoreMe);
        SecurityContextHolder.getContext().setAuthentication(authA);
        String company = "Monowai";
        String apiKey = regService.registerSystemUser(new RegistrationBean(company, email, "bah")).getCompany()
                .getApiKey();
        Fortress fo = fortressService.registerFortress(new FortressInputBean("headerWithTagsProcess", false));
        DateTime now = new DateTime();
        MetaInputBean inputBean = new MetaInputBean(fo.getName(), "wally", "TestTrack", now, "ZZZ123");
        inputBean.setTag(new TagInputBean("testTagNameZZ", "someAuditRLX"));
        inputBean.setEvent("TagTest");
        TrackResultBean auditResult;
        auditResult = trackEP.trackHeader(inputBean, apiKey, apiKey).getBody();
        Thread.sleep(4000);
        TrackedSummaryBean summary = trackEP.getAuditSummary(auditResult.getMetaKey(), apiKey, apiKey).getBody();
        assertNotNull(summary);
        // Check we can find the Event in ElasticSearch
        doEsQuery(summary.getHeader().getIndexName(), inputBean.getEvent(), 1);
        // Can we find the Tag
        doEsQuery(summary.getHeader().getIndexName(), "testTagNameZZ", 1);

    }

    @Test
    public void immutableHeadersWithNoLogsAreIndexed() throws Exception {
        assumeTrue(!ignoreMe);
        SecurityContextHolder.getContext().setAuthentication(authA);
        String company = "Monowai";
        regService.registerSystemUser(new RegistrationBean(company, email, "bah"));
        Fortress fo = fortressService
                .registerFortress(new FortressInputBean("immutableHeadersWithNoLogsAreIndexed", false));
        DateTime now = new DateTime();
        MetaInputBean inputBean = new MetaInputBean(fo.getName(), "wally", "TestTrack", now, "ZZZ123");
        inputBean.setEvent("immutableHeadersWithNoLogsAreIndexed");
        TrackResultBean auditResult;
        auditResult = mediationFacade.createHeader(inputBean, null);
        Thread.sleep(4000);
        TrackedSummaryBean summary = mediationFacade.getTrackedSummary(auditResult.getMetaKey());
        assertNotNull(summary);
        assertSame("change logs were not expected", 0, summary.getChanges().size());
        assertNotNull("Search record not received", summary.getHeader().getSearchKey());
        // Check we can find the Event in ElasticSearch
        doEsQuery(summary.getHeader().getIndexName(), inputBean.getEvent(), 1);

        // No Event, so should not be in elasticsearch
        inputBean = new MetaInputBean(fo.getName(), "wally", "TestTrack", now, "ZZZ999");
        auditResult = mediationFacade.createHeader(inputBean, null);
        summary = mediationFacade.getTrackedSummary(auditResult.getMetaKey());
        assertNotNull(summary);
        assertSame("Not change logs were expected", 0, summary.getChanges().size());
        assertNull(summary.getHeader().getSearchKey());
        // Check we can find the Event in ElasticSearch
        doEsQuery(summary.getHeader().getIndexName(), "ZZZ999", 0);
    }

    @Test
    public void createHeaderTimeLogsWithSearchActivated() throws Exception {
        assumeTrue(!ignoreMe);
        int max = 3;
        String ahKey;
        logger.info("createHeaderTimeLogsWithSearchActivated started");
        SecurityContextHolder.getContext().setAuthentication(authA);
        String company = "Monowai";
        regService.registerSystemUser(new RegistrationBean(company, email, "bah"));
        Fortress fo = fortressService.registerFortress(new FortressInputBean("111", false));

        MetaInputBean inputBean = new MetaInputBean(fo.getName(), "wally", "TestTrack", new DateTime(), "ABC123");
        TrackResultBean auditResult;
        auditResult = mediationFacade.createHeader(inputBean, null);
        ahKey = auditResult.getMetaKey();

        assertNotNull(ahKey);

        MetaHeader metaHeader = trackService.getHeader(ahKey);
        assertNotNull(metaHeader);
        assertNotNull(trackService.findByCallerRef(fo, "TestTrack", "ABC123"));
        assertNotNull(fortressService.getFortressUser(fo, "wally", true));
        assertNull(fortressService.getFortressUser(fo, "wallyz", false));

        int i = 0;

        StopWatch watch = new StopWatch();
        logger.info("Start-");
        watch.start();
        while (i < max) {
            mediationFacade.processLog(new LogInputBean(ahKey, "wally", new DateTime(), "{\"blah\":" + i + "}"));
            i++;
        }
        watch.stop();
        Thread.sleep(8000);
        // Test that we get the expected number of log events
        if (!"rest".equals(System.getProperty("neo4j"))) // Don't check if running over rest
            assertEquals("This will fail if the DB is not cleared down, i.e. testing over REST", max,
                    trackService.getLogCount(ahKey));

        // Putting asserts On elasticsearch
        String query = "{" + "   \"query\": {  " + "\"query_string\" : { " + " \"default_field\" :\""
                + MetaSearchSchema.WHAT + ".blah\", " + " \"query\" :\"*\" " + "}  " + "}  " + "}";
        Search search = new Search.Builder(query).addIndex(metaHeader.getIndexName()).build();

        JestResult result = esClient.execute(search);
        assertNotNull(result);
        assertNotNull(result.getJsonObject());
        assertNotNull(result.getJsonObject().getAsJsonObject("hits"));
        assertNotNull(result.getJsonObject().getAsJsonObject("hits").get("total"));
        int nbrResult = result.getJsonObject().getAsJsonObject("hits").get("total").getAsInt();
        // Only the last change is indexed, so there should be one result
        Assert.assertEquals(1, nbrResult);

    }

    @Test
    public void auditsByPassGraphByCallerRef() throws Exception {
        assumeTrue(!ignoreMe);
        logger.info("auditsByPassGraphByCallerRef started");
        SecurityContextHolder.getContext().setAuthentication(authA);
        String company = "Monowai";
        regService.registerSystemUser(new RegistrationBean(company, email, "bah"));
        Fortress fortress = fortressService.registerFortress(new FortressInputBean("TrackGraph", false));

        MetaInputBean inputBean = new MetaInputBean(fortress.getName(), "wally", "TestTrack", new DateTime(),
                "ABC123");
        inputBean.setTrackSuppressed(true);
        mediationFacade.createHeader(inputBean, null);

        String indexName = MetaSearchSchema.parseIndex(fortress);

        // Putting asserts On elasticsearch
        Thread.sleep(2000); // Let the messaging take effect
        doEsQuery(indexName, "*", 1);
        inputBean = new MetaInputBean(fortress.getName(), "wally", "TestTrack", new DateTime(), "ABC124");
        inputBean.setTrackSuppressed(true);
        mediationFacade.createHeader(inputBean, null);
        Thread.sleep(2000); // Let the messaging take effect
        doEsQuery(indexName, "*", 2);

        inputBean = new MetaInputBean(fortress.getName(), "wally", "TestTrack", new DateTime(), "ABC124");
        inputBean.setTrackSuppressed(true);
        mediationFacade.createHeader(inputBean, null);
        Thread.sleep(2000); // Let the messaging take effect
        // Updating the same caller ref should not create a 3rd record
        doEsQuery(indexName, "*", 2);

        inputBean = new MetaInputBean(fortress.getName(), "wally", "TestTrack", new DateTime(), "abc124");
        inputBean.setTrackSuppressed(true);
        mediationFacade.createHeader(inputBean, null);
        Thread.sleep(2000); // Let the messaging take effect
        // Updating the same caller ref should not create a 3rd record
        doEsQuery(indexName, "*", 2);

        inputBean = new MetaInputBean(fortress.getName(), "wally", "TestTrack", new DateTime(), "abc125");
        inputBean.setTrackSuppressed(true);
        mediationFacade.createHeader(inputBean, null);
        Thread.sleep(2000); // Let the messaging take effect
        // Updating the same caller ref should not create a 3rd record
        doEsQuery(indexName, "*", 3);

    }

    /**
     * Suppresses the indexing of a log record even if the fortress is set to index everything
     *
     * @throws Exception
     */
    @Test
    public void suppressIndexingOnDemand() throws Exception {
        assumeTrue(!ignoreMe);

        String escJson = "{\"who\":";
        SecurityContextHolder.getContext().setAuthentication(authA);
        regService.registerSystemUser(new RegistrationBean("TestTrack", email, "bah"));
        Fortress iFortress = fortressService.registerFortress(new FortressInputBean("suppress", false));
        MetaInputBean inputBean = new MetaInputBean(iFortress.getName(), "olivia@sunnybell.com", "CompanyNode",
                new DateTime());

        //Transaction tx = getTransaction();
        TrackResultBean indexedResult = mediationFacade.createHeader(inputBean, null);
        MetaHeader indexHeader = trackService.getHeader(indexedResult.getMetaKey());

        LogResultBean resultBean = mediationFacade.processLog(new LogInputBean(indexHeader.getMetaKey(),
                inputBean.getFortressUser(), new DateTime(), escJson + "\"andy\"}"));
        junit.framework.Assert.assertNotNull(resultBean);

        waitForHeaderToUpdate(indexHeader);
        Thread.sleep(1000);
        String indexName = indexHeader.getIndexName();

        doEsQuery(indexName, "andy");

        inputBean = new MetaInputBean(iFortress.getName(), "olivia@sunnybell.com", "CompanyNode", new DateTime());
        inputBean.setSearchSuppressed(true);
        TrackResultBean noIndex = mediationFacade.createHeader(inputBean, null);
        MetaHeader noIndexHeader = trackService.getHeader(noIndex.getMetaKey());

        mediationFacade.processLog(new LogInputBean(noIndexHeader.getMetaKey(), inputBean.getFortressUser(),
                new DateTime(), escJson + "\"bob\"}"));
        Thread.sleep(1000);
        // Bob's not there because we said we didn't want to index that header
        doEsQuery(indexName, "bob", 0);
        doEsQuery(indexName, "andy");
    }

    @Autowired
    TagEP tagEP;

    @Test
    public void tagKeyReturnsUniqueResult() throws Exception {
        assumeTrue(!ignoreMe);

        String escJson = "{\"who\":";
        SecurityContextHolder.getContext().setAuthentication(authA);
        regService.registerSystemUser(new RegistrationBean("TestTrack", email, "bah"));
        Fortress iFortress = fortressService.registerFortress(new FortressInputBean("suppress", false));
        MetaInputBean metaInput = new MetaInputBean(iFortress.getName(), "olivia@sunnybell.com", "CompanyNode",
                new DateTime());
        String tagSearch = "example"; // Relationship names is indexed as @tag.relationship.key in ES
        TagInputBean tag = new TagInputBean("Key Test Works", tagSearch);
        metaInput.setTag(tag);

        TrackResultBean indexedResult = mediationFacade.createHeader(metaInput, null);
        MetaHeader indexHeader = trackService.getHeader(indexedResult.getMetaKey());

        Set<TrackTag> tags = trackEP.getAuditTags(indexHeader.getMetaKey(), null, null).getBody();
        assertNotNull(tags);
        assertEquals(1, tags.size());

        LogResultBean resultBean = mediationFacade.processLog(new LogInputBean(indexHeader.getMetaKey(),
                metaInput.getFortressUser(), new DateTime(), escJson + "\"andy\"}"));
        assertNotNull(resultBean);
        Thread.sleep(2000);

        waitForHeaderToUpdate(indexHeader);
        String indexName = indexHeader.getIndexName();
        doEsFieldQuery(indexName, "@tag." + tagSearch + ".key", "keytestworks", 1);

    }

    @Test
    public void testWhatIndexingDefaultAttributeWithNGram() throws Exception {
        assumeTrue(!ignoreMe);
        SecurityContextHolder.getContext().setAuthentication(authA);
        regService.registerSystemUser(new RegistrationBean("TestTrack", email, "bah"));
        Fortress iFortress = fortressService.registerFortress(new FortressInputBean("ngram", false));
        MetaInputBean inputBean = new MetaInputBean(iFortress.getName(), "olivia@sunnybell.com", "CompanyNode",
                new DateTime());

        TrackResultBean indexedResult = mediationFacade.createHeader(inputBean, null);
        MetaHeader indexHeader = trackService.getHeader(indexedResult.getMetaKey());
        String what = "{\"code\":\"AZERTY\",\"name\":\"NameText\",\"description\":\"this is a description\"}";
        mediationFacade.processLog(
                new LogInputBean(indexHeader.getMetaKey(), inputBean.getFortressUser(), new DateTime(), what));
        waitForHeaderToUpdate(indexHeader);
        String indexName = indexHeader.getIndexName();
        Thread.sleep(1000);

        doEsTermQuery(indexName, MetaSearchSchema.WHAT + "." + MetaSearchSchema.WHAT_DESCRIPTION, "des", 1);
        doEsTermQuery(indexName, MetaSearchSchema.WHAT + "." + MetaSearchSchema.WHAT_DESCRIPTION, "de", 0);
        doEsTermQuery(indexName, MetaSearchSchema.WHAT + "." + MetaSearchSchema.WHAT_DESCRIPTION, "descripti", 1);
        doEsTermQuery(indexName, MetaSearchSchema.WHAT + "." + MetaSearchSchema.WHAT_DESCRIPTION, "descriptio", 1);
        doEsTermQuery(indexName, MetaSearchSchema.WHAT + "." + MetaSearchSchema.WHAT_DESCRIPTION, "description", 0);
        doEsTermQuery(indexName, MetaSearchSchema.WHAT + "." + MetaSearchSchema.WHAT_DESCRIPTION, "is is a de", 1);
        doEsTermQuery(indexName, MetaSearchSchema.WHAT + "." + MetaSearchSchema.WHAT_DESCRIPTION, "is is a des", 0);

        doEsTermQuery(indexName, MetaSearchSchema.WHAT + "." + MetaSearchSchema.WHAT_NAME, "Name", 1);
        doEsTermQuery(indexName, MetaSearchSchema.WHAT + "." + MetaSearchSchema.WHAT_NAME, "Nam", 1);
        doEsTermQuery(indexName, MetaSearchSchema.WHAT + "." + MetaSearchSchema.WHAT_NAME, "NameText", 1);

        doEsTermQuery(indexName, MetaSearchSchema.WHAT + "." + MetaSearchSchema.WHAT_CODE, "AZ", 1);
        doEsTermQuery(indexName, MetaSearchSchema.WHAT + "." + MetaSearchSchema.WHAT_CODE, "AZER", 1);
        doEsTermQuery(indexName, MetaSearchSchema.WHAT + "." + MetaSearchSchema.WHAT_CODE, "AZERTY", 0);

    }

    @Test
    public void stressWithHighVolume() throws Exception {
        assumeTrue(!ignoreMe);
        logger.info("stressWithHighVolume started");
        SecurityContextHolder.getContext().setAuthentication(authA);
        //Neo4jHelper.cleanDb(graphDatabaseService, true);
        regService.registerSystemUser(new RegistrationBean("TestTrack", email, "bah"));

        int auditMax = 10;
        int logMax = 10;
        int fortress = 1;
        //fortressMax = 10;
        String simpleJson = "{\"who\":";
        ArrayList<Long> list = new ArrayList<>();

        logger.info("FortressCount: " + fortressMax + " AuditCount: " + auditMax + " LogCount: " + logMax);
        logger.info(
                "We will be expecting a total of " + (auditMax * logMax * fortressMax) + " messages to be handled");

        StopWatch watch = new StopWatch();
        watch.start();
        double splitTotals = 0;
        long totalRows = 0;
        int auditSleepCount; // Discount all the time we spent sleeping

        DecimalFormat f = new DecimalFormat("##.000");

        while (fortress <= fortressMax) {
            String fortressName = "bulkloada" + fortress;
            int audit = 1;
            long requests = 0;
            auditSleepCount = 0;

            Fortress iFortress = fortressService.registerFortress(new FortressInputBean(fortressName, false));
            requests++;
            logger.info("Starting run for " + fortressName);
            while (audit <= auditMax) {
                boolean searchChecked = false;
                MetaInputBean aib = new MetaInputBean(iFortress.getName(), fortress + "olivia@sunnybell.com",
                        "CompanyNode", new DateTime(), "ABC" + audit);
                TrackResultBean arb = mediationFacade.createHeader(aib, null);
                requests++;
                int log = 1;
                while (log <= logMax) {
                    //String escJson = Helper.getBigJsonText(log);
                    //trackService.createLog(new LogInputBean(arb.getMetaKey(), aib.getFortressUser(), new DateTime(), escJson ));

                    createLog(simpleJson, aib, arb, log);
                    //Thread.sleep(100);
                    requests++;
                    if (!searchChecked) {
                        searchChecked = true;
                        MetaHeader metaHeader = trackService.getHeader(arb.getMetaKey());
                        requests++;
                        int checkCount = waitForHeaderToUpdate(metaHeader);
                        auditSleepCount = auditSleepCount + (400 * checkCount);
                        requests = requests + checkCount;
                    } // searchCheck done
                    log++;
                } // Logs created
                audit++;
            } // Audit headers finished with
            watch.split();
            double fortressRunTime = (watch.getSplitTime() - auditSleepCount) / 1000d;
            logger.info("*** " + iFortress.getName() + " took " + fortressRunTime + "  avg processing time for ["
                    + requests + "] RPS= " + f.format(fortressRunTime / requests) + ". Requests per second "
                    + f.format(requests / fortressRunTime));

            splitTotals = splitTotals + fortressRunTime;
            totalRows = totalRows + requests;
            watch.reset();
            watch.start();
            list.add(iFortress.getId());
            fortress++;
        }

        logger.info("*** Created data set in " + f.format(splitTotals) + " fortress avg = "
                + f.format(splitTotals / fortressMax) + " avg processing time per request "
                + f.format(splitTotals / totalRows) + ". Requests per second " + f.format(totalRows / splitTotals));
        watch.reset();
        Thread.sleep(5000); // give things a final chance to complete

        validateLogsIndexed(list, auditMax, logMax);
        doSearchTests(auditMax, list, watch);
    }

    @Test
    public void simpleQueryEPWorksForImportedRecord() throws Exception {
        assumeTrue(!ignoreMe);

        String searchFor = "testing";
        String escJson = "{\"who\":\"" + searchFor + "\"}";

        //        RestTemplate restTemplate = new RestTemplate();
        //        restTemplate.getMessageConverters().add(new StringHttpMessageConverter());
        //        AbRestClient restClient = new AbRestClient("http://localhost:9081/", "mike", "123", 1);
        //        assertEquals("Pong!", restClient.ping());
        SystemUser su = regService.registerSystemUser(new RegistrationBean("TestCo", "mike", "123"));
        Fortress fortress = fortressService.registerFortress(su.getCompany(),
                new FortressInputBean("TestFortress", false));
        Thread.sleep(300);

        LogInputBean log = new LogInputBean("mikeTest", new DateTime(), escJson);
        MetaInputBean input = new MetaInputBean("TestFortress", "mikeTest", "Query", new DateTime(), "abzz");
        input.setLog(log);

        TrackResultBean result = trackEP
                .trackHeader(input, su.getCompany().getApiKey(), su.getCompany().getApiKey()).getBody();
        waitForHeaderToUpdate(result.getMetaHeader(), su.getCompany().getApiKey());
        //        restClient.writeAudit(input, "Hello World");

        QueryParams q = new QueryParams().setSimpleQuery(searchFor);
        q.setCompany(su.getCompany().getName());
        q.setFortress(fortress.getName());
        doEsQuery("*", searchFor, 1);

        String qResult = runQuery(q);
        assertNotNull(qResult);
        assertTrue("Couldn't find a hit in the result [" + result + "]", qResult.contains("total\" : 1"));

    }

    private String runQuery(QueryParams queryParams) throws Exception {
        RestTemplate restTemplate = new RestTemplate();
        restTemplate.getMessageConverters().add(new StringHttpMessageConverter());

        HttpHeaders httpHeaders = AbRestClient.getHeaders("mike", "123");
        HttpEntity<QueryParams> requestEntity = new HttpEntity<>(queryParams, httpHeaders);

        try {
            return restTemplate.exchange("http://localhost:9081/ab-search/v1/query/", HttpMethod.POST,
                    requestEntity, String.class).getBody();
        } catch (HttpClientErrorException e) {
            logger.error("AB Client Audit error {}", e);
        } catch (HttpServerErrorException e) {
            logger.error("AB Server Audit error {}", e);

        }
        return null;
    }

    private void createLog(String simpleJson, MetaInputBean aib, TrackResultBean arb, int log)
            throws DatagioException {
        mediationFacade.processLog(
                new LogInputBean(arb.getMetaKey(), aib.getFortressUser(), new DateTime(), simpleJson + log + "}"));
    }

    private void validateLogsIndexed(ArrayList<Long> list, int auditMax, int expectedLogCount) throws Exception {
        logger.info("Validating logs are indexed");
        int fortress = 2;
        int audit = 1;
        //DecimalFormat f = new DecimalFormat("##.000");
        while (fortress <= fortressMax) {
            while (audit <= auditMax) {
                MetaHeader header = trackService.findByCallerRefFull(list.get(fortress), "CompanyNode",
                        "ABC" + audit);
                StopWatch watch = new StopWatch();
                watch.start();
                Set<TrackLog> logs = trackService.getLogs(header.getId());
                watch.split();
                assertNotNull(logs);
                //logger.info("retrieved [{}] logs in [{}] millis", logs.size(), f.format(watch.getSplitTime()));
                assertEquals("Wrong number of logs returned", expectedLogCount, logs.size());
                for (TrackLog log : logs) {
                    assertEquals("logId [" + log.getId() + "] changeId[" + log.getChange().getId() + "], event[ "
                            + log.getChange().getEvent() + "]", true, log.isIndexed());
                }

                audit++;
            } // Audit headers finished with
            fortress++;
        }

    }

    private int waitForHeaderToUpdate(MetaHeader header, String apiKey) throws Exception {
        // Looking for the first searchKey to be logged against the metaHeader
        int i = 0;
        int timeout = 50;

        MetaHeader metaHeader = trackEP.getMetaHeader(header.getMetaKey(), apiKey, apiKey).getBody();
        if (metaHeader.getSearchKey() != null)
            return 0;
        while (metaHeader.getSearchKey() == null && i <= timeout) {
            metaHeader = trackEP.getMetaHeader(header.getMetaKey(), apiKey, apiKey).getBody();
            Thread.sleep(400);
            i++;
        }
        if (i > 10)
            logger.info("Wait for search got to [{}] for metaId [{}]", i, metaHeader.getId());
        boolean searchWorking = metaHeader.getSearchKey() != null;
        assertTrue("Search reply not received from ab-search", searchWorking);
        return i;
    }

    private int waitForHeaderToUpdate(MetaHeader header) throws Exception {
        return waitForHeaderToUpdate(header, null);
    }

    private void doSearchTests(int auditCount, ArrayList<Long> list, StopWatch watch) throws Exception {
        int fortress;
        int searchLoops = 200;
        int search = 0;
        int totalSearchRequests = 0;

        watch.start();

        do {
            fortress = 0;
            do {
                int x = 1;
                do {
                    int random = (int) (Math.random() * ((auditCount) + 1));
                    if (random == 0)
                        random = 1;

                    MetaHeader header = trackService.findByCallerRefFull(list.get(fortress), "CompanyNode",
                            "ABC" + random);
                    assertNotNull("ABC" + random, header);
                    assertNotNull("Looks like ab-search is not sending back results", header.getSearchKey());
                    //TrackLog when = trackService.getLastAuditLog(header);
                    TrackLog trackLog = trackService.getLastLog(header);
                    assertNotNull(trackLog);

                    //logger.info(header.getMetaKey() + " - " + when);
                    assertTrue("fortress " + fortress + " run " + x + " header " + header.getMetaKey() + " - "
                            + trackLog.getId(), trackLog.isIndexed());
                    String result = doEsFieldQuery(header.getIndexName(), MetaSearchSchema.META_KEY,
                            header.getMetaKey(), 1);
                    totalSearchRequests++;
                    validateResultFieds(result);

                    x++;
                } while (x < auditCount);
                fortress++;
            } while (fortress < fortressMax);
            search++;
        } while (search < searchLoops);

        watch.stop();
        double end = watch.getTime() / 1000d;
        logger.info("Total Search Requests = " + totalSearchRequests + ". Total time for searches " + end
                + " avg requests per second = " + totalSearchRequests / end);
    }

    private ObjectMapper objectMapper = new ObjectMapper();

    private void validateResultFieds(String result) throws Exception {
        JsonNode node = objectMapper.readTree(result);

        assertNotNull(node.get(MetaSearchSchema.CREATED));
        assertNotNull(node.get(MetaSearchSchema.WHO));
        assertNotNull(node.get(MetaSearchSchema.WHEN));
        assertNotNull(node.get(MetaSearchSchema.META_KEY));
        assertNotNull(node.get(MetaSearchSchema.DOC_TYPE));
        assertNotNull(node.get(MetaSearchSchema.FORTRESS));

    }

    private String doEsQuery(String index, String queryString) throws Exception {
        return doEsQuery(index, queryString, 1);
    }

    private String doEsQuery(String index, String queryString, int expectedHitCount) throws Exception {
        // There should only ever be one document for a given AuditKey.
        // Let's assert that
        String query = "{\n" + "    query: {\n" + "          query_string : {\n" + "              \"query\" : \""
                + queryString + "\"" + "           }\n" + "      }\n" + "}";

        logger.info("searching index [{}] for [{}]", index, queryString);
        Search search = new Search.Builder(query).addIndex(index).build();

        JestResult result = esClient.execute(search);
        assertNotNull(result);
        assertNotNull(result.getErrorMessage(), result.getJsonObject());
        assertNotNull(result.getErrorMessage(), result.getJsonObject().getAsJsonObject("hits"));
        assertNotNull(result.getErrorMessage(), result.getJsonObject().getAsJsonObject("hits").get("total"));
        int nbrResult = result.getJsonObject().getAsJsonObject("hits").get("total").getAsInt();
        Assert.assertEquals(index + "\r\n" + result.getJsonString(), expectedHitCount, nbrResult);
        return null;

        //return result.getJsonString();
    }

    private String doEsTermQuery(String index, String field, String queryString, int expectedHitCount)
            throws Exception {
        // There should only ever be one document for a given AuditKey.
        // Let's assert that
        String query = "{\n" + "    query: {\n" + "          term : {\n" + "              \"" + field + "\" : \""
                + queryString + "\"\n" + "           }\n" + "      }\n" + "}";
        Search search = new Search.Builder(query).addIndex(index).build();

        JestResult result = esClient.execute(search);
        String message = index + " - " + field + " - " + queryString
                + (result == null ? "[noresult]" : "\r\n" + result.getJsonString());
        assertNotNull(message, result);
        assertNotNull(message, result.getJsonObject());
        assertNotNull(message, result.getJsonObject().getAsJsonObject("hits"));
        assertNotNull(message, result.getJsonObject().getAsJsonObject("hits").get("total"));
        int nbrResult = result.getJsonObject().getAsJsonObject("hits").get("total").getAsInt();
        logger.info("searching index [{}] for [{}]", index, queryString);
        Assert.assertEquals(result.getJsonString(), expectedHitCount, nbrResult);
        if (nbrResult != 0) {
            return result.getJsonObject().getAsJsonObject("hits").getAsJsonArray("hits").getAsJsonArray().iterator()
                    .next().getAsJsonObject().get("_source").toString();
        } else {
            return null;
        }
    }

    private String doEsFieldQuery(String index, String field, String queryString, int expectedHitCount)
            throws Exception {
        // There should only ever be one document for a given AuditKey.
        // Let's assert that
        String query = "{\n" + "    query: {\n" + "          query_string : {\n"
                + "              \"default_field\" : \"" + field + "\",\n" + "              \"query\" : \""
                + queryString + "\"\n" + "           }\n" + "      }\n" + "}";
        Search search = new Search.Builder(query).addIndex(index).build();

        JestResult result = esClient.execute(search);
        String message = index + " - " + field + " - " + queryString
                + (result == null ? "[noresult]" : "\r\n" + result.getJsonString());
        assertNotNull(message, result);
        assertNotNull(message, result.getJsonObject());
        assertNotNull(message, result.getJsonObject().getAsJsonObject("hits"));
        assertNotNull(message, result.getJsonObject().getAsJsonObject("hits").get("total"));
        int nbrResult = result.getJsonObject().getAsJsonObject("hits").get("total").getAsInt();

        Assert.assertEquals("Unexpected hit count searching for {" + queryString + "} in field {" + field + "}",
                expectedHitCount, nbrResult);
        return result.getJsonObject().getAsJsonObject("hits").getAsJsonArray("hits").getAsJsonArray().iterator()
                .next().getAsJsonObject().get("_source").toString();
    }

}