org.structr.rest.resource.LogResource.java Source code

Java tutorial

Introduction

Here is the source code for org.structr.rest.resource.LogResource.java

Source

/**
 * Copyright (C) 2010-2016 Structr GmbH
 *
 * This file is part of Structr <http://structr.org>.
 *
 * Structr is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * Structr is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Structr.  If not, see <http://www.gnu.org/licenses/>.
 */
package org.structr.rest.resource;

import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang3.StringUtils;
import org.structr.common.SecurityContext;
import org.structr.common.error.EmptyPropertyToken;
import org.structr.common.error.ErrorBuffer;
import org.structr.common.error.FrameworkException;
import org.structr.core.GraphObject;
import org.structr.core.GraphObjectMap;
import org.structr.core.Result;
import org.structr.core.Services;
import org.structr.core.app.App;
import org.structr.core.app.StructrApp;
import org.structr.core.graph.Tx;
import org.structr.core.property.GenericProperty;
import org.structr.core.property.ISO8601DateProperty;
import org.structr.core.property.IntProperty;
import org.structr.core.property.Property;
import org.structr.core.property.PropertyKey;
import org.structr.core.property.PropertyMap;
import org.structr.core.property.StringProperty;
import org.structr.rest.RestMethodResult;
import org.structr.rest.exception.IllegalMethodException;
import org.structr.rest.logging.entity.LogEvent;

/**
 *
 *
 *
 */
public class LogResource extends Resource {

    private static final Logger logger = Logger.getLogger(LogResource.class.getName());
    private static final Pattern RangeQueryPattern = Pattern.compile("\\[(.+) TO (.+)\\]");

    private static final String SUBJECTS = "/s/";
    private static final String CORRELATION_SEPARATOR = "::";

    private static final Property<String> subjectProperty = new StringProperty("subject");
    private static final Property<String> objectProperty = new StringProperty("object");
    private static final Property<String> actionProperty = new StringProperty("action");
    private static final Property<String> actionsProperty = new StringProperty("actions");
    private static final Property<String> messageProperty = new StringProperty("message");
    private static final Property<Integer> entryCountProperty = new IntProperty("entryCount");
    private static final Property<Integer> totalProperty = new IntProperty("total");
    private static final ISO8601DateProperty timestampProperty = new ISO8601DateProperty("timestamp");
    private static final ISO8601DateProperty firstEntryProperty = new ISO8601DateProperty("firstEntry");
    private static final ISO8601DateProperty lastEntryProperty = new ISO8601DateProperty("lastEntry");

    private static final Set<String> ReservedRequestParameters = new LinkedHashSet<>(Arrays.asList(new String[] {
            "subject", "object", "action", "message", "timestamp", "aggregate", "histogram", "correlate" }));

    public static final String LOG_RESOURCE_URI = "log";

    @Override
    public Resource tryCombineWith(Resource next) throws FrameworkException {
        return null;
    }

    @Override
    public boolean checkAndConfigure(String part, SecurityContext securityContext, HttpServletRequest request)
            throws FrameworkException {

        subjectProperty.setDeclaringClass(LogResource.class);
        objectProperty.setDeclaringClass(LogResource.class);
        actionProperty.setDeclaringClass(LogResource.class);
        messageProperty.setDeclaringClass(LogResource.class);
        timestampProperty.setDeclaringClass(LogResource.class);

        this.securityContext = securityContext;
        this.securityContext.setRequest(request);

        return LOG_RESOURCE_URI.equals(part);
    }

    @Override
    public String getUriPart() {
        return LOG_RESOURCE_URI;
    }

    @Override
    public Class<? extends GraphObject> getEntityClass() {
        return GraphObject.class;
    }

    @Override
    public String getResourceSignature() {
        return "Log";
    }

    @Override
    public boolean isCollectionResource() throws FrameworkException {
        return true;
    }

    @Override
    public Result doGet(PropertyKey sortKey, boolean sortDescending, int pageSize, int page, String offsetId)
            throws FrameworkException {

        final HttpServletRequest request = securityContext.getRequest();
        if (request != null) {

            final String subjectId = request.getParameter(subjectProperty.jsonName());
            final String objectId = request.getParameter(objectProperty.jsonName());
            final GraphObjectMap overviewMap = new GraphObjectMap();
            final LogState logState = new LogState(request);

            if (StringUtils.isNotEmpty(subjectId) && StringUtils.isNotEmpty(objectId)) {

                processData(logState, StructrApp.getInstance(securityContext).nodeQuery(LogEvent.class)
                        .and(LogEvent.subjectProperty, subjectId).and(LogEvent.objectProperty, objectId)
                        .and(LogEvent.actionProperty, logState.logAction).andRange(LogEvent.timestampProperty,
                                new Date(logState.beginTimestamp()), new Date(logState.endTimestamp()))
                        .getAsList());

            } else if (StringUtils.isNotEmpty(subjectId) && StringUtils.isEmpty(objectId)) {

                processData(logState, StructrApp.getInstance(securityContext).nodeQuery(LogEvent.class)
                        .and(LogEvent.subjectProperty, subjectId).and(LogEvent.actionProperty, logState.logAction)
                        .andRange(LogEvent.timestampProperty, new Date(logState.beginTimestamp()),
                                new Date(logState.endTimestamp()))
                        .getAsList());

            } else if (StringUtils.isEmpty(subjectId) && StringUtils.isNotEmpty(objectId)) {

                logState.inverse(true);

                processData(logState, StructrApp.getInstance(securityContext).nodeQuery(LogEvent.class)
                        .and(LogEvent.objectProperty, objectId).and(LogEvent.actionProperty, logState.logAction)
                        .andRange(LogEvent.timestampProperty, new Date(logState.beginTimestamp()),
                                new Date(logState.endTimestamp()))
                        .getAsList());

            } else if (logState.doActionQuery()) {

                processData(logState);

            } else {

                // create overview of existing logs
                logState.overview(true);

                processData(logState,
                        StructrApp.getInstance(securityContext).nodeQuery(LogEvent.class).getAsList());
            }

            if (logState.overview()) {

                overviewMap.put(actionsProperty, logState.actions());
                overviewMap.put(entryCountProperty, logState.actionCount());
                overviewMap.put(firstEntryProperty, new Date(logState.beginTimestamp()));
                overviewMap.put(lastEntryProperty, new Date(logState.endTimestamp()));

                return new Result(overviewMap, false);

            } else if (logState.doHistogram()) {

                // aggregate results
                return histogram(logState);

            } else if (logState.doAggregate()) {

                // aggregate results
                return aggregate(logState);

            } else {

                // sort result
                logState.sortEntries();

                return new Result(wrap(logState.entries()), logState.size(), true, false);
            }
        }

        // no request object, this is fatal
        throw new FrameworkException(500, "No request object present, aborting.");
    }

    @Override
    public RestMethodResult doPost(Map<String, Object> propertySet) throws FrameworkException {

        final HttpServletRequest request = securityContext.getRequest();
        if (request != null) {

            // initialize?!
            if ("true".equals(request.getParameter("initialize"))) {

                final String filesPath = Services.getInstance().getConfigurationValue(Services.FILES_PATH);

                try (final Context context = new Context(1000)) {

                    collectFilesAndStore(context, new File(filesPath + SUBJECTS).toPath(), 0);

                } catch (FrameworkException fex) {
                    fex.printStackTrace();
                }

                return new RestMethodResult(200);
            }

            final String subjectId = (String) propertySet.get(subjectProperty.jsonName());
            final String objectId = (String) propertySet.get(objectProperty.jsonName());
            final String action = (String) propertySet.get(actionProperty.jsonName());
            final String message = (String) propertySet.get(messageProperty.jsonName());

            if (subjectId != null && objectId != null && action != null) {

                final App app = StructrApp.getInstance(securityContext);
                LogEvent event = null;

                try (final Tx tx = app.tx()) {

                    final PropertyMap properties = new PropertyMap();
                    properties.put(LogEvent.timestampProperty, new Date());
                    properties.put(LogEvent.actionProperty, action);
                    properties.put(LogEvent.subjectProperty, subjectId);
                    properties.put(LogEvent.objectProperty, objectId);
                    properties.put(LogEvent.messageProperty, message);
                    properties.put(LogEvent.visibleToPublicUsers, true);
                    properties.put(LogEvent.visibleToAuthenticatedUsers, true);

                    event = app.create(LogEvent.class, properties);

                    tx.success();
                }

                final RestMethodResult result = new RestMethodResult(201);
                result.addContent(event);

                return result;

                //
                //            try {
                //
                //               final StringBuilder data = new StringBuilder();
                //               data.append(System.currentTimeMillis());
                //               data.append(",");
                //               data.append(action);
                //               data.append(",");
                //               data.append(message);
                //               data.append("\n");
                //
                //               // write data and create link
                //               final Path actualPath = write(filesPath + SUBJECTS, mergeIds(subjectId, objectId), data.toString());
                //               link(filesPath + OBJECTS, mergeIds(objectId, subjectId), actualPath);
                //
                //               return new RestMethodResult(200);
                //
                //            } catch (IOException ioex) {
                //
                //               ioex.printStackTrace();
                //               throw new FrameworkException(500, ioex.getMessage());
                //            }

            } else {

                final ErrorBuffer errorBuffer = new ErrorBuffer();

                if (StringUtils.isEmpty(subjectId)) {
                    errorBuffer.add(new EmptyPropertyToken("LogFile", subjectProperty));
                }

                if (StringUtils.isEmpty(objectId)) {
                    errorBuffer.add(new EmptyPropertyToken("LogFile", objectProperty));
                }

                if (StringUtils.isEmpty(action)) {
                    errorBuffer.add(new EmptyPropertyToken("LogFile", actionProperty));
                }

                throw new FrameworkException(422, errorBuffer);
            }
        }

        // no request object, this is fatal
        throw new FrameworkException(500, "No request object present, aborting.");
    }

    @Override
    public RestMethodResult doPut(final Map<String, Object> propertySet) throws FrameworkException {
        throw new IllegalMethodException();
    }

    @Override
    public RestMethodResult doDelete() throws FrameworkException {
        throw new IllegalMethodException();
    }

    @Override
    public RestMethodResult doHead() throws FrameworkException {
        throw new IllegalMethodException();
    }

    @Override
    public RestMethodResult doOptions() throws FrameworkException {

        final RestMethodResult result = new RestMethodResult(HttpServletResponse.SC_OK);

        result.addHeader("Allow", "GET,POST,OPTIONS");

        return result;
    }

    @Override
    public boolean createPostTransaction() {
        return false;
    }

    // ----- private methods -----
    private void collectFilesAndStore(final Context context, final Path dir, final int level)
            throws FrameworkException {

        if (level == 1) {
            logger.log(Level.INFO, "Path {0}", dir);
        }

        try (final DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {

            for (final Path p : stream) {

                if (Files.isDirectory(p)) {

                    collectFilesAndStore(context, p, level + 1);

                } else {

                    context.update(storeLogEntry(p));

                    // update object count and commit
                    context.commit(true);
                }

                Files.delete(p);

            }

        } catch (IOException ioex) {
            ioex.printStackTrace();
        }
    }

    private void processData(final LogState state) throws FrameworkException {

        if (state.doCorrelate()) {

            // get the basic correlation set (pds_click in the test case)
            final List<LogEvent> correlationResult = StructrApp.getInstance(securityContext)
                    .nodeQuery(LogEvent.class).and(LogEvent.actionProperty, state.correlationAction).getAsList();

            for (final LogEvent entry : correlationResult) {

                final String pathSubjectId = state.inverse() ? entry.getObjectId() : entry.getSubjectId();
                final String pathObjectId = state.inverse() ? entry.getSubjectId() : entry.getObjectId();
                final String entryMessage = entry.getMessage();

                if (state.correlationPattern != null) {

                    final Matcher matcher = state.correlationPattern.matcher(entryMessage);
                    if (matcher.matches()) {

                        state.addCorrelationEntry(matcher.group(1), entry);

                    }

                } else {
                    // fallback: subjectId and objectId
                    state.addCorrelationEntry(key(pathSubjectId, pathObjectId), entry);
                }
            }
        }

        logger.log(Level.FINE, "No. of correlations: {0}", state.getCorrelations().entrySet().size());

        final List<LogEvent> result = StructrApp.getInstance(securityContext).nodeQuery(LogEvent.class)
                .and(LogEvent.actionProperty, state.logAction).andRange(LogEvent.timestampProperty,
                        new Date(state.beginTimestamp()), new Date(state.endTimestamp()))
                .getAsList();

        processData(state, result);
    }

    private void processData(final LogState state, final Iterable<LogEvent> result) throws FrameworkException {

        int count = 0;

        for (final LogEvent event : result) {

            if ((++count % 100000) == 0) {
                System.out.println(count);
            }

            final String pathSubjectId = state.inverse() ? event.getObjectId() : event.getSubjectId();
            final String pathObjectId = state.inverse() ? event.getSubjectId() : event.getObjectId();
            final long timestamp = event.getTimestamp();
            final String entryAction = event.getAction();
            final String entryMessage = event.getMessage();

            // determine first timestamp
            if (timestamp <= state.beginTimestamp()) {
                state.beginTimestamp(timestamp);
            }

            // determine last timestamp
            if (timestamp >= state.endTimestamp()) {
                state.endTimestamp(timestamp);
            }

            if (state.overview()) {

                if (entryAction != null) {

                    state.countAction(entryAction);

                } else {

                    state.countAction("null");
                }

            } else {

                // passes filter? action present or matching?
                if (state.passesFilter(entryMessage)
                        && state.correlates(pathSubjectId, pathObjectId, entryMessage)) {

                    final Map<String, Object> map = new HashMap<>();

                    map.put(subjectProperty.jsonName(), pathSubjectId);
                    map.put(objectProperty.jsonName(), pathObjectId);
                    map.put(actionProperty.jsonName(), entryAction);
                    map.put(timestampProperty.jsonName(), timestamp);
                    map.put(messageProperty.jsonName(), entryMessage);

                    state.addEntry(map);
                }
            }
        }
    }

    private int storeLogEntry(final Path path) throws IOException, FrameworkException {

        final App app = StructrApp.getInstance(securityContext);
        final String fileName = path.getFileName().toString();
        int count = 0;

        if (fileName.length() == 64) {

            final String subjectId = fileName.substring(0, 32);
            final String objectId = fileName.substring(32, 64);

            for (final String line : Files.readAllLines(path, Charset.forName("utf-8"))) {

                final int pos1 = line.indexOf(",", 14);

                final String part0 = line.substring(0, 13);
                final String part1 = line.substring(14, pos1);
                final String part2 = line.substring(pos1 + 1);

                final long timestamp = Long.valueOf(part0);
                final String action = part1;
                final String message = part2;

                final PropertyMap properties = new PropertyMap();

                properties.put(LogEvent.messageProperty, message);
                properties.put(LogEvent.actionProperty, action);
                properties.put(LogEvent.subjectProperty, subjectId);
                properties.put(LogEvent.objectProperty, objectId);
                properties.put(LogEvent.timestampProperty, new Date(timestamp));
                properties.put(LogEvent.visibleToPublicUsers, true);
                properties.put(LogEvent.visibleToAuthenticatedUsers, true);

                app.create(LogEvent.class, properties);

                count++;
            }

        } else {

            System.out.println("Skipping entry " + fileName);
        }

        return count;
    }

    private String getDirectoryPath(final String uuid, final int depth) {

        final StringBuilder buf = new StringBuilder();

        if (StringUtils.isNotEmpty(uuid) && uuid.length() > depth) {

            for (int i = 0; i < depth; i++) {

                buf.append(uuid.substring(i, i + 1));
                buf.append("/");
            }
        }

        return buf.toString();
    }

    private Result aggregate(final LogState state) throws FrameworkException {

        // sort entries before aggregation
        state.sortEntries();

        final long startTimestamp = state.beginTimestamp();
        final long endTimestamp = state.endTimestamp();
        final GraphObjectMap result = new GraphObjectMap();
        final long interval = findInterval(state.aggregate());
        final long start = alignDateOnFormat(state.aggregate(), startTimestamp);
        final TreeMap<Long, Map<String, Object>> countMap = toAggregatedCountMap(state);
        final Set<String> countProperties = getCountProperties(countMap);

        for (long current = start; current <= endTimestamp; current += interval) {

            final Map<Long, Map<String, Object>> counts = countMap.subMap(current, true, current + interval, false);
            final GraphObjectMap sum = new GraphObjectMap();

            // initialize interval sums with 0 (so each
            // interval contains all keys regardless of
            // whether there are actual values or not)
            for (final String key : countProperties) {
                sum.put(new IntProperty(key), 0);
            }

            // evaluate counts
            for (final Map<String, Object> count : counts.values()) {

                for (final String key : countProperties) {

                    final IntProperty prop = new IntProperty(key);
                    Integer sumValue = sum.get(prop);
                    if (sumValue == null) {
                        sumValue = 0;
                    }

                    Integer entryValue = (Integer) count.get(key);
                    if (entryValue == null) {
                        entryValue = 0;
                    }

                    sum.put(prop, sumValue + entryValue);
                }
            }

            result.put(new GenericProperty(Long.toString(current)), sum);
        }

        return new Result(result, false);
    }

    private Result histogram(final LogState state) throws FrameworkException {

        // sort entries before creating the histogram
        state.sortEntries();

        final String dateFormat = state.aggregate();
        final long startTimestamp = state.beginTimestamp();
        final long endTimestamp = state.endTimestamp();
        final GraphObjectMap result = new GraphObjectMap();
        final long interval = findInterval(dateFormat);
        final long start = alignDateOnFormat(dateFormat, startTimestamp);
        final TreeMap<Long, Map<String, Object>> countMap = toHistogramCountMap(state);
        final Set<String> countProperties = getCountProperties(countMap);

        for (long current = start; current <= endTimestamp; current += interval) {

            final Map<Long, Map<String, Object>> counts = countMap.subMap(current, true, current + interval, false);
            final GraphObjectMap sum = new GraphObjectMap();

            // initialize interval sums with 0 (so each
            // interval contains all keys regardless of
            // whether there are actual values or not)
            for (final String key : countProperties) {
                sum.put(new IntProperty(key), 0);
            }

            // evaluate counts
            for (final Map<String, Object> count : counts.values()) {

                for (final String key : countProperties) {

                    final IntProperty prop = new IntProperty(key);
                    Integer sumValue = sum.get(prop);
                    if (sumValue == null) {
                        sumValue = 0;
                    }

                    Integer entryValue = (Integer) count.get(key);
                    if (entryValue == null) {
                        entryValue = 0;
                    }

                    sum.put(prop, sumValue + entryValue);
                }
            }

            result.put(new GenericProperty(Long.toString(current)), sum);
        }

        return new Result(result, false);
    }

    private long alignDateOnFormat(final String dateFormat, final long timestamp) {

        try {

            final SimpleDateFormat format = new SimpleDateFormat(dateFormat);
            return format.parse(format.format(timestamp)).getTime();

        } catch (ParseException pex) {
            pex.printStackTrace();
        }

        return 0L;
    }

    /**
     * This method takes a date format and finds the time interval that it
     * represents.
     */
    private long findInterval(final String dateFormat) {

        final long max = TimeUnit.DAYS.toMillis(365);
        final long step = TimeUnit.SECONDS.toMillis(60);

        try {

            final SimpleDateFormat format = new SimpleDateFormat(dateFormat);
            final long initial = format.parse(format.format(3600)).getTime();

            for (long i = initial; i < max; i += step) {

                final long current = format.parse(format.format(i)).getTime();

                if (initial != current) {
                    return i - initial;
                }
            }

            return max;

        } catch (ParseException pex) {
            pex.printStackTrace();
        }

        return max;
    }

    private TreeMap<Long, Map<String, Object>> toAggregatedCountMap(final LogState state)
            throws FrameworkException {

        final TreeMap<Long, Map<String, Object>> countMap = new TreeMap<>();

        for (final Map<String, Object> entry : state.entries()) {

            final String message = (String) entry.get(messageProperty.jsonName());
            final long timestamp = (Long) entry.get(timestampProperty.jsonName());
            Map<String, Object> obj = countMap.get(timestamp);

            if (obj == null) {
                obj = new LinkedHashMap<>();
            }

            Integer count = (Integer) obj.get(totalProperty.jsonName());
            if (count == null) {
                count = 1;
            } else {
                count = count + 1;
            }
            obj.put(totalProperty.jsonName(), count);

            // iterate over patterns
            for (final Entry<String, Pattern> patternEntry : state.aggregationPatterns().entrySet()) {

                if (patternEntry.getValue().matcher(message).matches()) {

                    final String key = patternEntry.getKey();

                    final int multiplier = getMultiplier(message, state);

                    Integer c = (Integer) obj.get(key);
                    if (c == null) {
                        c = multiplier;
                    } else {
                        c = c + multiplier;
                    }

                    obj.put(key, c);
                }
            }

            countMap.put(timestamp, obj);
        }

        return countMap;
    }

    private TreeMap<Long, Map<String, Object>> toHistogramCountMap(final LogState state) throws FrameworkException {

        final Pattern pattern = Pattern.compile(state.histogram());
        final Matcher matcher = pattern.matcher("");
        final TreeMap<Long, Map<String, Object>> countMap = new TreeMap<>();

        for (final Map<String, Object> entry : state.entries()) {

            final String message = (String) entry.get(messageProperty.jsonName());
            final long timestamp = (Long) entry.get(timestampProperty.jsonName());
            Map<String, Object> obj = countMap.get(timestamp);

            if (obj == null) {
                obj = new LinkedHashMap<>();
            }

            Integer count = (Integer) obj.get(totalProperty.jsonName());
            if (count == null) {
                count = 1;
            } else {
                count = count + 1;
            }
            obj.put(totalProperty.jsonName(), count);

            // iterate over patterns
            matcher.reset(message);
            if (matcher.matches()) {

                final String key = matcher.group(1);

                final int multiplier = getMultiplier(message, state);

                Integer c = (Integer) obj.get(key);
                if (c == null) {
                    c = multiplier;
                } else {
                    c = c + multiplier;
                }

                obj.put(key, c);
            }

            countMap.put(timestamp, obj);
        }

        return countMap;
    }

    private int getMultiplier(final String message, final LogState state) {

        Integer multiplier = 1;

        if (state.multiplier != null) {

            final Matcher matcher = Pattern.compile(state.multiplier).matcher(message);

            if (matcher.matches()) {
                final String g = matcher.group(1);
                multiplier = Integer.parseInt(g);
            }

        }

        return multiplier;

    }

    private Set<String> getCountProperties(final Map<Long, Map<String, Object>> entries) {

        final Set<String> result = new LinkedHashSet<>();

        for (final Map<String, Object> obj : entries.values()) {

            for (final Entry<String, Object> entry : obj.entrySet()) {

                // collect the key names of integer values
                if (entry.getValue() instanceof Integer) {
                    result.add(entry.getKey());
                }
            }
        }

        return result;
    }

    private List<GraphObjectMap> wrap(final List<Map<String, Object>> entries) {

        final List<GraphObjectMap> result = new LinkedList<>();

        for (final Map<String, Object> entry : entries) {

            final GraphObjectMap map = new GraphObjectMap();
            for (final Entry<String, Object> e : entry.entrySet()) {

                final String key = e.getKey();

                if (timestampProperty.jsonName().equals(key)) {

                    map.put(timestampProperty, new Date((Long) e.getValue()));

                } else {

                    map.put(new GenericProperty(key), e.getValue());
                }
            }

            result.add(map);
        }

        return result;
    }

    private <T> Set<T> toSet(final T element) {

        final Set<T> set = new HashSet<>();
        set.add(element);

        return set;
    }

    private <T> Set<T> toSet(final Iterable<T> iterable) {

        final Set<T> set = new HashSet<>();
        for (final T path : iterable) {

            set.add(path);
        }

        return set;
    }

    private static class LogState {

        private final Map<String, Pattern> aggregationPatterns = new HashMap<>();
        private final List<Map<String, Object>> entries = new LinkedList<>();
        private final Map<String, LinkedList<LogEvent>> correlations = new ConcurrentHashMap<>();
        private final Map<String, Integer> actions = new HashMap<>();

        private long beginTimestamp = Long.MAX_VALUE;
        private long endTimestamp = 0L;
        private String logAction = null;
        private String aggregate = null;
        private String histogram = null;
        private String multiplier = null;
        private String correlate = null;
        private String correlationAction = null;
        private String correlationOp = null;
        private Pattern correlationPattern = null;
        private String[] filters = null;
        private boolean inverse = false;
        private boolean overview = false;
        private Range range = null;
        private int actionCount = 0;
        private boolean doCorrelate = false;

        public LogState(final HttpServletRequest request) {

            aggregationPatterns.putAll(getAggregationPatterns(request));

            this.logAction = request.getParameter(actionProperty.jsonName());
            this.aggregate = request.getParameter("aggregate");
            this.histogram = request.getParameter("histogram");
            this.correlate = request.getParameter("correlate");
            this.multiplier = request.getParameter("multiplier");
            this.filters = getFilterPatterns(request);
            this.range = getRange(request);

            if (StringUtils.isNotBlank(correlate)) {

                final String[] parts = correlate.split(CORRELATION_SEPARATOR);

                if (parts.length > 0) {
                    correlationAction = parts[0];
                }

                if (parts.length > 1) {
                    correlationOp = parts[1];
                }

                if (parts.length > 2) {
                    correlationPattern = Pattern.compile(parts[2]);
                }

                doCorrelate = true;
            }

        }

        public List<Map<String, Object>> entries() {
            return entries;
        }

        public void addEntry(final Map<String, Object> entry) {
            entries.add(entry);
        }

        public void addCorrelationEntry(final String key, final LogEvent event) {

            logger.log(Level.FINE, "No. of correllation entry lists: {0}, adding action: {1} {2}",
                    new Object[] { correlations.keySet().size(), key, event.getMessage() });

            LinkedList<LogEvent> existingEventList = correlations.get(key);

            if (existingEventList == null) {
                existingEventList = new LinkedList<>();
            }

            existingEventList.add(event);

            correlations.put(key, existingEventList);
        }

        public Map<String, LinkedList<LogEvent>> getCorrelations() {
            return correlations;
        }

        public Map<String, Integer> actions() {
            return actions;
        }

        public Map<String, Pattern> aggregationPatterns() {
            return aggregationPatterns;
        }

        public void countAction(final String action) {

            Integer actionCount = actions.get(action);
            if (actionCount == null) {

                actions.put(action, 1);

            } else {

                actions.put(action, actionCount + 1);
            }

            this.actionCount++;
        }

        public int actionCount() {
            return actionCount;
        }

        public boolean isRequestedActionOrNull(final String action) {
            return logAction == null || logAction.equals(action);
        }

        public void sortEntries() {
            Collections.sort(entries, new TimestampComparator());
        }

        public int size() {
            return entries.size();
        }

        public void inverse(final boolean inverse) {
            this.inverse = inverse;
        }

        public boolean inverse() {
            return inverse;
        }

        public void overview(final boolean overview) {
            this.overview = overview;
        }

        public boolean overview() {
            return overview;
        }

        public long beginTimestamp() {
            return range != null ? range.start : beginTimestamp;
        }

        public long endTimestamp() {
            return range != null ? range.end : endTimestamp;
        }

        public void beginTimestamp(final long beginTimestamp) {
            this.beginTimestamp = beginTimestamp;
        }

        public void endTimestamp(final long endTimestamp) {
            this.endTimestamp = endTimestamp;
        }

        public boolean isInRangeOrNull(final long timestamp) {
            return range == null || range.contains(timestamp);
        }

        public String histogram() {
            return histogram;
        }

        public String aggregate() {
            return aggregate;
        }

        public boolean passesFilter(final String message) {

            if (filters == null) {
                return true;
            }

            boolean passes = true;

            for (final String filter : filters) {

                passes &= Pattern.compile(filter).matcher(message).matches();

            }

            return passes;

        }

        public boolean correlates(final String pathSubjectId, final String pathObjectId, final String message) {

            if (correlations.isEmpty()) {

                // TODO: why would this be true? wouldnt it be false?
                return true;

            }

            LinkedList<LogEvent> correlationEntries;

            if (correlationOp != null && correlationPattern != null) {

                final Matcher matcher = correlationPattern.matcher(message);

                if (matcher.matches()) {

                    final String value = matcher.group(1);

                    switch (correlationOp) {

                    case "and":

                        return correlations.containsKey(value);

                    case "andSubject":

                        correlationEntries = correlations.get(value);

                        if (correlationEntries != null) {

                            for (LogEvent correlationEntry : correlationEntries) {

                                if (correlationEntry.getSubjectId().equals(pathSubjectId)) {

                                    return true;

                                }

                            }

                        }

                        return false;

                    case "andObject":

                        correlationEntries = correlations.get(value);

                        if (correlationEntries != null) {

                            for (LogEvent correlationEntry : correlationEntries) {

                                if (correlationEntry.getObjectId().equals(pathObjectId)) {

                                    return true;

                                }

                            }

                        }

                        return false;

                    case "not":

                        return !correlations.containsKey(value);

                    default:
                        return false;
                    }

                }

                return "not".equals(correlationOp);

            } else {

                // fallback
                return correlations.containsKey(key(pathSubjectId, pathObjectId));

            }

        }

        public boolean isCorrelatedAction(final String input) {
            return (correlationAction == null || correlationAction.equals(input));
        }

        public boolean doHistogram() throws FrameworkException {

            if (StringUtils.isNotBlank(histogram)) {

                if (StringUtils.isBlank(aggregate)) {
                    throw new FrameworkException(400,
                            "To use the histogram function, please supply an aggregation pattern.");
                }

                return true;
            }

            return false;
        }

        public boolean doAggregate() {
            return StringUtils.isNotBlank(aggregate);
        }

        public boolean doCorrelate() {
            return doCorrelate;
        }

        public boolean doActionQuery() {
            return StringUtils.isNotBlank(logAction);
        }

        public boolean includeFile(final File file) {
            return range == null || range.contains(file.lastModified());
        }

        // ----- private methods -----
        private Range getRange(final HttpServletRequest request) {

            final String value = request.getParameter(timestampProperty.jsonName());
            if (value != null) {

                if (StringUtils.startsWith(value, "[") && StringUtils.endsWith(value, "]")) {

                    // check for existance of range query string
                    Matcher matcher = RangeQueryPattern.matcher(value);
                    if (matcher.matches()) {

                        if (matcher.groupCount() == 2) {

                            final SimpleDateFormat parser = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
                            String rangeStart = matcher.group(1);
                            String rangeEnd = matcher.group(2);

                            try {

                                final Date startDate = parser.parse(rangeStart);
                                final Date endDate = parser.parse(rangeEnd);

                                return new Range(startDate.getTime(), endDate.getTime());

                            } catch (ParseException pex) {

                                pex.printStackTrace();
                            }
                        }
                    }
                }

            }

            return null;
        }

        private Map<String, Pattern> getAggregationPatterns(final HttpServletRequest request) {

            final Map<String, Pattern> patterns = new LinkedHashMap<>();

            for (final Entry<String, String[]> entry : request.getParameterMap().entrySet()) {

                final String key = entry.getKey();
                final String[] value = entry.getValue();

                if (value.length > 0 && !ReservedRequestParameters.contains(key)) {
                    patterns.put(key, Pattern.compile(value[0]));
                }
            }

            return patterns;
        }

        private String[] getFilterPatterns(final HttpServletRequest request) {

            final String filterString = request.getParameter("filters");
            if (StringUtils.isNotBlank(filterString)) {
                return filterString.split(CORRELATION_SEPARATOR);
            }

            return null;
        }
    }

    private static String key(final String subjectId, final String objectId) {

        if (subjectId != null && objectId != null) {
            return subjectId.concat(objectId);
        }

        return "NULLNULL";
    }

    private static class Range {

        private long start = 0L;
        private long end = 0L;

        public Range(final long start, final long end) {
            this.start = start;
            this.end = end;
        }

        public boolean contains(final long timestamp) {
            return timestamp >= start && timestamp <= end;
        }
    }

    private static class TimestampComparator implements Comparator<Map<String, Object>> {

        @Override
        public int compare(final Map<String, Object> o1, final Map<String, Object> o2) {

            final Long timestamp1 = (Long) o1.get(timestampProperty.jsonName());
            final Long timestamp2 = (Long) o2.get(timestampProperty.jsonName());

            return timestamp1.compareTo(timestamp2);
        }
    }

    private static class Context implements AutoCloseable {

        private final App app = StructrApp.getInstance();
        private Tx tx = null;
        private int total = 0;
        private int count = 0;
        private int commitCount = 0;

        public Context(final int commitCount) {

            this.commitCount = commitCount;
            this.tx = app.tx(false, false, false);
        }

        public void commit(final boolean intermediate) throws FrameworkException {

            if (count > commitCount) {

                logger.log(Level.INFO, "Committing transaction after {0} objects..", total);

                tx.success();
                tx.close();

                if (intermediate) {

                    tx = app.tx(false, false, false);
                }

                count = 0;
            }
        }

        public int getTotal() {
            return total;
        }

        public void update(final int count) {
            this.count += count;
            this.total += count;
        }

        @Override
        public void close() throws FrameworkException {

            tx.success();
            tx.close();
        }
    }
}