sf.net.experimaestro.manager.plans.Operator.java Source code

Java tutorial

Introduction

Here is the source code for sf.net.experimaestro.manager.plans.Operator.java

Source

package sf.net.experimaestro.manager.plans;

/*
 * This file is part of experimaestro.
 * Copyright (c) 2014 B. Piwowarski <benjamin@bpiwowar.net>
 *
 * experimaestro is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * experimaestro is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with experimaestro.  If not, see <http://www.gnu.org/licenses/>.
 */

import bpiwowar.argparser.utils.Output;
import com.google.common.base.Function;
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
import org.apache.commons.lang.mutable.MutableInt;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Scriptable;
import sf.net.experimaestro.manager.experiments.Experiment;
import sf.net.experimaestro.manager.experiments.TaskReference;
import sf.net.experimaestro.manager.js.JsonPathFunction;
import sf.net.experimaestro.manager.json.Json;
import sf.net.experimaestro.manager.scripting.Expose;
import sf.net.experimaestro.manager.scripting.Exposed;
import sf.net.experimaestro.manager.scripting.Help;
import sf.net.experimaestro.manager.scripting.ScriptContext;
import sf.net.experimaestro.utils.CachedIterable;
import sf.net.experimaestro.utils.WrappedResult;
import sf.net.experimaestro.utils.log.Logger;

import javax.xml.xpath.XPathExpressionException;
import java.io.PrintStream;
import java.util.*;

/**
 * Base class for all operators
 *
 * @author B. Piwowarski <benjamin@bpiwowar.net>
 */
@Exposed
public abstract class Operator {
    final static private Logger LOGGER = Logger.getLogger();

    /**
     * Specific name for the operator
     */
    protected String name;

    /**
     * Size of the output (1 per default)
     */
    int outputSize = 1;

    /**
     * List of mappings for context
     */
    Map<StreamReference, Integer> contextMappings = new HashMap<>();

    public Operator() {
    }

    public Operator(String name) {
        this.name = name;
    }

    static Operator getSimplified(Map<Operator, Operator> simplified, Operator operator) {
        Operator tmp;
        while ((tmp = simplified.get(operator)) != null)
            operator = tmp;
        return operator;
    }

    public static void ensureConnections(Map<Operator, Operator> simplified, List<Operator> operators) {
        for (int i = 0; i < operators.size(); i++) {
            operators.set(i, getSimplified(simplified, operators.get(i)));
        }
    }

    public static void ensureConnections(final Map<Operator, Operator> simplified, Set<Operator> set) {
        ArrayList<Operator> list = new ArrayList<>(set);
        set.clear();
        for (Operator operator : Iterables.transform(list, new Function<Operator, Operator>() {
            @Override
            public Operator apply(Operator input) {
                return getSimplified(simplified, input);
            }
        })) {
            set.add(operator);
        }

    }

    /**
     * Copy a collection of operators
     */
    protected static Iterable<Operator> copy(Iterable<Operator> collection, final boolean deep,
            final Map<Object, Object> map) {
        return Iterables.transform(collection, new Function<Operator, Operator>() {
            @Override
            public Operator apply(Operator input) {
                return input.copy(deep, map);
            }
        });
    }

    static public Operator simplify(Operator operator) {
        HashMap<Operator, Operator> map = new HashMap<>();
        operator = simplify(operator, map);
        operator.ensureConnections(map, new HashSet<Operator>());
        return operator;
    }

    static public Operator simplify(Operator operator, Map<Operator, Operator> simplified) {
        Operator cache = simplified.get(simplified);
        if (cache != null)
            return cache;

        // --- First, simplify all the parents
        List<Operator> parents = operator.getParents();
        for (int i = 0; i < parents.size(); i++) {
            Operator newParent = simplify(parents.get(i), simplified);
            if (newParent != parents.get(i)) {
                parents.set(i, newParent);
            }
        }

        // --- operator == Union
        if (operator instanceof Union) {
            if (parents.size() == 1) {
                return simplify(operator, parents.get(0), simplified);
            }
        }

        // --- operator == Product
        if (operator instanceof Product) {
            if (parents.size() == 1) {
                return simplify(operator, parents.get(0), simplified);
            }
        }

        // --- operator == Union
        if (operator instanceof OrderBy) {
            if (((OrderBy) operator).size() == 0) {
                return simplify(operator, parents.get(0), simplified);
            }
        }

        return operator;
    }

    private static Operator simplify(Operator operator, Operator optimised, Map<Operator, Operator> simplified) {
        simplified.put(operator, optimised);
        return optimised;
    }

    /**
     * Returns the parents of this node
     */
    public abstract List<Operator> getParents();

    public void addParent(Operator parent) {
        throw new UnsupportedOperationException();
    }

    final public Operator getParent(int i) {
        return getParents().get(i);
    }

    /**
     * Recursive initialization of operator
     */
    public Operator prepare(Map<Operator, Operator> map, OperatorMap opMap) {
        List<Operator> parents = getParents();
        for (int i = 0; i < parents.size(); i++) {
            parents.set(i, parents.get(i).prepare(map, opMap));
        }
        ensureConnections(map);
        return this;
    }

    /**
     * Prepare an operator
     *
     * @return
     * @throws XPathExpressionException
     */
    public Operator prepare() {
        return prepare(new HashMap<>(), new OperatorMap());
    }

    /**
     * Returns the size of the output
     */
    public int outputSize() {
        return outputSize;
    }

    /**
     * Copy the operator
     *
     * @param deep Deep copy
     */
    final public Operator copy(boolean deep) {
        return copy(deep, new IdentityHashMap<Object, Object>());
    }

    final protected Operator copy(boolean deep, Map<Object, Object> map) {
        Object o = map.get(this);
        if (o != null)
            return (Operator) o;
        Operator copy = doCopy(deep, map);
        map.put(this, copy);
        return copy;
    }

    protected abstract Operator doCopy(boolean deep, Map<Object, Object> map);

    public void getAncestors(HashSet<Operator> ancestors) {
        if (ancestors.contains(this))
            return;

        ancestors.add(this);
        for (Operator parent : getParents())
            parent.getAncestors(ancestors);
    }

    public void addParents(Operator... parents) {
        for (Operator parent : parents) {
            addParent(parent);
        }
    }

    /**
     * Creates a new iterator
     *
     * @param scriptContext Options
     * @return A new iterator over return values
     */
    protected abstract Iterator<ReturnValue> _iterator(ScriptContext scriptContext);

    //    CachedIterable<Value> cachedIterable;
    //    PlanContext cachedOptions;

    // TODO: implement the cache
    public Iterator<Value> iterator(ScriptContext scriptContext) {
        // No cache: just return the iterator
        if (!cacheIterator())
            return new OperatorIterator(scriptContext);

        // Retrieve the cached iterable or retrieve it
        CachedIterable<Value> cachedIterable = scriptContext.getCachedIterable(this);

        if (cachedIterable == null) {
            LOGGER.debug("Setting up a cached iterator");
            cachedIterable = new CachedIterable<>(new OperatorIterator(scriptContext));
            scriptContext.setCachedIterable(this, cachedIterable);
        } else {
            // Use the cached values
            LOGGER.debug("Using cached iterator");
        }

        return cachedIterable.iterator();
    }

    /**
     * Whether we should cache the result of the iterator to avoid recomputing the values
     */
    boolean cacheIterator() {
        return false;
    }

    /**
     * Initialize the node (called before the initialization of parents)
     *
     * @throws javax.xml.xpath.XPathExpressionException
     */
    protected void doPreInit() {
    }

    /**
     * Initialize the node  (called after the initialization of parents)
     * <p>
     * Top-down calls (parents are initialized before)
     *
     * @param parentStreams A map from the operators from parent streams to the context index
     */
    protected void doPostInit(List<Map<Operator, Integer>> parentStreams) {
    }

    /**
     * Initialize the operator.
     * <p>
     * <ol>
     * <li>Calls the {@linkplain #doPreInit()} method</li>
     * <li>Initialize the parents</li>
     * <li>Calls the {@linkplain #doPostInit(List)} method</li>
     * </ol>
     *
     * @param processed A map of already processed operators along with the returned value
     * @param needed    (input) The set of needed operator streams that should go out of this operator
     * @return A map operator -> integer
     * @throws XPathExpressionException
     */
    final private Map<Operator, Integer> init(HashMap<Operator, Map<Operator, Integer>> processed,
            Multimap<Operator, Operator> needed) throws XPathExpressionException {
        // Check if already visited, and if so, return cached value
        Map<Operator, Integer> cached = processed.get(this);
        if (cached != null) {
            return cached;
        }

        // Initialise the streams that we need
        doPreInit();

        // First, init the parents, and store the map
        List<Map<Operator, Integer>> list = new ArrayList<>();
        for (Operator parent : getParents()) {
            Map<Operator, Integer> parentMap = parent.init(processed, needed);
            list.add(parentMap);
        }

        // Map the previous streams
        HashMap<Operator, Integer> map = new HashMap<>();
        int count = 0;
        Collection<Operator> streams = needed.get(this);

        for (Operator operator : streams) {
            for (int streamIndex = 0; streamIndex < list.size(); streamIndex++) {
                Map<Operator, Integer> parentMap = list.get(streamIndex);
                Integer contextIndex = parentMap.get(operator);
                if (contextIndex != null) {
                    contextMappings.put(new StreamReference(streamIndex, contextIndex), count);
                    map.put(operator, count);
                    count++;
                    break;
                }
            }

        }

        // Check if we should add ourselves to the context
        if (streams.contains(this)) {
            contextMappings.put(new StreamReference(-1, -1), count);
            map.put(this, count);
            count++;
        }

        doPostInit(list);

        processed.put(this, map);

        return map;
    }

    /**
     * Init our ancestors and ourselves
     */
    public void init() throws XPathExpressionException {

        Multimap<Operator, Operator> needed = HashMultimap.create();

        // Compute children
        Multimap<Operator, Operator> childrenMap = HashMultimap.create();
        HashSet<Operator> roots = new HashSet<>();
        computeChildren(roots, childrenMap);

        // Compute needed streams
        for (Operator root : roots)
            root.computeNeededStreams(childrenMap, needed);

        // Compute orders
        Map<Operator, Order<Operator>> orders = new HashMap<>();
        for (Operator root : roots)
            root.computeOrder(childrenMap, orders);

        init(new HashMap<Operator, Map<Operator, Integer>>(), needed);
    }

    private void computeChildren(HashSet<Operator> roots, Multimap<Operator, Operator> childrenMap) {
        if (getParents().size() == 0)
            roots.add(this);
        else
            for (Operator parent : getParents()) {
                // Early quit: if we already had this parent-child, this means
                // we already visited this operator
                if (!childrenMap.put(parent, this))
                    return;

                parent.computeChildren(roots, childrenMap);
            }
    }

    /**
     * Top-down computation of the order: we ask our children
     * to compute their order and then compute ours
     *
     * @param childrenMap
     * @param orders
     * @return
     */
    private Order<Operator> computeOrder(Multimap<Operator, Operator> childrenMap,
            Map<Operator, Order<Operator>> orders) {
        if (orders.containsKey(this))
            return orders.get(this);

        Collection<Operator> children = childrenMap.get(this);

        // Get the orders needed by children
        Order<Operator> childrenOrders[] = new Order[children.size()];

        int i = 0;
        for (Operator child : children) {
            childrenOrders[i++] = child.computeOrder(childrenMap, orders);
        }

        WrappedResult<Order<Operator>> result = Order.combine(childrenOrders);

        // and combine this with our own order
        Order<Operator> order = doComputeOrder(result.get());
        orders.put(this, order);
        return order;
    }

    protected Order<Operator> doComputeOrder(Order<Operator> childrenOrder) {
        childrenOrder.remove(this);
        return childrenOrder;
    }

    /**
     * @param childrenMap
     * @param needed
     * @return A collection of streams needed by this operator and its descendants
     */
    private Collection<Operator> computeNeededStreams(Multimap<Operator, Operator> childrenMap,
            Multimap<Operator, Operator> needed) {
        if (needed.containsKey(this))
            return needed.get(this);

        Collection<Operator> streams = needed.get(this);
        Collection<Operator> children = childrenMap.get(this);

        // Add the needed streams from all children
        LOGGER.debug("Adding needed streams for %s", this);
        for (Operator child : children) {
            Collection<Operator> c = child.computeNeededStreams(childrenMap, needed);
            for (Operator op : c)
                // TODO: consider using the ancestors map to check if we need to add
                if (c != child)
                    streams.add(op);
            child.addNeededStreams(streams);
        }

        return streams;
    }

    /**
     * Add the streams neeed by this operator
     */
    protected void addNeededStreams(Collection<Operator> streams) {
    }

    /**
     * Print a graph starting from this node
     *
     * @param out The output stream
     */
    public void printDOT(PrintStream out) {
        printDOT(out, null);
    }

    /**
     * Print a graph starting from this node
     *
     * @param out    The output stream
     * @param counts Number of results output by each operator
     */
    public void printDOT(PrintStream out, Map<Operator, MutableInt> counts) {
        out.println("digraph G {");
        printDOT(out, new HashSet<>(), counts);
        out.println("}");
        out.flush();
    }

    /**
     * Print a node
     *
     * @param out       The output stream
     * @param planNodes The values already processed (case of shared ancestors)
     */
    public boolean printDOT(PrintStream out, HashSet<Operator> planNodes, Map<Operator, MutableInt> counts) {
        if (planNodes.contains(this))
            return false;
        planNodes.add(this);
        printDOTNode(out, counts);
        int streamIndex = 0;
        for (Operator parent : getParents()) {
            parent.printDOT(out, planNodes, counts);
            ArrayList<Map.Entry<StreamReference, Integer>> list = new ArrayList<>();
            out.format("p%s -> p%s", System.identityHashCode(parent), System.identityHashCode(this));

            // Build the list of context mappings
            for (Map.Entry<StreamReference, Integer> x : contextMappings.entrySet()) {
                if (x.getKey().inputIndex == streamIndex) {
                    list.add(x);
                }
            }

            String labelValue = Integer.toString(streamIndex);
            if (!list.isEmpty()) {
                labelValue += ";" + Output.toString(", ", list,
                        x -> String.format("%d/%d", x.getKey().contextIndex, x.getValue()));
            }

            Map<String, String> attributes = new HashMap<>();
            attributes.put("label", labelValue);

            out.print("[");
            Output.print(out, ", ", attributes.entrySet(),
                    o -> String.format("%s=\"%s\"", o.getKey(), o.getValue()));
            out.println("];");
            streamIndex++;
        }

        return true;
    }

    @Override
    public String toString() {
        return String.format("%s [%s]", getName(), System.identityHashCode(this));
    }

    protected void printDOTNode(PrintStream out, Map<Operator, MutableInt> counts) {
        String attribute = "";
        StringBuilder label = new StringBuilder();
        label.append(getName());

        // If the stream is used in a join, make it dashed
        for (StreamReference x : contextMappings.keySet())
            if (x.inputIndex == -1) {
                attribute = ", style=\"dashed\"";
                break;

            }

        // Verify that each child has this in its parents
        if (counts != null) {
            MutableInt outCount = counts.get(this);
            if (outCount != null) {
                label.append("\\n# = " + outCount.intValue());
                if (outCount.intValue() > 0)
                    attribute += ", peripheries=2";

            }
        }

        label.append("\\n s=" + outputSize());
        label.append("\\n" + System.identityHashCode(this));

        out.format("p%s [label=\"%s\"%s];%n", System.identityHashCode(this), label.toString(), attribute);
    }

    protected String getName() {
        if (name == null)
            this.getClass().getName();
        return name;
    }

    private void ensureConnections(HashMap<Operator, Operator> simplified, HashSet<Operator> visited) {
        if (visited.contains(this))
            return;
        visited.add(this);
        for (Operator parent : getParents())
            parent.ensureConnections(simplified, visited);

        ensureConnections(simplified);
    }

    static public class AMap {

    }

    /**
     * Get a simplified view of the plan
     * @return A map for task operators
     * @param experiment
     */
    public IdentityHashMap<TaskOperator, TaskReference> getTaskOperatorMap(Experiment experiment) {
        IdentityHashMap<TaskOperator, TaskReference> map = new IdentityHashMap<>();
        getTaskOperatorMap(experiment, map, null);
        return map;
    }

    /**
     * Recursion through the structure
     *
     * @see #getTaskOperatorMap(sf.net.experimaestro.manager.experiments.Experiment)
     * @param experiment
     * @param map The current map
     * @param descendant The current descendant
     */
    private void getTaskOperatorMap(Experiment experiment, IdentityHashMap<TaskOperator, TaskReference> map,
            TaskReference descendant) {
        if (this instanceof TaskOperator) {
            TaskOperator task = (TaskOperator) this;
            TaskReference reference = map.get(task);

            if (descendant != null) {
                descendant.addParent(reference);
            }

            if (reference != null) {
                // If we already were in the map, no need to go higher
                return;
            }

            reference = new TaskReference(experiment, task.getPlan().getFactory().getId());
            map.put(task, reference);
            descendant = reference;
        }

        for (Operator parent : getParents()) {
            parent.getTaskOperatorMap(experiment, map, descendant);
        }
    }

    // --- Simplify ---

    /**
     * After changes, this is used ensure that references to changed operators
     * are kept
     *
     * @param map The map for changed operators
     */
    protected void ensureConnections(Map<Operator, Operator> map) {
    }

    public interface Contexts {
        long get(int stream, int index);
    }

    public static class DefaultContexts implements Contexts {
        long[][] contexts;

        public DefaultContexts(long[]... contexts) {
            this.contexts = contexts;
        }

        @Override
        public long get(int stream, int index) {
            return contexts[stream][index];
        }
    }

    static public class ReturnValue {
        Json nodes[];
        Contexts contexts;

        public ReturnValue(Contexts contexts, Json... nodes) {
            this.nodes = nodes;
            this.contexts = contexts;
        }
    }

    public class OperatorIterator extends AbstractIterator<Value> {
        private final MutableInt counter;
        Iterator<ReturnValue> iterator;
        private long id = 0;

        OperatorIterator(ScriptContext scriptContext) {
            iterator = _iterator(scriptContext);
            if (scriptContext.counts() != null)
                scriptContext.counts().put(Operator.this, this.counter = new MutableInt(0));
            else
                this.counter = null;
        }

        @Override
        final protected Value computeNext() {
            // End of stream
            if (!iterator.hasNext())
                return endOfData();

            // Increment the counter (if used)
            if (counter != null)
                counter.increment();

            // Get the next value and sets its id
            ReturnValue next = iterator.next();
            Value value = new Value(next.nodes);
            value.id = id++;

            // Copy context
            if (!contextMappings.isEmpty()) {
                value.context = new long[contextMappings.size()];
                for (Map.Entry<StreamReference, Integer> entry : contextMappings.entrySet()) {
                    StreamReference key = entry.getKey();
                    value.context[entry.getValue()] = key.inputIndex < 0 ? value.id
                            : next.contexts.get(key.inputIndex, key.contextIndex);
                }

            }

            return value;
        }

    }

    @Expose(scope = true)
    @Help("Runs an XQuery against the input: each returned item is a new input")
    public Operator select(Context context, Scriptable scope, String query) throws XPathExpressionException {
        JsonPathFunction function = new JsonPathFunction(query, scope);
        Operator operator = new FunctionOperator(function);
        operator.addParent(this);
        return operator;
    }

}