Java tutorial
/* This file is part of the LarKC platform http://www.larkc.eu/ Copyright 2010 LarKC project consortium Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package eu.larkc.core.executor; import java.lang.reflect.InvocationTargetException; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.UUID; import org.apache.http.MethodNotSupportedException; import org.gridlab.gat.security.SecurityContext; import org.jgrapht.graph.DefaultEdge; import org.jgrapht.graph.SimpleDirectedGraph; import org.openrdf.model.Statement; import org.openrdf.query.MalformedQueryException; import org.openrdf.query.QueryEvaluationException; import org.openrdf.repository.RepositoryException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import eu.larkc.core.Larkc; import eu.larkc.core.data.SetOfStatements; import eu.larkc.core.data.workflow.EndpointNode; import eu.larkc.core.data.workflow.GraphNode; import eu.larkc.core.data.workflow.IllegalWorkflowGraphException; import eu.larkc.core.data.workflow.InputNode; import eu.larkc.core.data.workflow.MultiplePluginParametersException; import eu.larkc.core.data.workflow.OutputNode; import eu.larkc.core.data.workflow.PathNode; import eu.larkc.core.data.workflow.SparqlWorkflowDescription; import eu.larkc.core.endpoint.Endpoint; import eu.larkc.core.endpoint.EndpointException; import eu.larkc.core.endpoint.EndpointFactory; import eu.larkc.core.endpoint.EndpointShutdownException; import eu.larkc.core.executor.path.Input; import eu.larkc.core.executor.path.Output; import eu.larkc.core.executor.path.Path; import eu.larkc.core.pluginManager.ControlMessage; import eu.larkc.core.pluginManager.Message; import eu.larkc.core.pluginManager.PluginManager; import eu.larkc.core.pluginManager.local.LocalPluginManager; import eu.larkc.core.pluginManager.remote.GAT.GatPluginManager; import eu.larkc.core.pluginManager.remote.GAT.GatResourceDescription; import eu.larkc.core.pluginManager.remote.GAT.ResourceDescription; import eu.larkc.core.pluginManager.remote.GAT.UriList; import eu.larkc.core.pluginManager.remote.Servlet.Tomcat.JeePluginManager; import eu.larkc.core.pluginManager.remote.Servlet.Tomcat.JeeResourceDescription; import eu.larkc.core.queue.Queue; import eu.larkc.plugin.Plugin; /** * The executor is responsible for actually constructing a workflow (or * 'plumbing' the pipelines so to speak). The executor needs a workflow * (described in RDF) to operate. Workflows can be generated by hand or by any * plug-in (which is then usually called a 'decider'). * * @author Christoph Fuchs */ public class Executor { private UUID uuid; private static Logger logger = LoggerFactory.getLogger(Executor.class); private SparqlWorkflowDescription sparqlWorkflowDescription; private Map<String, Plugin> pluginInstances; private Map<String, PluginManager> pluginManagerInstances; private Map<String, Path> pathInstances; private Map<String, Endpoint> endpointMap; /** * Constructor. Initializes an executor for the given workflow, with the * given <code>decider</code> in charge. * * @param wd * the workflow to be executed. Everything that is needed to * execute the workflow has to be included in this workflow * description. This can either be a full description of the * workflow or some special parameters used by the decider (to * actually create the workflow). * @throws IllegalWorkflowGraphException * @throws MultiplePluginParametersException * @throws RepositoryException */ public Executor(SetOfStatements wd) throws IllegalWorkflowGraphException, MultiplePluginParametersException, RepositoryException { uuid = UUID.randomUUID(); pluginInstances = new HashMap<String, Plugin>(); pluginManagerInstances = new HashMap<String, PluginManager>(); pathInstances = new HashMap<String, Path>(); endpointMap = new HashMap<String, Endpoint>(); sparqlWorkflowDescription = new SparqlWorkflowDescription(wd); initializeWorkflow(); startPluginManagers(); Larkc.addWorkflowExecutorMapping(uuid, this); logger.debug("Initialized executor with UUID {}", this.getId()); } /** * Initializes the workflow by calling initializePlugins, * initializeEndpoints and initializeSourcesAndSinks. * * @param sparqlWorkflowDescription * the workflow description. * @throws IllegalWorkflowGraphException */ private void initializeWorkflow() throws IllegalWorkflowGraphException { try { initializePlugins(); initializeInputsAndOutputs(); initializeEndpoints(); // for testing purpose // for (Entry<String, PluginManager> entry : pluginManagerInstances // .entrySet()) { // for (Entry<String, Path> id : pathInstances.entrySet()) { // if (entry.getValue().getInputQueues(id.getKey()) != null) // logger.debug("{} :: {}", entry.getKey(), entry // .getValue().getInputQueues(id.getKey()).size()); // } // } } catch (IllegalWorkflowGraphException e) { logger.error(e.getLocalizedMessage()); e.printStackTrace(); throw (e); } catch (IllegalArgumentException e) { logger.error(e.getLocalizedMessage()); e.printStackTrace(); } catch (InstantiationException e) { logger.error(e.getLocalizedMessage()); e.printStackTrace(); } catch (IllegalAccessException e) { logger.error(e.getLocalizedMessage()); e.printStackTrace(); } catch (InvocationTargetException e) { logger.error(e.getLocalizedMessage()); e.printStackTrace(); } catch (RepositoryException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (MalformedQueryException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (QueryEvaluationException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (SecurityException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (NoSuchMethodException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (EndpointException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (MultiplePluginParametersException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * This method starts all plugin managers that are in the * pluginManagerInstances map. */ private void startPluginManagers() { for (PluginManager pluginManager : pluginManagerInstances.values()) { pluginManager.start(); } } /** * This method saves the connection between inputs and outputs. * * @throws InstantiationException * @throws IllegalWorkflowGraphException * @throws QueryEvaluationException * @throws MalformedQueryException * @throws RepositoryException */ private void initializeInputsAndOutputs() throws InstantiationException, IllegalWorkflowGraphException, RepositoryException, MalformedQueryException, QueryEvaluationException { Map<String, Input> inputInstances = new HashMap<String, Input>(); Map<String, Output> outputInstances = new HashMap<String, Output>(); // initialize all outputs Map<String, OutputNode> outputs = sparqlWorkflowDescription.getOutputs(); Output output; String outputPluginId; PluginManager outputPluginManager; Queue<SetOfStatements> outputOutputQueue; for (Entry<String, OutputNode> entry : outputs.entrySet()) { output = new Output(); outputPluginId = entry.getValue().getPluginId(); outputOutputQueue = new Queue<SetOfStatements>(); outputPluginManager = pluginManagerInstances.get(outputPluginId); outputPluginManager.addOutputQueue(outputOutputQueue, entry.getKey()); logger.debug("Added output queue for {} ({})", pluginInstances.get(outputPluginId).toString(), entry.getKey()); output.setPathOutputQueue(outputOutputQueue); output.setPluginManager(outputPluginManager); outputInstances.put(entry.getKey(), output); } Map<String, PathNode> stringPaths = sparqlWorkflowDescription.getPaths(); // initialize all inputs Map<String, InputNode> inputs = sparqlWorkflowDescription.getInputs(); Input input; List<String> inputPluginIds; PluginManager inputPluginManager; Queue<SetOfStatements> inputInputQueue; for (Entry<String, InputNode> entry : inputs.entrySet()) { input = new Input(); inputPluginIds = entry.getValue().getPluginIds(); for (String pluginId : inputPluginIds) { inputPluginManager = pluginManagerInstances.get(pluginId); for (Entry<String, PathNode> pathEntry : stringPaths.entrySet()) { inputInputQueue = new Queue<SetOfStatements>(); inputPluginManager.addInputQueue(inputInputQueue, pathEntry.getKey()); logger.debug("Added input queue for {} ({})", pluginInstances.get(pluginId).toString(), pathEntry.getKey()); if (pathEntry.getKey().equals(entry.getKey())) { input.addPathInputQueue(inputInputQueue); logger.debug("Added path input queue for {} ({})", pluginInstances.get(pluginId).toString(), pathEntry.getKey()); } } } inputInstances.put(entry.getKey(), input); } // initialize all paths Input tmpInput; Output tmpOutput; for (Entry<String, PathNode> path : stringPaths.entrySet()) { tmpInput = inputInstances.get(path.getKey()); tmpOutput = outputInstances.get(path.getKey()); if (tmpInput != null && tmpOutput != null) { pathInstances.put(path.getKey(), new Path(path.getKey(), tmpInput, tmpOutput)); logger.debug("Added path {}", path.getKey()); } else { throw new InstantiationException("Path could not be created: input or output is null"); } } } /** * Uses the plugin registry to initialize the provided plug-ins. * * @throws IllegalArgumentException * @throws InstantiationException * @throws IllegalAccessException * @throws InvocationTargetException * @throws IllegalWorkflowGraphException * @throws MultiplePluginParametersException * @throws QueryEvaluationException * @throws MalformedQueryException * @throws RepositoryException */ private void initializePlugins() throws IllegalArgumentException, InstantiationException, IllegalAccessException, InvocationTargetException, IllegalWorkflowGraphException, MultiplePluginParametersException, RepositoryException, MalformedQueryException, QueryEvaluationException { if (!Larkc.isInitialized()) { throw new IllegalStateException( "LarKC was not initialized. Initialize LarKC before trying to initialize the plug-ins of your workflow."); } SimpleDirectedGraph<GraphNode, DefaultEdge> graph = sparqlWorkflowDescription.getWorkflowGraph(); Plugin plugin; PluginManager pluginManager; for (GraphNode node : graph.vertexSet()) { // check that node has a URI and an ID if (node.getPluginID() == null) throw new IllegalWorkflowGraphException("Node " + node + " has no PluginID"); if (node.getUri() == null) throw new IllegalWorkflowGraphException("Node " + node + " has no URI"); plugin = Larkc.getPluginRegistry().getNewPluginInstance(node.getUri()); if (plugin == null) throw new IllegalWorkflowGraphException("Could not initialise plugin for node " + node); // initialize plugin with workflow parameters plugin.initialise(sparqlWorkflowDescription.getPluginParameters(node.getPluginID())); pluginInstances.put(node.getPluginID(), plugin); // initialize plugin managers SetOfStatements deploymentHostProperties = sparqlWorkflowDescription .getDeploymentProperties(node.getPluginID()); if (deploymentHostProperties == null) { logger.debug("Initializing LocalPluginManager"); pluginManager = new LocalPluginManager(plugin); pluginManagerInstances.put(node.getPluginID(), pluginManager); } else { try { // get a statement with resource type Statement resourceType = ResourceDescription.getRemoteHostType(deploymentHostProperties); if (resourceType.getObject().equals(UriList.GAT_RESOURCE)) { logger.debug("Initializing GatPluginManager"); GatResourceDescription gatResDecr = ResourceDescription .getGatResourceDescriptionFromProperties(deploymentHostProperties); SecurityContext gatSecCon = ResourceDescription .getGatSecurityContextFromProperties(deploymentHostProperties); if (gatSecCon != null) pluginManager = new GatPluginManager(plugin, gatResDecr, gatSecCon); else pluginManager = new GatPluginManager(plugin, gatResDecr); pluginManagerInstances.put(node.getPluginID(), pluginManager); } else if (resourceType.getObject().equals(UriList.JEE_RESOURCE)) { logger.debug("Initializing JeePluginManager"); JeeResourceDescription jeeResDecr = ResourceDescription .getJeeResourceDescriptionFromProperties(deploymentHostProperties); jeeResDecr.URI = jeeResDecr.URI + "/" + plugin.getClass().getSimpleName() + "/LarkcPlugin"; pluginManager = new JeePluginManager(plugin, jeeResDecr); pluginManagerInstances.put(node.getPluginID(), pluginManager); } } catch (Exception e) { logger.error("Can not initialize remote host properties for plug-in {}. Local manager is used", node.getPluginID()); pluginManager = new LocalPluginManager(plugin); pluginManagerInstances.put(node.getPluginID(), pluginManager); } } } GraphNode sourceNode; GraphNode targetNode; Queue<SetOfStatements> queue; PluginManager sourcePluginManager; PluginManager targetPluginManager; Set<String> paths = sparqlWorkflowDescription.getPaths().keySet(); for (DefaultEdge edge : graph.edgeSet()) { logger.debug("Creating edge {}", edge.toString()); sourceNode = graph.getEdgeSource(edge); targetNode = graph.getEdgeTarget(edge); sourcePluginManager = pluginManagerInstances.get(sourceNode.getPluginID()); targetPluginManager = pluginManagerInstances.get(targetNode.getPluginID()); for (String path : paths) { queue = new Queue<SetOfStatements>(); logger.debug( "Created queue: " + path + ", " + pluginInstances.get(sourceNode.getPluginID()).toString() + ", " + pluginInstances.get(targetNode.getPluginID()).toString()); sourcePluginManager.addOutputQueue(queue, path); targetPluginManager.addInputQueue(queue, path); // logger.debug("Connected {} -> {}", sourceNode.getUri(), // targetNode.getUri()); } targetPluginManager.addPrevious(sourcePluginManager); } } /** * Execute the given workflow by invoking the plugins. * * @param query * the query */ public void execute(SetOfStatements query) { String pathId = getOnlyPathId(); if (pathId != null) { execute(query, pathId); } else { logger.debug("No path defined to execute the query!"); throw new RuntimeException("No path defined to execute the query!"); } } /** * Execute the given workflow by invoking the plugins. * * @param query * the query * @param pathId * the ID of the path that should be called */ public void execute(SetOfStatements query, String pathId) { logger.debug("Got query to execute."); Path path = pathInstances.get(pathId); if (path != null) { logger.debug("Stop previous execution ..."); for (PluginManager pluginManager : pluginManagerInstances.values()) { pluginManager.stopWaiting(); } path.execute(query); } else { logger.debug("No path defined to execute the query!"); throw new RuntimeException("No path defined to execute the query!"); } } /** * Get the next x results. * * @param offset * @param limit * @return a set of statements containing the specified results * @throws MethodNotSupportedException */ public SetOfStatements getNextResults(int offset, int limit) throws MethodNotSupportedException { throw new MethodNotSupportedException( "In the current platform version specific pulling of N results is not implemented."); } /** * Getter. Retrieves the uuid of this executor. * * @return the uuid * @see java.util.UUID */ public UUID getId() { return uuid; } /** * Removes the executor and the endpoints from the mapping and stops all * endpoints. * * @throws EndpointShutdownException * thrown if one or more endpoints are unable to shutdown * @throws IllegalWorkflowGraphException * @throws QueryEvaluationException * @throws MalformedQueryException * @throws RepositoryException */ public void terminate() throws EndpointShutdownException, RepositoryException, MalformedQueryException, QueryEvaluationException, IllegalWorkflowGraphException { stopExecution(); stopEndpoints(); endpointMap.clear(); pathInstances.clear(); } /** * Stops the execution of the workflow but does not terminate the executor. * * @throws IllegalWorkflowGraphException * @throws QueryEvaluationException * @throws MalformedQueryException * @throws RepositoryException */ public void stopExecution() throws RepositoryException, MalformedQueryException, QueryEvaluationException, IllegalWorkflowGraphException { for (Entry<String, PluginManager> entry : pluginManagerInstances.entrySet()) { for (String path : sparqlWorkflowDescription.getPaths().keySet()) { entry.getValue().accept(new ControlMessage(Message.STOP, path)); entry.getValue().stopWaiting(); } } pluginManagerInstances.clear(); pluginInstances.clear(); } /** * Returns the next results of the workflow. * * @param pathId * the ID of the path that sould be called * * @return the next results of the workflow. */ public SetOfStatements getNextResults(String pathId) { Path path = pathInstances.get(pathId); return path.getNextResults(); } /** * Returns the next results of the workflow. * * @return the next results of the workflow. */ public SetOfStatements getNextResults() { String pathId = getOnlyPathId(); if (pathId != null) { return getNextResults(pathId); } else { logger.debug("No path defined to execute the query!"); throw new RuntimeException("No path defined to execute the query!"); } } /** * Checks if only one path is created and returns the ID of this path, * otherwise null * * @return the ID of the path if only one is defined, null otherwise */ private String getOnlyPathId() { if (pathInstances.size() == 1) { for (String id : pathInstances.keySet()) { return id; } } return null; } /** * This method initializes all endpoints that refer to this instance of the * executor. * * @param sparqlWorkflowDescription * The workflow description * @throws ClassNotFoundException * @throws InvocationTargetException * @throws IllegalAccessException * @throws InstantiationException * @throws NoSuchMethodException * @throws IllegalArgumentException * @throws SecurityException * @throws EndpointException * @throws QueryEvaluationException * @throws MalformedQueryException * @throws RepositoryException * @throws Exception */ private void initializeEndpoints() throws SecurityException, IllegalArgumentException, NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException, ClassNotFoundException, EndpointException, RepositoryException, MalformedQueryException, QueryEvaluationException { Map<String, EndpointNode> endpoints = sparqlWorkflowDescription.getEndpoints(); Endpoint endpoint; int port = 8183; for (Entry<String, EndpointNode> entry : endpoints.entrySet()) { logger.debug("Initializing endpoint {}", entry.getValue().getType()); endpoint = EndpointFactory.constructEndpoint(entry.getValue().getType(), this); endpoint.setPathId(entry.getValue().getPath()); endpoint.start(Larkc.findNextFreePort(port)); endpointMap.put(entry.getKey(), endpoint); port++; } } /** * This method returns the endpoint with the specific URN. * * @param endpointUrn * URN of the endpoint * * @return endpoint the specific endpoint */ public Endpoint getEndpoint(String endpointUrn) { return endpointMap.get(endpointUrn); } /** * Removes all endpoints of this executor. * * @throws EndpointShutdownException * if one or more endpoints are unable to shutdown */ public void stopEndpoints() throws EndpointShutdownException { for (Entry<String, Endpoint> entry : endpointMap.entrySet()) { entry.getValue().stop(); } } /** * Method returns all path IDs. * * @return path IDs */ @Deprecated public Set<String> getPathIds() { return pathInstances.keySet(); } /* * (non-Javadoc) * * @see java.lang.Object#toString() */ @Override public String toString() { return getId().toString(); } /** * Returns the available endpoints of this executor * * @return the set of available endpoint keys, which are simply identifier * strings */ public Set<String> getAvailableEndpoints() { return endpointMap.keySet(); } }