Java tutorial
/** * Licensed to Odiago, Inc. under one or more contributor license * agreements. See the NOTICE.txt file distributed with this work for * additional information regarding copyright ownership. Odiago, Inc. * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the * License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations * under the License. */ package com.odiago.flumebase.exec.local; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; import org.antlr.runtime.RecognitionException; import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.cloudera.util.Pair; import com.odiago.flumebase.client.ClientConsoleImpl; import com.odiago.flumebase.exec.BuiltInSymbolTable; import com.odiago.flumebase.exec.EventWrapper; import com.odiago.flumebase.exec.ExecEnvironment; import com.odiago.flumebase.exec.FlowElement; import com.odiago.flumebase.exec.FlowId; import com.odiago.flumebase.exec.FlowInfo; import com.odiago.flumebase.exec.HashSymbolTable; import com.odiago.flumebase.exec.OutputElement; import com.odiago.flumebase.exec.QuerySubmitResponse; import com.odiago.flumebase.exec.SymbolTable; import com.odiago.flumebase.flume.EmbeddedFlumeConfig; import com.odiago.flumebase.lang.AssignFieldLabelsVisitor; import com.odiago.flumebase.lang.CountStarVisitor; import com.odiago.flumebase.lang.IdentifyAggregates; import com.odiago.flumebase.lang.JoinKeyVisitor; import com.odiago.flumebase.lang.JoinNameVisitor; import com.odiago.flumebase.lang.ReplaceWindows; import com.odiago.flumebase.lang.TypeChecker; import com.odiago.flumebase.lang.VisitException; import com.odiago.flumebase.parser.ASTGenerator; import com.odiago.flumebase.parser.SQLStatement; import com.odiago.flumebase.plan.FlowSpecification; import com.odiago.flumebase.plan.PlanContext; import com.odiago.flumebase.plan.PropagateSchemas; import com.odiago.flumebase.server.SessionId; import com.odiago.flumebase.server.UserSession; import com.odiago.flumebase.util.DAG; import com.odiago.flumebase.util.DAGOperatorException; import com.odiago.flumebase.util.Ref; import com.odiago.flumebase.util.StringUtils; import com.odiago.flumebase.util.concurrent.ArrayBoundedSelectableQueue; import com.odiago.flumebase.util.concurrent.Select; import com.odiago.flumebase.util.concurrent.Selectable; import com.odiago.flumebase.util.concurrent.SelectableQueue; import com.odiago.flumebase.util.concurrent.SyncSelectableQueue; /** * Standalone local execution environment for flows. */ public class LocalEnvironment extends ExecEnvironment { private static final Logger LOG = LoggerFactory.getLogger(LocalEnvironment.class.getName()); /** Config key specifying whether we automatically watch a flow when we create it or not. */ public static final String AUTO_WATCH_FLOW_KEY = "flumebase.flow.autowatch"; public static final boolean DEFAULT_AUTO_WATCH_FLOW = true; /** Config key specifying the session id of the submitting user for a query. */ public static final String SUBMITTER_SESSION_ID_KEY = "flumebase.query.submitter.session.id"; static class ControlOp { enum Code { AddFlow, // A new flow shold be deployed. CancelFlow, // An entire flow should be canceled. CancelAll, // All flows should be canceled. ShutdownThread, // Stop processing anything else, immediately. Noop, // Do no control action; just service data events. ElementComplete, // A flow element is complete and should be freed. Join, // Add an object to the list of objects to be notified when a // flow is canceled. ListFlows, // Enumerate the running flows. WatchFlow, // Subscribe to a flow's output. UnwatchFlow, // Unsubscribe from a flow's output. GetWatchList, // Get a list of flows being watched by a session. SetFlowName, // Set the name of the output stream for a flow. }; /** What operation should be performed by the worker thread? */ private final Code mOpCode; /** What add'l data is required to do this operation? */ private final Object mDatum; public ControlOp(Code opCode, Object datum) { mOpCode = opCode; mDatum = datum; } public Code getOpCode() { return mOpCode; } public Object getDatum() { return mDatum; } } /** * A request to watch/unwatch a flow. */ private static class WatchRequest { public final boolean mIsWatch; // true for watch, false for unwatch. public final SessionId mSessionId; public final FlowId mFlowId; public WatchRequest(SessionId sessionId, FlowId flowId, boolean isWatch) { mIsWatch = isWatch; mSessionId = sessionId; mFlowId = flowId; } } /** * The thread where the active flows in the local environment actually operate. */ private class LocalEnvThread extends Thread { /** The set of running flows. */ private Map<FlowId, ActiveFlowData> mActiveFlows; /** Mapping from an input queue to the FlowElement it is feeding values to. */ private Map<SelectableQueue<Object>, FlowElement> mInputQueues; /** The selector that lets us read from multiple producers */ private Select<Object> mSelect; /** Unbounded queue used in-thread to post completion events back to the main loop. */ private SelectableQueue<Object> mCompletionEventQueue; /** * Set of queues which should be watched for emptiness; when they transition * to empty, notify the associated downstream element of the upstream element's * closure. */ private Set<SelectableQueue<Object>> mCloseQueues; public LocalEnvThread() { mActiveFlows = new HashMap<FlowId, ActiveFlowData>(); mSelect = new Select<Object>(); mCompletionEventQueue = new SyncSelectableQueue<Object>(); mInputQueues = new HashMap<SelectableQueue<Object>, FlowElement>(); mCloseQueues = new HashSet<SelectableQueue<Object>>(); setName("LocalEnvWorker"); } private void deployFlow(LocalFlow newFlow) throws IOException, InterruptedException { final ActiveFlowData activeFlowData = new ActiveFlowData(newFlow); Configuration flowConf = newFlow.getConf(); if (flowConf.getBoolean(AUTO_WATCH_FLOW_KEY, DEFAULT_AUTO_WATCH_FLOW)) { // Subscribe to this flow before running it, so we guarantee the user // sees all the results. long idNum = flowConf.getLong(SUBMITTER_SESSION_ID_KEY, -1); SessionId submitterSessionId = new SessionId(idNum); UserSession session = getSession(submitterSessionId); if (session != null) { activeFlowData.addSession(session); } else { LOG.warn("Invalid session id number: " + idNum); } } // If we haven't yet started Flume, and the flow requires Flume-based sources, // start Flume. if (newFlow.requiresFlume() && !mFlumeConfig.isRunning()) { mFlumeConfig.start(); } // Open all FlowElements in the flow, in reverse bfs order // (so sinks are always ready before sources). Add the output // queue(s) from the FlowElement to the set of output queues // we monitor for further event processing. try { newFlow.reverseBfs(new DAG.Operator<FlowElementNode>() { public void process(FlowElementNode elemNode) throws DAGOperatorException { FlowElement flowElem = elemNode.getFlowElement(); // All FlowElements that we see will have LocalContext subclass contexts. // Get the output queue from this. LocalContext elemContext = (LocalContext) flowElem.getContext(); elemContext.initControlQueue(mCompletionEventQueue); elemContext.setFlowData(activeFlowData); elemContext.createDownstreamQueues(); List<SelectableQueue<Object>> elemBuffers = elemContext.getDownstreamQueues(); if (null != elemBuffers) { List<FlowElement> downstreams = elemContext.getDownstream(); // Bind each queue to its downstream element. for (int i = 0; i < elemBuffers.size(); i++) { SelectableQueue<Object> elemBuffer = elemBuffers.get(i); if (null != elemBuffer) { FlowElement downstream = downstreams.get(i); mInputQueues.put(elemBuffer, downstream); mSelect.add(elemBuffer); // And watch this queue for updates. } } } try { LOG.debug("Opening flow element of class: " + flowElem.getClass().getName()); flowElem.open(); } catch (IOException ioe) { throw new DAGOperatorException(ioe); } catch (InterruptedException ie) { throw new DAGOperatorException(ie); } } }); } catch (DAGOperatorException doe) { // This is a wrapper exception; unpack and rethrow with the appropriate type. Throwable cause = doe.getCause(); if (cause instanceof IOException) { throw (IOException) cause; } else if (cause instanceof InterruptedException) { throw (InterruptedException) cause; } else { // Don't know how we got here. In any case, do not consider this // flow active. LOG.error("Unexpected DAG exception: " + doe); return; } } mActiveFlows.put(newFlow.getId(), activeFlowData); } private void cancelFlowInner(ActiveFlowData flowData) { // Close all FlowElements in the flow, and remove their output queues // from the set of queues we track. LocalFlow flow = flowData.getFlow(); try { flow.rankTraversal(new DAG.Operator<FlowElementNode>() { public void process(FlowElementNode elemNode) { FlowElement flowElem = elemNode.getFlowElement(); if (!flowElem.isClosed()) { try { flowElem.close(); } catch (IOException ioe) { LOG.error("IOException when closing flow element: " + ioe); } catch (InterruptedException ie) { LOG.error("InterruptedException when closing flow element: " + ie); } } // All FlowElements that we see will have LocalContext subclass contexts. // Get the output queue from this, and remove it from the tracking set. LocalContext elemContext = (LocalContext) flowElem.getContext(); List<SelectableQueue<Object>> outQueues = elemContext.getDownstreamQueues(); if (null != outQueues) { for (SelectableQueue<Object> outQueue : outQueues) { if (null != outQueue) { mSelect.remove(outQueue); mInputQueues.remove(outQueue); mCloseQueues.remove(outQueue); } } } } }); } catch (DAGOperatorException doe) { // Shouldn't get here with this operator. LOG.error("Unexpected dag op exn: " + doe); } // Notify external threads that this flow is complete. flowData.cancel(); } private void cancelFlow(FlowId id) { LOG.info("Closing flow: " + id); ActiveFlowData flowData = mActiveFlows.get(id); if (null == flowData) { LOG.error("Cannot cancel flow: No flow available for id: " + id); return; } cancelFlowInner(flowData); mActiveFlows.remove(id); } /** @return true if 'id' refers to an active flow. */ private boolean isActive(FlowId id) { return mActiveFlows.get(id) != null; } private void cancelAllFlows() { if (mActiveFlows.size() == 0) { return; } LOG.info("Closing all flows"); Set<Map.Entry<FlowId, ActiveFlowData>> flowSet = mActiveFlows.entrySet(); Iterator<Map.Entry<FlowId, ActiveFlowData>> flowIter = flowSet.iterator(); while (flowIter.hasNext()) { Map.Entry<FlowId, ActiveFlowData> entry = flowIter.next(); cancelFlowInner(entry.getValue()); } mActiveFlows.clear(); } /** * Populate the provided map with info about all running flows. This map * is provided by the user process, so we need to synchronize on it before * writing. Furthermore, we must notify the calling thread when it is ready * for consumption. */ private void listFlows(Map<FlowId, FlowInfo> outMap) { assert outMap != null; synchronized (outMap) { for (Map.Entry<FlowId, ActiveFlowData> entry : mActiveFlows.entrySet()) { FlowId id = entry.getKey(); ActiveFlowData activeData = entry.getValue(); outMap.put(id, new FlowInfo(id, activeData.getFlow().getQuery(), activeData.getStreamName())); } // Notify the calling thread when we're done. outMap.notify(); } } /** * Sign up the specified session to watch a given flow. */ private void watch(WatchRequest watchReq) { UserSession userSession = getSession(watchReq.mSessionId); if (null == userSession) { LOG.warn("Cannot watch flow from user session " + watchReq.mSessionId + "; no such session"); return; } ActiveFlowData flow = mActiveFlows.get(watchReq.mFlowId); if (null == flow) { LOG.warn("Cannot watch flow from user session " + watchReq.mSessionId + "; no such flow"); return; } if (watchReq.mIsWatch) { flow.addSession(userSession); } else { flow.removeSession(userSession); } } /** * Populate the specified flowList with a list of FlowIds that are * being watched by the specified sessionId. * The flowList is provided by a client in another thread; synchronize * on it and notify the other thread when the request is complete. */ private void getWatchList(SessionId sessionId, List<FlowId> flowList) { synchronized (flowList) { UserSession session = getSession(sessionId); if (null != session) { for (ActiveFlowData activeFlow : mActiveFlows.values()) { List<UserSession> subscribers = activeFlow.getSubscribers(); if (subscribers.contains(session)) { flowList.add(activeFlow.getFlowId()); } } } else { LOG.error("GetWatchList for sessionId " + sessionId + ": no such session"); } flowList.notify(); // We're done. Wake up the client. } } /** * The specified queue is empty and its upstream element is closed. Notify * the downstream element of this closure, and remove the queue from the * set of things we track. */ private void closeQueue(SelectableQueue<Object> queue, FlowElement flowElem) throws IOException, InterruptedException { flowElem.closeUpstream(); mSelect.remove(queue); mInputQueues.remove(queue); mCloseQueues.remove(queue); } /** * Update the OutputElement of a flow to use a different output stream * name for the output. */ private void setFlowName(final FlowId flowId, final String name) { ActiveFlowData flowData = mActiveFlows.get(flowId); if (null == flowData) { LOG.error("Cannot set flow name for flow id " + flowId + ": no such flow."); return; } try { // NOTE - This assumes a single OutputElement per flow; we find it by // reverseBfs because we assume it's at the end. If there are multiple // OutputElements in the flow, we'll get them all trying to open the // same node... flowData.getFlow().reverseBfs(new DAG.Operator<FlowElementNode>() { public void process(FlowElementNode node) throws DAGOperatorException { FlowElement flowElem = node.getFlowElement(); if (flowElem instanceof OutputElement) { try { ((OutputElement) flowElem).setFlumeTarget(name); } catch (IOException ioe) { throw new DAGOperatorException(ioe); } } } }); } catch (DAGOperatorException doe) { LOG.error("Error setting output stream name: " + doe); } } @Override public void run() { mSelect.add(mControlQueue); // Listen to events on the control queue. mSelect.add(mCompletionEventQueue); try { boolean isFinished = false; while (true) { Selectable<Object> nextQueue = null; try { nextQueue = mSelect.join(); } catch (InterruptedException ie) { // This can happen to notify us we're done processing, etc. } if (null == nextQueue) { continue; } Object nextAction = null; synchronized (nextQueue) { if (nextQueue.canRead()) { try { nextAction = nextQueue.read(); } catch (InterruptedException ie) { // This can happen if we're closing shop fast. We'll just loop around. } } } if (null == nextAction) { continue; } else if (nextAction instanceof ControlOp) { ControlOp nextOp = (ControlOp) nextAction; switch (nextOp.getOpCode()) { case AddFlow: LocalFlow newFlow = (LocalFlow) nextOp.getDatum(); try { deployFlow(newFlow); } catch (Exception e) { LOG.error("Exception deploying flow: " + StringUtils.stringifyException(e)); } finally { // Client waited on this object to know when deployment is done. synchronized (newFlow) { newFlow.setDeployed(true); newFlow.notify(); } } break; case CancelFlow: FlowId cancelId = (FlowId) nextOp.getDatum(); cancelFlow(cancelId); break; case CancelAll: cancelAllFlows(); break; case ShutdownThread: isFinished = true; break; case WatchFlow: WatchRequest watchReq = (WatchRequest) nextOp.getDatum(); watch(watchReq); break; case UnwatchFlow: WatchRequest unwatchReq = (WatchRequest) nextOp.getDatum(); watch(unwatchReq); // the request has the isWatch flag set false. break; case GetWatchList: Pair<SessionId, List<FlowId>> getReq = (Pair<SessionId, List<FlowId>>) nextOp .getDatum(); getWatchList(getReq.getLeft(), getReq.getRight()); break; case SetFlowName: Pair<FlowId, String> flowNameData = (Pair<FlowId, String>) nextOp.getDatum(); setFlowName(flowNameData.getLeft(), flowNameData.getRight()); case Noop: // Don't do any control operation; skip ahead to event processing. break; case ElementComplete: // Remove a specific FlowElement from service; it's done. LocalCompletionEvent completionEvent = (LocalCompletionEvent) nextOp.getDatum(); try { LocalContext context = completionEvent.getContext(); List<SelectableQueue<Object>> downstreamQueues = context.getDownstreamQueues(); List<FlowElement> downstreamElements = context.getDownstream(); if (null == downstreamElements || downstreamElements.size() == 0) { // We have received close() notification from the last element in a flow. // Remove the entire flow from service. // TODO(aaron): Are multiple SinkFlowElemContexts possible per flow? // If so, we need to wait for the last of these... SinkFlowElemContext sinkContext = (SinkFlowElemContext) context; FlowId id = sinkContext.getFlowId(); LOG.info("Processing complete for flow: " + id); if (isActive(id)) { // If the flow is closing naturally, cancel it. If it's // already canceled (inactive), don't do this twice. cancelFlow(id); } } else if (null == downstreamQueues || downstreamQueues.size() == 0) { // Has elements, but no queues. Notify the downstream // FlowElement(s) to close too. for (FlowElement downstream : downstreamElements) { downstream.closeUpstream(); } } else { // May have downstream queues. For each downstream element, close it // immediately if it has no queue, or an empty queue. Otherwise, // watch these queues for emptiness. assert downstreamQueues.size() == downstreamElements.size(); for (int i = 0; i < downstreamElements.size(); i++) { SelectableQueue<Object> downstreamQueue = downstreamQueues.get(i); FlowElement downstreamElement = downstreamElements.get(i); if (downstreamQueue == null) { // Close directly. downstreamElement.closeUpstream(); } else if (downstreamQueue.size() == 0) { // Queue's dry, close it down. closeQueue(downstreamQueue, downstreamElement); } else { // Watch this queue for completion. mCloseQueues.add(downstreamQueue); } } } } catch (IOException ioe) { LOG.error("IOException closing flow element: " + ioe); } catch (InterruptedException ie) { LOG.error("Interruption closing downstream element: " + ie); } break; case Join: FlowJoinRequest joinReq = (FlowJoinRequest) nextOp.getDatum(); FlowId id = joinReq.getFlowId(); Ref<Boolean> waitObj = joinReq.getJoinObj(); ActiveFlowData flowData = mActiveFlows.get(id); if (null == flowData) { // This flow id is already canceled. Return immediately. synchronized (waitObj) { waitObj.item = Boolean.TRUE; waitObj.notify(); } } else { // Mark the waitObj as one we should notify when the flow is canceled. flowData.subscribeToCancelation(waitObj); } break; case ListFlows: Map<FlowId, FlowInfo> resultMap = (Map<FlowId, FlowInfo>) nextOp.getDatum(); listFlows(resultMap); break; } if (isFinished) { // Stop immediately; ignore any further event processing or control work. break; } } else if (nextAction instanceof EventWrapper) { // Process this event with its associated FlowElement. // Look up the correct FlowElement based on the queue->FE map. FlowElement processor = mInputQueues.get(nextQueue); if (null == processor) { LOG.error("No FlowElement for input queue " + nextQueue); } else { try { processor.takeEvent((EventWrapper) nextAction); } catch (IOException ioe) { // TODO(aaron): Encountering an exception mid-flow should cancel the flow. LOG.error("Flow element encountered IOException: " + ioe); } catch (InterruptedException ie) { LOG.error("Flow element encountered InterruptedException: " + ie); } } if (((SelectableQueue<Object>) nextQueue).size() == 0 && mCloseQueues.contains(nextQueue)) { // We just transitioned this FE's input queue to empty, and it was closed // upstream. Notify the downstream element of this closure. try { closeQueue((SelectableQueue<Object>) nextQueue, processor); } catch (IOException ioe) { LOG.error("IOException closing flow element: " + ioe); } catch (InterruptedException ie) { LOG.error("InterruptedException closing flow element: " + ie); } } } else { LOG.error("Do not know what to do with queue element " + nextAction + " of class " + nextAction.getClass().getName()); } } } finally { // Shut down the embedded Flume instance before we exit the thread. if (mFlumeConfig.isRunning()) { mFlumeConfig.stop(); } } } } /** A UserSession describing the console of this process. */ private static final UserSession LOCAL_SESSION; static { LOCAL_SESSION = new UserSession(new SessionId(0), null, new ClientConsoleImpl()); } /** The configuration for this environment instance. */ private Configuration mConf; /** Next flow id to assign to new flows. */ private long mNextFlowId; /** The AST generator used to parse user queries. */ private ASTGenerator mGenerator; /** * Mapping from named outputs to the memory elements fulfilling those outputs. * If clients will be accessing elements of this map, then it needs to be * internally synchronized. */ private Map<String, MemoryOutputElement> mMemoryOutputMap; /** * Manager for the embedded Flume instances in this environment. * References to this object are distributed in the client thread, * but its methods are used only in the execution thread. */ private EmbeddedFlumeConfig mFlumeConfig; /** The thread that does the actual flow execution. */ private LocalEnvThread mLocalThread; /** set to true after connect(). */ private boolean mConnected; /** * Queue of control events passed from the console thread to the worker thread * (e.g., "deploy stream", "cancel stream", etc.) */ private SelectableQueue<Object> mControlQueue; // Actually full of ControlOp instances /** Max len for mControlQueue, or any FlowElement's input queue. */ static final int MAX_QUEUE_LEN = 100; /** * The root symbol table where streams, etc are defined. Used in the * user thread for AST and plan walking. */ private SymbolTable mRootSymbolTable; /** * Main constructor. */ public LocalEnvironment(Configuration conf) { this(conf, new HashSymbolTable(new BuiltInSymbolTable()), new HashMap<String, MemoryOutputElement>(), new EmbeddedFlumeConfig(conf)); } /** * Constructor for testing; allows dependency injection. */ public LocalEnvironment(Configuration conf, SymbolTable rootSymbolTable, Map<String, MemoryOutputElement> memoryOutputMap, EmbeddedFlumeConfig flumeConfig) { mConf = conf; mRootSymbolTable = rootSymbolTable; mMemoryOutputMap = memoryOutputMap; mGenerator = new ASTGenerator(); mNextFlowId = 0; mControlQueue = new ArrayBoundedSelectableQueue<Object>(MAX_QUEUE_LEN); mFlumeConfig = flumeConfig; mLocalThread = this.new LocalEnvThread(); } /** Given a Configuration that has SUBMITTER_SESSION_ID_KEY set, return the * UserSession corresponding to this SessionId. This is used to resolve the * submitter of a LocalFlow, FlowSpecification, etc. */ private UserSession getSessionForConf(Configuration conf) { SessionId id = new SessionId(conf.getLong(SUBMITTER_SESSION_ID_KEY, -1)); return getSession(id); } @Override public SessionId connect() throws IOException { Runtime.getRuntime().addShutdownHook(new ShutdownThread()); mLocalThread.start(); mConnected = true; return new SessionId(0); // Local user is always session 0. } @Override public boolean isConnected() { return mConnected; } @Override public String getEnvName() { return "local"; } /** * Take the user's query, convert it into a local plan, * and execute it. */ @Override public QuerySubmitResponse submitQuery(String query, Map<String, String> options) throws InterruptedException { StringBuilder msgBuilder = new StringBuilder(); FlowId flowId = null; // Build a configuration out of our conf and the user's options. Configuration planConf = new Configuration(mConf); for (Map.Entry<String, String> entry : options.entrySet()) { planConf.set(entry.getKey(), entry.getValue()); } try { // Send the parser's error messages into a buffer rather than stderr. ByteArrayOutputStream errBufferStream = new ByteArrayOutputStream(); PrintStream errStream = new PrintStream(errBufferStream); SQLStatement stmt = mGenerator.parse(query, errStream); errStream.close(); String errMsg = new String(errBufferStream.toByteArray()); msgBuilder.append(errMsg); if (null == stmt) { msgBuilder.append("(Could not parse command)"); return new QuerySubmitResponse(msgBuilder.toString(), null); } stmt.accept(new AssignFieldLabelsVisitor()); stmt.accept(new CountStarVisitor()); // Must be after assign labels, before TC. stmt.accept(new TypeChecker(mRootSymbolTable)); stmt.accept(new ReplaceWindows()); // Must be after TC. stmt.accept(new JoinKeyVisitor()); // Must be after TC. stmt.accept(new JoinNameVisitor()); stmt.accept(new IdentifyAggregates()); // Must be after TC. PlanContext planContext = new PlanContext(); planContext.setConf(planConf); planContext.setSymbolTable(mRootSymbolTable); PlanContext retContext = stmt.createExecPlan(planContext); msgBuilder.append(retContext.getMsgBuilder().toString()); FlowSpecification spec = retContext.getFlowSpec(); if (null != spec) { spec.setQuery(query); spec.setConf(planConf); // Given a flow specification from the AST, run it through // necessary post-processing and optimization phases. spec.bfs(new PropagateSchemas()); if (retContext.isExplain()) { // We just should explain this flow, but not actually add it. msgBuilder.append("Execution plan:\n"); msgBuilder.append(spec.toString()); msgBuilder.append("\n"); } else { flowId = addFlow(spec); } } } catch (VisitException ve) { msgBuilder.append("Error processing command: " + ve.getMessage()); } catch (RecognitionException re) { msgBuilder.append("Error parsing command: " + re.getMessage()); } catch (DAGOperatorException doe) { msgBuilder.append("Error processing plan: " + doe.getMessage()); } return new QuerySubmitResponse(msgBuilder.toString(), flowId); } @Override public FlowId addFlow(FlowSpecification spec) throws InterruptedException { if (null != spec) { // Turn the specification into a physical plan and run it. FlowId flowId = new FlowId(mNextFlowId++); UserSession userSession = getSessionForConf(spec.getConf()); LocalFlowBuilder flowBuilder = new LocalFlowBuilder(flowId, mRootSymbolTable, mFlumeConfig, mMemoryOutputMap, userSession); try { spec.reverseBfs(flowBuilder); } catch (DAGOperatorException doe) { // An exception occurred when creating the physical plan. // LocalFlowBuilder put a message for the user in here; print it // without a stack trace. The flow cannot be executed. userSession.sendErr(doe.getMessage()); return null; } LocalFlow localFlow = flowBuilder.getLocalFlow(); localFlow.setQuery(spec.getQuery()); localFlow.setConf(spec.getConf()); if (localFlow.getRootSet().size() == 0) { // No nodes created (empty flow, or DDL-only flow, etc.) return null; } else { synchronized (localFlow) { mControlQueue.put(new ControlOp(ControlOp.Code.AddFlow, localFlow)); while (!localFlow.isDeployed()) { localFlow.wait(); } } return flowId; } } else { return null; } } @Override public void cancelFlow(FlowId id) throws InterruptedException, IOException { mControlQueue.put(new ControlOp(ControlOp.Code.CancelFlow, id)); } @Override public void joinFlow(FlowId id) throws InterruptedException { Ref<Boolean> joinObj = new Ref<Boolean>(); synchronized (joinObj) { mControlQueue.put(new ControlOp(ControlOp.Code.Join, new FlowJoinRequest(id, joinObj))); joinObj.wait(); } } @Override public boolean joinFlow(FlowId id, long timeout) throws InterruptedException { Ref<Boolean> joinObj = new Ref<Boolean>(); joinObj.item = Boolean.FALSE; synchronized (joinObj) { mControlQueue.put(new ControlOp(ControlOp.Code.Join, new FlowJoinRequest(id, joinObj))); joinObj.wait(timeout); return joinObj.item; } } @Override public void watchFlow(SessionId sessionId, FlowId flowId) throws InterruptedException { mControlQueue.put(new ControlOp(ControlOp.Code.WatchFlow, new WatchRequest(sessionId, flowId, true))); } @Override public void unwatchFlow(SessionId sessionId, FlowId flowId) throws InterruptedException { mControlQueue.put(new ControlOp(ControlOp.Code.UnwatchFlow, new WatchRequest(sessionId, flowId, false))); } @Override public Map<FlowId, FlowInfo> listFlows() throws InterruptedException { Map<FlowId, FlowInfo> outData = new TreeMap<FlowId, FlowInfo>(); synchronized (outData) { mControlQueue.put(new ControlOp(ControlOp.Code.ListFlows, outData)); outData.wait(); } return outData; } @Override public List<FlowId> listWatchedFlows(SessionId sessionId) throws InterruptedException { List<FlowId> outList = new ArrayList<FlowId>(); Pair<SessionId, List<FlowId>> args = new Pair<SessionId, List<FlowId>>(sessionId, outList); synchronized (outList) { mControlQueue.put(new ControlOp(ControlOp.Code.GetWatchList, args)); outList.wait(); } return outList; } @Override public void setFlowName(FlowId flowId, String name) throws InterruptedException { Pair<FlowId, String> nameReq = new Pair<FlowId, String>(flowId, name); mControlQueue.put(new ControlOp(ControlOp.Code.SetFlowName, nameReq)); } /** * Stop the local environment and shut down any flows operating therein. */ @Override public void disconnect(SessionId sessionId) throws InterruptedException { shutdown(); } @Override public void shutdown() throws InterruptedException { mControlQueue.put(new ControlOp(ControlOp.Code.CancelAll, null)); mControlQueue.put(new ControlOp(ControlOp.Code.ShutdownThread, null)); mLocalThread.join(); mConnected = false; } /** * For the LocalEnvironment, there is only the local user session. */ @Override protected UserSession getSession(SessionId id) { return LOCAL_SESSION; } /** * Thread that closes the server impl and cleans up if the server itself is shut down. */ private class ShutdownThread extends Thread { public ShutdownThread() { super("RemoteServerImpl-shutdown"); } @Override public void run() { if (LocalEnvironment.this.mConnected) { LOG.info("Stopping exec environment due to service shutdown..."); try { LocalEnvironment.this.shutdown(); LOG.info("Exec environment completely stopped."); } catch (Exception e) { LOG.error("Exception shutting down LocalEnvironment in shutdown thread: " + StringUtils.stringifyException(e)); } } } } }