Java tutorial
/* * Copyright 2010 LinkedIn, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package azkaban.flow; import azkaban.common.utils.Props; import azkaban.jobs.Status; import org.apache.commons.lang.StringUtils; import org.joda.time.DateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; /** * A grouping of executable flows. * * For example, if you had two functions f(x) and g(x) that you wanted to execute together, you * could conceivably create a new function h(x) = { f(x); g(x); }. That is essentially what * this class does with ExecutableFlow objects. * * It will run the sub flows in parallel and aggregate all of their "return properties" into a single * properties object. It aggregates the properties in the order that the flows are specified on the * constructor, so order does matter if subflows return properties with the same key (last one wins) * * You should never really have to create one of these directly. Try to use MultipleDependencyExecutableFlow * instead. */ public class GroupedExecutableFlow implements ExecutableFlow { private final Object sync = new Object(); private final String id; private final ExecutableFlow[] flows; private final ExecutableFlow[] sortedFlows; private volatile Status jobState; private volatile List<FlowCallback> callbacksToCall; private volatile DateTime startTime; private volatile DateTime endTime; private volatile GroupedExecutableFlow.GroupedFlowCallback theGroupCallback; private volatile Map<String, Throwable> exceptions = new HashMap<String, Throwable>(); private volatile Props parentProps; private volatile Props returnProps; private final String name; public GroupedExecutableFlow(String id, ExecutableFlow... flows) { this.id = id; this.flows = flows; this.sortedFlows = Arrays.copyOf(this.flows, this.flows.length); Arrays.sort(this.sortedFlows, new Comparator<ExecutableFlow>() { @Override public int compare(ExecutableFlow o1, ExecutableFlow o2) { return o1.getName().compareTo(o2.getName()); } }); String[] names = new String[flows.length]; for (int i = 0; i < flows.length; i++) { names[i] = flows[i].getName(); } name = StringUtils.join(names, " + "); jobState = Status.READY; updateState(); callbacksToCall = new ArrayList<FlowCallback>(); theGroupCallback = new GroupedFlowCallback(); switch (jobState) { case SUCCEEDED: case COMPLETED: case FAILED: DateTime theStartTime = new DateTime(); DateTime theEndTime = new DateTime(0); for (ExecutableFlow flow : flows) { final DateTime subFlowStartTime = flow.getStartTime(); if (theStartTime.isAfter(subFlowStartTime)) { theStartTime = subFlowStartTime; } final DateTime subFlowEndTime = flow.getEndTime(); if (subFlowEndTime != null && subFlowEndTime.isAfter(theEndTime)) { theEndTime = subFlowEndTime; } } setAndVerifyParentProps(); startTime = theStartTime; endTime = theEndTime; break; default: // Check for Flows that are "RUNNING" boolean allRunning = true; List<ExecutableFlow> runningFlows = new ArrayList<ExecutableFlow>(); DateTime thisStartTime = null; for (ExecutableFlow flow : flows) { if (flow.getStatus() != Status.RUNNING) { allRunning = false; final DateTime subFlowStartTime = flow.getStartTime(); if (subFlowStartTime != null && subFlowStartTime.isBefore(thisStartTime)) { thisStartTime = subFlowStartTime; } } else { runningFlows.add(flow); } } if (allRunning) { jobState = Status.RUNNING; } for (ExecutableFlow runningFlow : runningFlows) { final DateTime subFlowStartTime = runningFlow.getStartTime(); if (subFlowStartTime != null && subFlowStartTime.isBefore(thisStartTime)) { thisStartTime = subFlowStartTime; } } setAndVerifyParentProps(); startTime = thisStartTime; endTime = null; // Make sure everything is initialized before leaking the pointer to "this". // This is just installing the callback in an already running flow. for (ExecutableFlow runningFlow : runningFlows) { runningFlow.execute(parentProps, theGroupCallback); } } } @Override public String getId() { return id; } @Override public String getName() { return name; } @Override public Props getReturnProps() { return returnProps; } @Override public void execute(Props parentProps, final FlowCallback callback) { if (parentProps == null) { parentProps = new Props(); } synchronized (sync) { if (this.parentProps == null) { this.parentProps = parentProps; } else if (jobState != Status.COMPLETED && !this.parentProps.equalsProps(parentProps)) { throw new IllegalArgumentException(String.format( "%s.execute() called with multiple differing parentProps objects. " + "Call reset() before executing again with a different Props object. this.parentProps[%s], parentProps[%s]", getClass().getSimpleName(), this.parentProps, parentProps)); } switch (jobState) { case READY: jobState = Status.RUNNING; callbacksToCall.add(callback); break; case RUNNING: callbacksToCall.add(callback); return; case COMPLETED: case SUCCEEDED: case IGNORED: callback.completed(Status.SUCCEEDED); return; case FAILED: callback.completed(Status.FAILED); return; } } if (startTime == null) { startTime = new DateTime(); } for (ExecutableFlow flow : flows) { if (jobState != Status.FAILED) { try { flow.execute(this.parentProps, theGroupCallback); } catch (RuntimeException e) { final List<FlowCallback> callbacks; synchronized (sync) { jobState = Status.FAILED; callbacks = callbacksToCall; } callCallbacks(callbacks, Status.FAILED); throw e; } } } } @Override public boolean cancel() { boolean retVal = true; for (ExecutableFlow flow : flows) { retVal &= flow.cancel(); } return retVal; } private void updateState() { synchronized (sync) { boolean allComplete = true; for (ExecutableFlow flow : flows) { switch (flow.getStatus()) { case FAILED: jobState = Status.FAILED; returnProps = new Props(); return; case COMPLETED: case SUCCEEDED: continue; default: allComplete = false; } } if (allComplete) { jobState = Status.SUCCEEDED; returnProps = new Props(); for (ExecutableFlow flow : flows) { returnProps = new Props(returnProps, flow.getReturnProps()); } returnProps.logProperties("Output properties for " + getName()); } } } @Override public Status getStatus() { return jobState; } @Override public boolean reset() { synchronized (sync) { switch (jobState) { case RUNNING: return false; default: jobState = Status.READY; callbacksToCall = new ArrayList<FlowCallback>(); theGroupCallback = new GroupedFlowCallback(); parentProps = null; returnProps = null; startTime = null; endTime = null; exceptions.clear(); } } return true; } @Override public boolean markCompleted() { synchronized (sync) { switch (jobState) { case RUNNING: return false; default: jobState = Status.COMPLETED; parentProps = new Props(); returnProps = new Props(); } } return true; } @Override public boolean hasChildren() { return true; } @Override public List<ExecutableFlow> getChildren() { return Arrays.asList(sortedFlows); } @Override public String toString() { return "GroupedExecutableFlow{" + "flows=" + (flows == null ? null : Arrays.asList(flows)) + ", jobState=" + jobState + '}'; } @Override public DateTime getStartTime() { return startTime; } @Override public DateTime getEndTime() { return endTime; } @Override public Props getParentProps() { return parentProps; } @Override public Map<String, Throwable> getExceptions() { return exceptions; } private void callCallbacks(final List<FlowCallback> callbacksList, final Status status) { if (endTime == null) { endTime = new DateTime(); } for (FlowCallback callback : callbacksList) { try { callback.completed(status); } catch (RuntimeException t) { // TODO: Figure out how to use the logger to log that a callback threw an exception. } } } private void setAndVerifyParentProps() { for (ExecutableFlow flow : flows) { if (flow.getStatus() == Status.READY) { continue; } final Props childsParentProps = flow.getParentProps(); if (parentProps == null) { parentProps = childsParentProps; } else { if (childsParentProps != null && !parentProps.equalsProps(childsParentProps)) { throw new IllegalStateException( String.format("Parent props differ for sub flows. Flow Id[%s]", id)); } } } } private class GroupedFlowCallback implements FlowCallback { private final AtomicBoolean notifiedCallbackAlready; public GroupedFlowCallback() { this.notifiedCallbackAlready = new AtomicBoolean(false); } @Override public void progressMade() { final List<FlowCallback> callbackList; synchronized (sync) { callbackList = callbacksToCall; } for (FlowCallback flowCallback : callbackList) { flowCallback.progressMade(); } } @Override public void completed(final Status status) { final List<FlowCallback> callbackList; synchronized (sync) { updateState(); callbackList = callbacksToCall; // Get the reference before leaving the synchronized } if (jobState == Status.SUCCEEDED && notifiedCallbackAlready.compareAndSet(false, true)) { callCallbacks(callbackList, Status.SUCCEEDED); } else if (jobState == Status.FAILED && notifiedCallbackAlready.compareAndSet(false, true)) { for (ExecutableFlow flow : flows) { exceptions.putAll(flow.getExceptions()); } callCallbacks(callbackList, Status.FAILED); } else { for (FlowCallback flowCallback : callbackList) { flowCallback.progressMade(); } } } } }