Java tutorial
/*********************************************************************************************************************** * Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu) * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. **********************************************************************************************************************/ package eu.stratosphere.nephele.execution; import eu.stratosphere.configuration.Configuration; import eu.stratosphere.core.fs.Path; import eu.stratosphere.core.io.IOReadableWritable; import eu.stratosphere.nephele.deployment.TaskDeploymentDescriptor; import eu.stratosphere.nephele.execution.librarycache.LibraryCacheManager; import eu.stratosphere.nephele.jobgraph.JobID; import eu.stratosphere.nephele.protocols.AccumulatorProtocol; import eu.stratosphere.nephele.services.iomanager.IOManager; import eu.stratosphere.nephele.services.memorymanager.MemoryManager; import eu.stratosphere.nephele.template.AbstractInvokable; import eu.stratosphere.nephele.template.InputSplitProvider; import eu.stratosphere.runtime.io.Buffer; import eu.stratosphere.runtime.io.channels.ChannelID; import eu.stratosphere.runtime.io.channels.OutputChannel; import eu.stratosphere.runtime.io.gates.GateID; import eu.stratosphere.runtime.io.gates.InputGate; import eu.stratosphere.runtime.io.gates.OutputGate; import eu.stratosphere.runtime.io.network.bufferprovider.BufferAvailabilityListener; import eu.stratosphere.runtime.io.network.bufferprovider.BufferProvider; import eu.stratosphere.runtime.io.network.bufferprovider.GlobalBufferPool; import eu.stratosphere.runtime.io.network.bufferprovider.LocalBufferPool; import eu.stratosphere.runtime.io.network.bufferprovider.LocalBufferPoolOwner; import eu.stratosphere.util.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import java.io.IOException; import java.util.ArrayDeque; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Queue; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.FutureTask; /** * The user code of every Nephele task runs inside a <code>RuntimeEnvironment</code> object. The environment provides * important services to the task. It keeps track of setting up the communication channels and provides access to input * splits, memory manager, etc. * <p/> * This class is thread-safe. */ public class RuntimeEnvironment implements Environment, BufferProvider, LocalBufferPoolOwner, Runnable { /** * The log object used for debugging. */ private static final Log LOG = LogFactory.getLog(RuntimeEnvironment.class); /** * The interval to sleep in case a communication channel is not yet entirely set up (in milliseconds). */ private static final int SLEEPINTERVAL = 100; /** * List of output gates created by the task. */ private final List<OutputGate> outputGates = new CopyOnWriteArrayList<OutputGate>(); /** * List of input gates created by the task. */ private final List<InputGate<? extends IOReadableWritable>> inputGates = new CopyOnWriteArrayList<InputGate<? extends IOReadableWritable>>(); /** * Queue of unbound input gate IDs which are required for deserializing an environment in the course of an RPC * call. */ private final Queue<GateID> unboundInputGateIDs = new ArrayDeque<GateID>(); /** * The memory manager of the current environment (currently the one associated with the executing TaskManager). */ private final MemoryManager memoryManager; /** * The io manager of the current environment (currently the one associated with the executing TaskManager). */ private final IOManager ioManager; /** * Class of the task to run in this environment. */ private final Class<? extends AbstractInvokable> invokableClass; /** * Instance of the class to be run in this environment. */ private final AbstractInvokable invokable; /** * The thread executing the task in the environment. */ private volatile Thread executingThread = null; /** * The ID of the job this task belongs to. */ private final JobID jobID; /** * The job configuration encapsulated in the environment object. */ private final Configuration jobConfiguration; /** * The task configuration encapsulated in the environment object. */ private final Configuration taskConfiguration; /** * The input split provider that can be queried for new input splits. */ private final InputSplitProvider inputSplitProvider; /** * The observer object for the task's execution. */ private volatile ExecutionObserver executionObserver = null; /** * The RPC proxy to report accumulators to JobManager */ private AccumulatorProtocol accumulatorProtocolProxy = null; /** * The index of this subtask in the subtask group. */ private final int indexInSubtaskGroup; /** * The current number of subtasks the respective task is split into. */ private final int currentNumberOfSubtasks; /** * The name of the task running in this environment. */ private final String taskName; private LocalBufferPool outputBufferPool; private Map<String, FutureTask<Path>> cacheCopyTasks = new HashMap<String, FutureTask<Path>>(); /** * Creates a new runtime environment object which contains the runtime information for the encapsulated Nephele * task. * * @param jobID the ID of the original Nephele job * @param taskName the name of task running in this environment * @param invokableClass invokableClass the class that should be instantiated as a Nephele task * @param taskConfiguration the configuration object which was attached to the original JobVertex * @param jobConfiguration the configuration object which was attached to the original JobGraph * @throws Exception thrown if an error occurs while instantiating the invokable class */ public RuntimeEnvironment(final JobID jobID, final String taskName, final Class<? extends AbstractInvokable> invokableClass, final Configuration taskConfiguration, final Configuration jobConfiguration) throws Exception { this.jobID = jobID; this.taskName = taskName; this.invokableClass = invokableClass; this.taskConfiguration = taskConfiguration; this.jobConfiguration = jobConfiguration; this.indexInSubtaskGroup = 0; this.currentNumberOfSubtasks = 0; this.memoryManager = null; this.ioManager = null; this.inputSplitProvider = null; this.invokable = this.invokableClass.newInstance(); this.invokable.setEnvironment(this); this.invokable.registerInputOutput(); } /** * Constructs a runtime environment from a task deployment description. * * @param tdd the task deployment description * @param memoryManager the task manager's memory manager component * @param ioManager the task manager's I/O manager component * @param inputSplitProvider the input split provider for this environment * @throws Exception thrown if an error occurs while instantiating the invokable class */ public RuntimeEnvironment(final TaskDeploymentDescriptor tdd, final MemoryManager memoryManager, final IOManager ioManager, final InputSplitProvider inputSplitProvider, AccumulatorProtocol accumulatorProtocolProxy, Map<String, FutureTask<Path>> cpTasks) throws Exception { this.jobID = tdd.getJobID(); this.taskName = tdd.getTaskName(); this.invokableClass = tdd.getInvokableClass(); this.jobConfiguration = tdd.getJobConfiguration(); this.taskConfiguration = tdd.getTaskConfiguration(); this.indexInSubtaskGroup = tdd.getIndexInSubtaskGroup(); this.currentNumberOfSubtasks = tdd.getCurrentNumberOfSubtasks(); this.memoryManager = memoryManager; this.ioManager = ioManager; this.inputSplitProvider = inputSplitProvider; this.accumulatorProtocolProxy = accumulatorProtocolProxy; this.cacheCopyTasks = cpTasks; this.invokable = this.invokableClass.newInstance(); this.invokable.setEnvironment(this); this.invokable.registerInputOutput(); int numOutputGates = tdd.getNumberOfOutputGateDescriptors(); for (int i = 0; i < numOutputGates; ++i) { this.outputGates.get(i).initializeChannels(tdd.getOutputGateDescriptor(i)); } int numInputGates = tdd.getNumberOfInputGateDescriptors(); for (int i = 0; i < numInputGates; i++) { this.inputGates.get(i).initializeChannels(tdd.getInputGateDescriptor(i)); } } /** * Returns the invokable object that represents the Nephele task. * * @return the invokable object that represents the Nephele task */ public AbstractInvokable getInvokable() { return this.invokable; } @Override public JobID getJobID() { return this.jobID; } @Override public GateID getNextUnboundInputGateID() { return this.unboundInputGateIDs.poll(); } @Override public OutputGate createAndRegisterOutputGate() { OutputGate gate = new OutputGate(getJobID(), new GateID(), getNumberOfOutputGates()); this.outputGates.add(gate); return gate; } @Override public void run() { if (invokable == null) { LOG.fatal("ExecutionEnvironment has no Invokable set"); } // Now the actual program starts to run changeExecutionState(ExecutionState.RUNNING, null); // If the task has been canceled in the mean time, do not even start it if (this.executionObserver.isCanceled()) { changeExecutionState(ExecutionState.CANCELED, null); return; } try { ClassLoader cl = LibraryCacheManager.getClassLoader(jobID); Thread.currentThread().setContextClassLoader(cl); this.invokable.invoke(); // Make sure, we enter the catch block when the task has been canceled if (this.executionObserver.isCanceled()) { throw new InterruptedException(); } } catch (Throwable t) { if (!this.executionObserver.isCanceled()) { // Perform clean up when the task failed and has been not canceled by the user try { this.invokable.cancel(); } catch (Throwable t2) { LOG.error(StringUtils.stringifyException(t2)); } } // Release all resources that may currently be allocated by the individual channels releaseAllChannelResources(); if (this.executionObserver.isCanceled() || t instanceof CancelTaskException) { changeExecutionState(ExecutionState.CANCELED, null); } else { changeExecutionState(ExecutionState.FAILED, StringUtils.stringifyException(t)); } return; } // Task finished running, but there may be unconsumed output data in some of the channels changeExecutionState(ExecutionState.FINISHING, null); try { // If there is any unclosed input gate, close it and propagate close operation to corresponding output gate closeInputGates(); // First, close all output gates to indicate no records will be emitted anymore requestAllOutputGatesToClose(); // Wait until all input channels are closed waitForInputChannelsToBeClosed(); // Now we wait until all output channels have written out their data and are closed waitForOutputChannelsToBeClosed(); } catch (Throwable t) { // Release all resources that may currently be allocated by the individual channels releaseAllChannelResources(); if (this.executionObserver.isCanceled() || t instanceof CancelTaskException) { changeExecutionState(ExecutionState.CANCELED, null); } else { changeExecutionState(ExecutionState.FAILED, StringUtils.stringifyException(t)); } return; } // Release all resources that may currently be allocated by the individual channels releaseAllChannelResources(); // Finally, switch execution state to FINISHED and report to job manager changeExecutionState(ExecutionState.FINISHED, null); } @Override public <T extends IOReadableWritable> InputGate<T> createAndRegisterInputGate() { InputGate<T> gate = new InputGate<T>(getJobID(), new GateID(), getNumberOfInputGates()); this.inputGates.add(gate); return gate; } public int getNumberOfOutputGates() { return this.outputGates.size(); } @Override public int getNumberOfInputGates() { return this.inputGates.size(); } @Override public int getNumberOfOutputChannels() { int numberOfOutputChannels = 0; for (int i = 0; i < this.outputGates.size(); ++i) { numberOfOutputChannels += this.outputGates.get(i).getNumChannels(); } return numberOfOutputChannels; } @Override public int getNumberOfInputChannels() { int numberOfInputChannels = 0; for (int i = 0; i < this.inputGates.size(); ++i) { numberOfInputChannels += this.inputGates.get(i).getNumberOfInputChannels(); } return numberOfInputChannels; } /** * Returns the registered input gate with index <code>pos</code>. * * @param pos the index of the input gate to return * @return the input gate at index <code>pos</code> or <code>null</code> if no such index exists */ public InputGate<? extends IOReadableWritable> getInputGate(final int pos) { if (pos < this.inputGates.size()) { return this.inputGates.get(pos); } return null; } /** * Returns the registered output gate with index <code>pos</code>. * * @param index the index of the output gate to return * @return the output gate at index <code>pos</code> or <code>null</code> if no such index exists */ public OutputGate getOutputGate(int index) { if (index < this.outputGates.size()) { return this.outputGates.get(index); } return null; } /** * Returns the thread which is assigned to execute the user code. * * @return the thread which is assigned to execute the user code */ public Thread getExecutingThread() { synchronized (this) { if (this.executingThread == null) { if (this.taskName == null) { this.executingThread = new Thread(this); } else { this.executingThread = new Thread(this, getTaskNameWithIndex()); } } return this.executingThread; } } /** * Blocks until all output channels are closed. * * @throws IOException thrown if an error occurred while closing the output channels * @throws InterruptedException thrown if the thread waiting for the channels to be closed is interrupted */ private void waitForOutputChannelsToBeClosed() throws InterruptedException { // Make sure, we leave this method with an InterruptedException when the task has been canceled if (this.executionObserver.isCanceled()) { return; } for (OutputGate og : this.outputGates) { og.waitForGateToBeClosed(); } } /** * Blocks until all input channels are closed. * * @throws IOException thrown if an error occurred while closing the input channels * @throws InterruptedException thrown if the thread waiting for the channels to be closed is interrupted */ private void waitForInputChannelsToBeClosed() throws IOException, InterruptedException { // Wait for disconnection of all output gates while (true) { // Make sure, we leave this method with an InterruptedException when the task has been canceled if (this.executionObserver.isCanceled()) { throw new InterruptedException(); } boolean allClosed = true; for (int i = 0; i < getNumberOfInputGates(); i++) { final InputGate<? extends IOReadableWritable> eig = this.inputGates.get(i); if (!eig.isClosed()) { allClosed = false; } } if (allClosed) { break; } else { Thread.sleep(SLEEPINTERVAL); } } } /** * Closes all input gates which are not already closed. */ private void closeInputGates() throws IOException, InterruptedException { for (int i = 0; i < this.inputGates.size(); i++) { final InputGate<? extends IOReadableWritable> eig = this.inputGates.get(i); // Important: close must be called on each input gate exactly once eig.close(); } } /** * Requests all output gates to be closed. */ private void requestAllOutputGatesToClose() throws IOException, InterruptedException { for (int i = 0; i < this.outputGates.size(); i++) { this.outputGates.get(i).requestClose(); } } @Override public IOManager getIOManager() { return this.ioManager; } @Override public MemoryManager getMemoryManager() { return this.memoryManager; } @Override public Configuration getTaskConfiguration() { return this.taskConfiguration; } @Override public Configuration getJobConfiguration() { return this.jobConfiguration; } @Override public int getCurrentNumberOfSubtasks() { return this.currentNumberOfSubtasks; } @Override public int getIndexInSubtaskGroup() { return this.indexInSubtaskGroup; } private void changeExecutionState(final ExecutionState newExecutionState, final String optionalMessage) { if (this.executionObserver != null) { this.executionObserver.executionStateChanged(newExecutionState, optionalMessage); } } @Override public String getTaskName() { return this.taskName; } /** * Returns the name of the task with its index in the subtask group and the total number of subtasks. * * @return the name of the task with its index in the subtask group and the total number of subtasks */ public String getTaskNameWithIndex() { return String.format("%s (%d/%d)", this.taskName, getIndexInSubtaskGroup() + 1, getCurrentNumberOfSubtasks()); } /** * Sets the execution observer for this environment. * * @param executionObserver the execution observer for this environment */ public void setExecutionObserver(final ExecutionObserver executionObserver) { this.executionObserver = executionObserver; } @Override public InputSplitProvider getInputSplitProvider() { return this.inputSplitProvider; } @Override public void userThreadStarted(final Thread userThread) { if (this.executionObserver != null) { this.executionObserver.userThreadStarted(userThread); } } @Override public void userThreadFinished(final Thread userThread) { if (this.executionObserver != null) { this.executionObserver.userThreadFinished(userThread); } } /** * Releases the allocated resources (particularly buffer) of input and output channels attached to this task. This * method should only be called after the respected task has stopped running. */ private void releaseAllChannelResources() { for (int i = 0; i < this.inputGates.size(); i++) { this.inputGates.get(i).releaseAllChannelResources(); } for (int i = 0; i < this.outputGates.size(); i++) { this.outputGates.get(i).releaseAllChannelResources(); } } @Override public Set<ChannelID> getOutputChannelIDs() { Set<ChannelID> ids = new HashSet<ChannelID>(); for (OutputGate gate : this.outputGates) { for (OutputChannel channel : gate.channels()) { ids.add(channel.getID()); } } return Collections.unmodifiableSet(ids); } @Override public Set<ChannelID> getInputChannelIDs() { final Set<ChannelID> inputChannelIDs = new HashSet<ChannelID>(); final Iterator<InputGate<? extends IOReadableWritable>> gateIterator = this.inputGates.iterator(); while (gateIterator.hasNext()) { final InputGate<? extends IOReadableWritable> outputGate = gateIterator.next(); for (int i = 0; i < outputGate.getNumberOfInputChannels(); ++i) { inputChannelIDs.add(outputGate.getInputChannel(i).getID()); } } return Collections.unmodifiableSet(inputChannelIDs); } @Override public Set<GateID> getInputGateIDs() { final Set<GateID> inputGateIDs = new HashSet<GateID>(); final Iterator<InputGate<? extends IOReadableWritable>> gateIterator = this.inputGates.iterator(); while (gateIterator.hasNext()) { inputGateIDs.add(gateIterator.next().getGateID()); } return Collections.unmodifiableSet(inputGateIDs); } @Override public Set<GateID> getOutputGateIDs() { final Set<GateID> outputGateIDs = new HashSet<GateID>(); final Iterator<OutputGate> gateIterator = this.outputGates.iterator(); while (gateIterator.hasNext()) { outputGateIDs.add(gateIterator.next().getGateID()); } return Collections.unmodifiableSet(outputGateIDs); } @Override public Set<ChannelID> getOutputChannelIDsOfGate(final GateID gateID) { OutputGate outputGate = null; final Iterator<OutputGate> gateIterator = this.outputGates.iterator(); while (gateIterator.hasNext()) { final OutputGate candidateGate = gateIterator.next(); if (candidateGate.getGateID().equals(gateID)) { outputGate = candidateGate; break; } } if (outputGate == null) { throw new IllegalArgumentException("Cannot find output gate with ID " + gateID); } final Set<ChannelID> outputChannelIDs = new HashSet<ChannelID>(); for (int i = 0; i < outputGate.getNumChannels(); ++i) { outputChannelIDs.add(outputGate.getChannel(i).getID()); } return Collections.unmodifiableSet(outputChannelIDs); } @Override public Set<ChannelID> getInputChannelIDsOfGate(final GateID gateID) { InputGate<? extends IOReadableWritable> inputGate = null; final Iterator<InputGate<? extends IOReadableWritable>> gateIterator = this.inputGates.iterator(); while (gateIterator.hasNext()) { final InputGate<? extends IOReadableWritable> candidateGate = gateIterator.next(); if (candidateGate.getGateID().equals(gateID)) { inputGate = candidateGate; break; } } if (inputGate == null) { throw new IllegalArgumentException("Cannot find input gate with ID " + gateID); } final Set<ChannelID> inputChannelIDs = new HashSet<ChannelID>(); for (int i = 0; i < inputGate.getNumberOfInputChannels(); ++i) { inputChannelIDs.add(inputGate.getInputChannel(i).getID()); } return Collections.unmodifiableSet(inputChannelIDs); } public List<OutputGate> outputGates() { return this.outputGates; } public List<InputGate<? extends IOReadableWritable>> inputGates() { return this.inputGates; } @Override public AccumulatorProtocol getAccumulatorProtocolProxy() { return accumulatorProtocolProxy; } public void addCopyTaskForCacheFile(String name, FutureTask<Path> copyTask) { this.cacheCopyTasks.put(name, copyTask); } @Override public Map<String, FutureTask<Path>> getCopyTask() { return this.cacheCopyTasks; } @Override public BufferProvider getOutputBufferProvider() { return this; } // ----------------------------------------------------------------------------------------------------------------- // BufferProvider methods // ----------------------------------------------------------------------------------------------------------------- @Override public Buffer requestBuffer(int minBufferSize) throws IOException { return this.outputBufferPool.requestBuffer(minBufferSize); } @Override public Buffer requestBufferBlocking(int minBufferSize) throws IOException, InterruptedException { return this.outputBufferPool.requestBufferBlocking(minBufferSize); } @Override public int getBufferSize() { return this.outputBufferPool.getBufferSize(); } @Override public BufferAvailabilityRegistration registerBufferAvailabilityListener(BufferAvailabilityListener listener) { return this.outputBufferPool.registerBufferAvailabilityListener(listener); } // ----------------------------------------------------------------------------------------------------------------- // LocalBufferPoolOwner methods // ----------------------------------------------------------------------------------------------------------------- @Override public int getNumberOfChannels() { return getNumberOfOutputChannels(); } @Override public void setDesignatedNumberOfBuffers(int numBuffers) { this.outputBufferPool.setNumDesignatedBuffers(numBuffers); } @Override public void clearLocalBufferPool() { this.outputBufferPool.destroy(); } @Override public void registerGlobalBufferPool(GlobalBufferPool globalBufferPool) { if (this.outputBufferPool == null) { this.outputBufferPool = new LocalBufferPool(globalBufferPool, 1); } } @Override public void logBufferUtilization() { LOG.info(String.format("\t%s: %d available, %d requested, %d designated", getTaskNameWithIndex(), this.outputBufferPool.numAvailableBuffers(), this.outputBufferPool.numRequestedBuffers(), this.outputBufferPool.numDesignatedBuffers())); } @Override public void reportAsynchronousEvent() { this.outputBufferPool.reportAsynchronousEvent(); } }