Java tutorial
/* * Copyright [2013-2014] PayPal Software Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package ml.shifu.guagua.yarn; import java.io.IOException; import java.lang.Thread.UncaughtExceptionHandler; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; import ml.shifu.guagua.GuaguaConstants; import ml.shifu.guagua.GuaguaRuntimeException; import ml.shifu.guagua.hadoop.io.GuaguaInputSplit; import ml.shifu.guagua.yarn.util.GsonUtils; import ml.shifu.guagua.yarn.util.YarnUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.lib.input.FileSplit; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; import org.apache.hadoop.yarn.client.api.async.NMClientAsync; import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.jboss.netty.bootstrap.ServerBootstrap; import org.jboss.netty.channel.ChannelEvent; import org.jboss.netty.channel.ChannelHandlerContext; import org.jboss.netty.channel.ChannelPipeline; import org.jboss.netty.channel.ChannelPipelineFactory; import org.jboss.netty.channel.ChannelState; import org.jboss.netty.channel.ChannelStateEvent; import org.jboss.netty.channel.Channels; import org.jboss.netty.channel.ExceptionEvent; import org.jboss.netty.channel.MessageEvent; import org.jboss.netty.channel.SimpleChannelUpstreamHandler; import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory; import org.jboss.netty.handler.codec.serialization.ClassResolvers; import org.jboss.netty.handler.codec.serialization.ObjectDecoder; import org.jboss.netty.handler.codec.serialization.ObjectEncoder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Maps; /** * {@link GuaguaAppMaster} is application master to launch master and worker tasks. * * <p> * This app master is used to check and launch all tasks, not to run the master task of distributed training. Master * task is run on another container. * * <p> * TODO: Web monitor is not supported in current app master. * * <p> * Fail-over is added like mapreduce. If one container is failed, it will be launched until 4 times by default. * * <p> * In each container, use a number start from 1 to mark as the id of the container for fail-over. */ public class GuaguaAppMaster { private static final Logger LOG = LoggerFactory.getLogger(GuaguaAppMaster.class); /** Exit code for YARN containers that were manually killed/aborted */ private static final int YARN_ABORT_EXIT_STATUS = -100; /** Exit code for successfully run YARN containers */ private static final int YARN_SUCCESS_EXIT_STATUS = 0; /** millis to sleep between heartbeats during long loops */ private static final int SLEEP_BETWEEN_HEARTBEATS_MSECS = 900; /** * Container id for current master container */ private ContainerId masterContainerId; /** * App attempt id */ private ApplicationAttemptId appAttemptId; /** * Whether master is done. */ private volatile boolean done; /** * Yarn conf */ private Configuration yarnConf; /** * Number of completed containers. */ private AtomicInteger completedCount; /** * Number of failed containers. */ private AtomicInteger failedCount; /** * Number of allocated containers. */ private AtomicInteger allocatedCount; /** * Number of successful containers. */ private AtomicInteger successfulCount; /** * Number of completed containers. */ private int containersToLaunch; /** * executor to launch container. */ private ExecutorService executor; /** * executor to check whether task is time out. */ private ExecutorService taskTimeoutExecutor; /** * Like mapred.task.timeout, if no update on this time, container will be killed. */ private long taskTimeOut = GuaguaYarnConstants.DEFAULT_TIME_OUT; /** * Heap memory setting for worker container. */ private int heapPerContainer; /** * Handle to communicate with resource manager. */ private AMRMClientAsync<ContainerRequest> amRmClient; /** * Handle to communicate with the Node Manager */ private NMClientAsync nmClientAsync; /** * Listen to process the response from the Node Manager */ private NMCallbackHandler containerListener; /** * A reusable map of resources already in HDFS for each task to copy-to-local env and use to launch each * GuaguaYarnTask. */ private static Map<String, LocalResource> localResources; /** * For status update for clients - yet to be implemented\\ * Hostname of the container */ private String appMasterHostname; /** Port on which the app master listens for status updates from clients */ private int appMasterRpcPort = 1234; /** Tracking url to which app master publishes info for clients to monitor */ private String appMasterTrackingUrl = ""; /** * Setting container args */ private String containerArgs; private List<InputSplit> inputSplits; private ApplicationId appId; private Map<Integer, List<Container>> partitionContainerMap; private Map<String, Integer> containerPartitionMap; private static enum PartitionStatus { INIT, SUCCESSFUL, FAILED, RETRY, } private Map<Integer, PartitionStatus> partitionStatusMap; private List<Integer> failedPartitions; private AtomicInteger partitionIndex; private int maxContainerAttempts; private int totalIterations; private ByteBuffer allTokens; private int rpcPort = GuaguaYarnConstants.DEFAULT_STATUS_RPC_PORT; private String rpcHostName; private static final Object LOCK = new Object(); private Map<Integer, GuaguaIterationStatus> partitionProgress; private ServerBootstrap rpcServer; static { // pick up new conf XML file and populate it with stuff exported from client Configuration.addDefaultResource(GuaguaYarnConstants.GUAGUA_CONF_FILE); } /** * Construct the GuaguappMaster, populate fields using env vars and set up by YARN framework in this execution * container. * * @param cId * the ContainerId * @param aId * the ApplicationAttemptId */ public GuaguaAppMaster(ContainerId cId, ApplicationAttemptId aId, Configuration conf) { try { this.rpcHostName = this.appMasterHostname = InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException e) { LOG.error("Error in getting local host name.", e); } this.masterContainerId = cId; // future good stuff will need me to operate. this.appAttemptId = aId; this.appId = this.getAppAttemptId().getApplicationId(); this.yarnConf = conf; this.completedCount = new AtomicInteger(0); this.failedCount = new AtomicInteger(0); this.allocatedCount = new AtomicInteger(0); this.successfulCount = new AtomicInteger(0); this.partitionContainerMap = new ConcurrentHashMap<Integer, List<Container>>(); this.containerPartitionMap = new ConcurrentHashMap<String, Integer>(); this.partitionStatusMap = new ConcurrentHashMap<Integer, GuaguaAppMaster.PartitionStatus>(); this.partitionIndex = new AtomicInteger(0); this.failedPartitions = new CopyOnWriteArrayList<Integer>(); ; this.maxContainerAttempts = this.getYarnConf().getInt( GuaguaYarnConstants.GUAGUA_YARN_MAX_CONTAINER_ATTEMPTS, GuaguaYarnConstants.GUAGUA_YARN_DEFAULT_MAX_CONTAINER_ATTEMPTS); this.heapPerContainer = this.getYarnConf().getInt(GuaguaYarnConstants.GUAGUA_CHILD_MEMORY, GuaguaYarnConstants.GUAGUA_CHILD_DEFAULT_MEMORY); this.totalIterations = this.getYarnConf().getInt(GuaguaConstants.GUAGUA_ITERATION_COUNT, 1); String containerArgs = this.getYarnConf().get(GuaguaYarnConstants.GUAGUA_YARN_CONTAINER_ARGS); if (containerArgs == null) { containerArgs = GuaguaYarnConstants.GUAGUA_YARN_DEFAULT_CONTAINER_JAVA_OPTS; } else { containerArgs = GuaguaYarnConstants.GUAGUA_YARN_DEFAULT_CONTAINER_JAVA_OPTS + " " + containerArgs; } this.containerArgs = containerArgs; this.rpcPort = getYarnConf().getInt(GuaguaYarnConstants.GUAGUA_YARN_STATUS_RPC_PORT, GuaguaYarnConstants.DEFAULT_STATUS_RPC_PORT); this.partitionProgress = new ConcurrentHashMap<Integer, GuaguaIterationStatus>(); this.executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); this.taskTimeoutExecutor = Executors.newSingleThreadExecutor(); this.taskTimeOut = getYarnConf().getLong(GuaguaYarnConstants.GUAGUA_TASK_TIMEOUT, GuaguaYarnConstants.DEFAULT_TIME_OUT); LOG.info("{}:{}", taskTimeOut, GuaguaYarnConstants.DEFAULT_TIME_OUT); LOG.info("GuaguaAppMaster for ContainerId {} ApplicationAttemptId {}", cId, aId); } /** * Coordinates all requests for guagua's worker/master task containers, and manages application liveness heartbeat, * completion status, teardown, etc. * * @return success or failure */ public boolean run() throws YarnException, IOException { boolean success = false; try { // 1. get input from conf, generate input splits like GuaguaMapReduce // 2. store all splits into conf and export to hdfs // 3. for each conntainer, according to container host and split host to select a partition to the container // add transfer partition as program args. // 4. Store <partition, failed container number> // 5. If one container failed, try to request another container // 6. for containers get, try failed partition firstly prepareInputSplits(); // set tokens to make app master and task work well. getAllTokens(); registerRMCallBackHandler(); registerNMCallbackHandler(); registerAMToRM(); startRPCServer(); startTaskTimeoutExecutor(); madeAllContainerRequestToRM(); LOG.info("Wait to finish .."); while (!isDone()) { try { Thread.sleep(SLEEP_BETWEEN_HEARTBEATS_MSECS); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } } LOG.info("Done {}", isDone()); } catch (Throwable t) { LOG.error("Error in AppMaster run.", t); throw new GuaguaRuntimeException(t); } finally { shutdown(); success = finish(); } return success; } private void startTaskTimeoutExecutor() { this.taskTimeoutExecutor.submit(new Runnable() { @Override public void run() { while (true) { try { Thread.sleep(GuaguaAppMaster.this.taskTimeOut); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } LOG.debug(GuaguaAppMaster.this.partitionProgress.toString()); for (Map.Entry<Integer, GuaguaIterationStatus> entry : GuaguaAppMaster.this.partitionProgress .entrySet()) { GuaguaIterationStatus status = entry.getValue(); // doesn't work in the first iteration if (status.getTime() != 0l && status.getCurrentIteration() != 1 && (System.currentTimeMillis() - status.getTime()) > GuaguaAppMaster.this.taskTimeOut) { List<Container> containers = GuaguaAppMaster.this.partitionContainerMap .get(entry.getKey()); Container container = containers.get(containers.size() - 1); LOG.info( "Container {} is timeout with timeout period {}, will be killed by node manager {}.", container.getId(), GuaguaAppMaster.this.taskTimeOut, container.getNodeId()); GuaguaAppMaster.this.getNmClientAsync().stopContainerAsync(container.getId(), container.getNodeId()); } } } } }); } protected void shutdown() { // if we get here w/o problems, the executor is already long finished. if (null != getExecutor() && !getExecutor().isTerminated()) { LOG.info("Forcefully terminating executors with done ={}", isDone()); getExecutor().shutdownNow(); // force kill, especially if got here by throw } if (this.rpcServer != null) { this.rpcServer.shutdown(); this.rpcServer.releaseExternalResources(); } if (this.taskTimeoutExecutor != null) { this.taskTimeoutExecutor.shutdownNow(); } } /** * Start rpc server which is used to update progress. */ private void startRPCServer() { this.rpcServer = new ServerBootstrap(new NioServerSocketChannelFactory( Executors.newFixedThreadPool(GuaguaYarnConstants.DEFAULT_STATUS_RPC_SERVER_THREAD_COUNT), Executors.newCachedThreadPool(new MasterThreadFactory()))); // Set up the pipeline factory. this.rpcServer.setPipelineFactory(new ChannelPipelineFactory() { public ChannelPipeline getPipeline() throws Exception { return Channels.pipeline(new ObjectEncoder(), new ObjectDecoder(ClassResolvers.cacheDisabled(getClass().getClassLoader())), new ServerHandler()); } }); // Bind and start to accept incoming connections. this.rpcServer.bind(new InetSocketAddress(rpcPort)); } /** * The master thread factory. Main feature is to print error log of worker thread. */ private static class MasterThreadFactory implements ThreadFactory { static final AtomicInteger poolNumber = new AtomicInteger(1); final ThreadGroup group; final AtomicInteger threadNumber = new AtomicInteger(1); final String namePrefix; MasterThreadFactory() { SecurityManager s = System.getSecurityManager(); group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup(); namePrefix = "pool-" + poolNumber.getAndIncrement() + "-thread-"; } public Thread newThread(Runnable r) { Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0); if (t.isDaemon()) { t.setDaemon(false); } if (t.getPriority() != Thread.NORM_PRIORITY) { t.setPriority(Thread.NORM_PRIORITY); } t.setUncaughtExceptionHandler(new UncaughtExceptionHandler() { @Override public void uncaughtException(Thread t, Throwable e) { LOG.warn("Error message in thread {} with error message {}, error root cause {}.", t, e, e.getCause()); // print stack??? } }); return t; } } /** * {@link ServerHandler} is used to receive message and update progress for this yarn app. */ private class ServerHandler extends SimpleChannelUpstreamHandler { @Override public void handleUpstream(ChannelHandlerContext ctx, ChannelEvent e) throws Exception { if (e instanceof ChannelStateEvent && ((ChannelStateEvent) e).getState() != ChannelState.INTEREST_OPS) { LOG.debug(e.toString()); } super.handleUpstream(ctx, e); } @Override public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) { GuaguaIterationStatus status = GsonUtils.fromJson(e.getMessage().toString(), GuaguaIterationStatus.class); LOG.info("Receive RPC status:{}", status); synchronized (LOCK) { GuaguaAppMaster.this.partitionProgress.put(status.getPartition(), status); } if (status.isKillContainer()) { List<Container> containers; synchronized (LOCK) { containers = GuaguaAppMaster.this.partitionContainerMap.get(status.getPartition()); } LOG.info("containers:{}", containers); Container container = containers.get(containers.size() - 1); LOG.info("Container {} in node {} is killed because of straggler condition.", container.getId(), container.getNodeId()); GuaguaAppMaster.this.getNmClientAsync().stopContainerAsync(container.getId(), container.getNodeId()); } } @Override public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) { e.getChannel().close(); } } /** * Prepare input splits for containers */ private void prepareInputSplits() throws IOException { this.inputSplits = getNewSplits(getYarnConf()); this.setContainersToLaunch(this.inputSplits.size()); LOG.info("Input split size including master: {}", this.inputSplits.size()); } public List<InputSplit> getNewSplits(Configuration conf) throws IOException { int masters = conf.getInt(GuaguaConstants.GUAGUA_MASTER_NUMBER, GuaguaConstants.DEFAULT_MASTER_NUMBER); int size = getYarnConf().getInt(GuaguaConstants.GUAGUA_WORKER_NUMBER, 0) + masters; List<InputSplit> newSplits = new ArrayList<InputSplit>(size); for (int i = 1; i <= size; i++) { newSplits.add( GsonUtils.fromJson(getYarnConf().get(GuaguaYarnConstants.GUAGUA_YARN_INPUT_SPLIT_PREFIX + i), GuaguaInputSplit.class)); this.partitionProgress.put(i, new GuaguaIterationStatus()); } return newSplits; } /** * Populate allTokens with the tokens received */ private void getAllTokens() throws IOException { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (LOG.isDebugEnabled()) { LOG.debug("Token type : {}", token.getKind()); } if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } this.allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); } /** * Register RM callback and start listening */ private void registerRMCallBackHandler() { AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler(); setAmRMClient(AMRMClientAsync.createAMRMClientAsync(1000, allocListener)); getAmRMClient().init(getYarnConf()); getAmRMClient().start(); } /** * Register NM callback and start listening */ private void registerNMCallbackHandler() { setContainerListener(new NMCallbackHandler()); setNmClientAsync(new NMClientAsyncImpl(getContainerListener())); getNmClientAsync().init(getYarnConf()); getNmClientAsync().start(); } /** * Register AM to RM * * @return AM register response */ private RegisterApplicationMasterResponse registerAMToRM() throws YarnException { // register Application Master with the YARN Resource Manager so we can begin requesting resources. try { if (UserGroupInformation.isSecurityEnabled()) { LOG.info("SECURITY ENABLED "); } RegisterApplicationMasterResponse response = getAmRMClient().registerApplicationMaster( this.appMasterHostname, this.appMasterRpcPort, this.appMasterTrackingUrl); return response; } catch (IOException ioe) { throw new IllegalStateException("GuaguaAppMaster failed to register with RM.", ioe); } } /** * Add all containers' request */ private void madeAllContainerRequestToRM() { // Setup ask for containers from RM // Send request for containers to RM Until we get our fully allocated quota, we keep on polling RM for // containers. Keep looping until all the containers are launched and shell script executed on them ( regardless // of success/failure). for (int i = 0; i < getContainersToLaunch(); i++) { ContainerRequest containerAsk = setupContainerAskForRM(); getAmRMClient().addContainerRequest(containerAsk); } } private void madeOneContainerRequestToRM() { ContainerRequest containerAsk = setupContainerAskForRM(); getAmRMClient().addContainerRequest(containerAsk); } /** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ private ContainerRequest setupContainerAskForRM() { // setup requirements for hosts, request containers firstly and then check allocated containers and splits to // get data locality. // TODO, better here to requests according to hosts of splits. Priority pri = Records.newRecord(Priority.class); pri.setPriority(GuaguaYarnConstants.GUAGUA_YARN_DEFAULT_PRIORITY); Resource capability = Records.newRecord(Resource.class); capability.setMemory(getHeapPerContainer()); capability.setVirtualCores(getYarnConf().getInt(GuaguaYarnConstants.GUAGUA_YARN_TASK_VCORES, GuaguaYarnConstants.GUAGUA_YARN_TASK_DEFAULT_VCORES)); ContainerRequest request = new ContainerRequest(capability, null, null, pri); LOG.info("Requested container ask: {}", request.toString()); return request; } /** * Call when the application is done * * @return if all containers succeed */ private boolean finish() { // When the application completes, it should stop all running containers LOG.info("Application completed. Stopping running containers"); getNmClientAsync().stop(); // When the application completes, it should send a finish application // signal to the RM LOG.info("Application completed. Signalling finish to RM"); FinalApplicationStatus appStatus; String appMessage = null; boolean success = true; if (getSuccessfulCount().get() == getContainersToLaunch()) { appStatus = FinalApplicationStatus.SUCCEEDED; } else { appStatus = FinalApplicationStatus.FAILED; appMessage = String.format("Diagnostics total=%s, completed=%s, failed=%s.", getContainersToLaunch(), getCompletedCount().get(), getFailedCount().get()); success = false; } try { getAmRMClient().unregisterApplicationMaster(appStatus, appMessage, this.appMasterTrackingUrl); } catch (YarnException ex) { LOG.error("Failed to unregister application", ex); } catch (IOException e) { LOG.error("Failed to unregister application", e); } getAmRMClient().stop(); return success; } /** * CallbackHandler to process RM async calls */ private class RMCallbackHandler implements AMRMClientAsync.CallbackHandler { @Override public void onContainersCompleted(List<ContainerStatus> completedContainers) { LOG.info("Got response from RM for container ask, completedCnt={}", completedContainers.size()); for (ContainerStatus containerStatus : completedContainers) { LOG.info("Got container status for containerID={}, state={}, exitStatus={}, diagnostics={}.", containerStatus.getContainerId(), containerStatus.getState(), containerStatus.getExitStatus(), containerStatus.getDiagnostics()); if (!GuaguaAppMaster.this.containerPartitionMap .containsKey(containerStatus.getContainerId().toString())) { getCompletedCount().incrementAndGet(); LOG.info("Why such container {} is started, no partition. Exited with status:{}", containerStatus.getContainerId(), containerStatus.getExitStatus()); continue; } int partition = GuaguaAppMaster.this.containerPartitionMap .get(containerStatus.getContainerId().toString()); if (GuaguaAppMaster.this.partitionContainerMap.get(partition) .size() >= GuaguaAppMaster.this.maxContainerAttempts) { setDone(true); LOG.info("One partition {} has more than max attempt {} ", partition, GuaguaAppMaster.this.maxContainerAttempts); return; } switch (containerStatus.getExitStatus()) { case YARN_SUCCESS_EXIT_STATUS: GuaguaAppMaster.this.partitionStatusMap.put(partition, PartitionStatus.SUCCESSFUL); getSuccessfulCount().incrementAndGet(); break; case YARN_ABORT_EXIT_STATUS: LOG.info("YARN_ABORT_EXIT_STATUS: Container id {} exits with {}", containerStatus.getContainerId(), YARN_ABORT_EXIT_STATUS); break; // not success or fail default: LOG.info("default: Container id {} exits with {}", containerStatus.getContainerId(), containerStatus.getExitStatus()); GuaguaAppMaster.this.partitionStatusMap.put(partition, PartitionStatus.FAILED); GuaguaAppMaster.this.failedPartitions.add(partition); GuaguaAppMaster.this.madeOneContainerRequestToRM(); getFailedCount().incrementAndGet(); break; } getCompletedCount().incrementAndGet(); } if (getSuccessfulCount().get() == getContainersToLaunch()) { setDone(true); LOG.info("All container compeleted. done = {} ", isDone()); } else { LOG.info( "After completion of one conatiner. current status is: completedCount:{} containersToLaunch:{} successfulCount:{} failedCount:{}.", getCompletedCount().get(), getContainersToLaunch(), getSuccessfulCount().get(), getFailedCount().get()); } } @Override public void onContainersAllocated(List<Container> allocatedContainers) { LOG.info("Got response from RM for container ask, allocatedCnt={}", allocatedContainers.size()); getAllocatedCount().addAndGet(allocatedContainers.size()); LOG.info("Total allocated # of container so far {} : allocated out of required {}.", getAllocatedCount().get(), getContainersToLaunch()); startContainerLaunchingThreads(allocatedContainers); } @Override public void onShutdownRequest() { setDone(true); getAmRMClient().stop(); } @Override public void onNodesUpdated(List<NodeReport> updatedNodes) { } @Override public float getProgress() { // set progress to deliver to RM on next heartbeat int sum = 0, totalSum = 0; synchronized (LOCK) { for (Map.Entry<Integer, GuaguaIterationStatus> entry : GuaguaAppMaster.this.partitionProgress .entrySet()) { sum += entry.getValue().getCurrentIteration(); totalSum += GuaguaAppMaster.this.totalIterations; } return (sum * 1.0f) / totalSum; } } @Override public void onError(Throwable e) { setDone(true); getAmRMClient().stop(); } } /** * For each container successfully allocated, attempt to set up and launch a Guagua worker/master task. * * @param allocatedContainers * the containers we have currently allocated. */ private void startContainerLaunchingThreads(final List<Container> allocatedContainers) { Map<String, List<Container>> hostContainterMap = getHostContainersMap(allocatedContainers); int size = allocatedContainers.size(); while (size > 0) { int currentPartition = getCurrentPartition(); if (currentPartition == -1) { LOG.warn("Request too many resources. TODO, remove containers no needed."); for (Container container : allocatedContainers) { GuaguaAppMaster.this.getAmRMClient().releaseAssignedContainer(container.getId()); } break; } Container container = getDataLocalityContainer(hostContainterMap, currentPartition); if (container == null) { container = allocatedContainers.get(0); } allocatedContainers.remove(container); LOG.info( "Launching command on a new container., containerId={}, containerNode={}, containerPort={}, containerNodeURI={}, containerResourceMemory={}", container.getId(), container.getNodeId().getHost(), container.getNodeId().getPort(), container.getNodeHttpAddress(), container.getResource().getMemory()); List<Container> list = this.partitionContainerMap.get(currentPartition); if (list == null) { list = new ArrayList<Container>(); } list.add(container); this.partitionContainerMap.put(currentPartition, list); this.containerPartitionMap.put(container.getId().toString(), currentPartition); this.partitionStatusMap.put(currentPartition, PartitionStatus.INIT); LaunchContainerRunnable runnableLaunchContainer = new LaunchContainerRunnable(container, getContainerListener(), currentPartition); getExecutor().execute(runnableLaunchContainer); size = allocatedContainers.size(); } } private Map<String, List<Container>> getHostContainersMap(final List<Container> allocatedContainers) { Map<String, List<Container>> hostContainterMap = new HashMap<String, List<Container>>(); for (Container container : allocatedContainers) { String host = container.getNodeId().getHost(); List<Container> containers = hostContainterMap.get(host); if (containers == null) { containers = new ArrayList<Container>(); } containers.add(container); hostContainterMap.put(host, containers); } return hostContainterMap; } /** * Find a container with the same host for input split. Not a good implementation for data locality. Check * map-reduce implementation. * * TODO RACK-LOCAL implementation */ private Container getDataLocalityContainer(Map<String, List<Container>> hostContainterMap, int currentPartition) { GuaguaInputSplit inputSplit = (GuaguaInputSplit) (this.inputSplits.get(currentPartition - 1)); String host = null; FileSplit[] fileSplits = inputSplit.getFileSplits(); if (fileSplits != null) { try { host = fileSplits[0].getLocations()[0]; } catch (Exception mayNotHappen) { host = null; } } List<Container> containers = hostContainterMap.get(host); Container container = null; if (containers != null && !containers.isEmpty()) { container = containers.remove(0); hostContainterMap.put(host, containers); LOG.info("find a container {} with host {} for partition {} and split {}.", container, host, currentPartition, inputSplit); return container; } // if not find a container, try to choose the first one. Set<Entry<String, List<Container>>> entrySet = hostContainterMap.entrySet(); String firstHost = null; List<Container> firstContainers = null; for (Entry<String, List<Container>> entry : entrySet) { firstHost = entry.getKey(); firstContainers = entry.getValue(); if (firstContainers != null && !firstContainers.isEmpty()) { container = firstContainers.remove(0); break; } } hostContainterMap.put(firstHost, firstContainers); LOG.info("find a container {} with host {} for partition {} and split {}.", container, host, currentPartition, inputSplit); return container; } private int getCurrentPartition() { LOG.info("failed container request size:{} {}", this.failedPartitions.size(), this.failedPartitions); Iterator<Integer> it = this.failedPartitions.iterator(); // Launch and start the container on a separate thread to keep the main thread unblocked as all containers // may not be allocated at one go. int currentPartition = 0; if (it.hasNext()) { currentPartition = it.next(); // because we use CopyOnWriteArrayList, we can remove object in iteration this.failedPartitions.remove(Integer.valueOf(currentPartition)); LOG.info("failed container request size after remove:{} {}", this.failedPartitions.size(), this.failedPartitions); } else { LOG.info("partitionIndex{} containersToLaunch {}", this.partitionIndex.get(), this.containersToLaunch); if (this.partitionIndex.get() >= this.containersToLaunch) { return -1; } currentPartition = this.partitionIndex.addAndGet(1); } return currentPartition; } /** * Thread to connect to the {@link ContainerManager} and launch the container that will house one of our Guagua * worker (or master) tasks. */ private class LaunchContainerRunnable implements Runnable { /** Allocated container */ private Container container; /** NM listener */ private NMCallbackHandler containerListener; private final int partition; /** * Constructor. * * @param container * Allocated container * @param containerListener * container listener. */ public LaunchContainerRunnable(final Container container, NMCallbackHandler containerListener, int partition) { this.container = container; this.containerListener = containerListener; this.partition = partition; } /** * Connects to CM, sets up container launch context for shell command and eventually dispatches the container * start request to the CM. */ @Override public void run() { // Connect to ContainerManager // configure the launcher for the guagua task it will host ContainerLaunchContext ctx = buildContainerLaunchContext(); // request CM to start this container as spec'd in ContainerLaunchContext this.containerListener.addContainer(this.container.getId(), this.container); getNmClientAsync().startContainerAsync(this.container, ctx); } /** * Boilerplate to set up the ContainerLaunchContext to tell the Container Manager how to launch our guagua task * in the execution container we have already allocated. * * @return a populated ContainerLaunchContext object. */ private ContainerLaunchContext buildContainerLaunchContext() { LOG.info("Setting up container launch container for containerid={}", container.getId()); ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class); // args inject the CLASSPATH, heap MB, and TaskAttemptID for launched task final List<String> commands = generateShellExecCommand(); LOG.info("Conatain launch Commands :{}" + commands); launchContext.setCommands(commands); // Set up tokens for the container too. We are populating them mainly for NodeManagers to be able to // download any files in the distributed file-system. The tokens are otherwise also useful in cases, for // e.g., when one is running a "hadoop dfs" like command launchContext.setTokens(allTokens.slice()); // Set the environment variables to inject into remote task's container buildEnvironment(launchContext); // Set the local resources: just send the copies already in HDFS launchContext.setLocalResources(getTaskResourceMap()); return launchContext; } /** * Generates our command line string used to launch our guagua tasks. * * @return the BASH shell commands to launch the job. */ private List<String> generateShellExecCommand() { String programArgs = new StringBuilder(300) .append(getAppAttemptId().getApplicationId().getClusterTimestamp()).append(" ") .append(getAppAttemptId().getApplicationId().getId()).append(" ") .append(this.container.getId().getId()).append(" ").append(getAppAttemptId().getAttemptId()) .append(" ").append(this.partition).append(" ").append(GuaguaAppMaster.this.rpcHostName) .append(" ").append(GuaguaAppMaster.this.rpcPort).toString(); return YarnUtils.getCommand(GuaguaYarnTask.class.getName(), GuaguaAppMaster.this.containerArgs, programArgs, getHeapPerContainer() + ""); } /** * Utility to populate the environment vars we wish to inject into the new containter's env when the guagua BSP * task is executed. * * @param launchContext * the launch context which will set our environment vars in the app master's execution container. */ private void buildEnvironment(final ContainerLaunchContext launchContext) { Map<String, String> classPathForEnv = Maps.newHashMap(); // pick up the local classpath so when we instantiate a Configuration remotely. YarnUtils.addLocalClasspathToEnv(classPathForEnv, getYarnConf()); // set this map of env vars into the launch context. launchContext.setEnvironment(classPathForEnv); } } /** * CallbackHandler to process NM async calls */ private class NMCallbackHandler implements NMClientAsync.CallbackHandler { /** List of containers */ private ConcurrentMap<ContainerId, Container> containers = new ConcurrentHashMap<ContainerId, Container>(); /** * Add a container * * @param containerId * id of container * @param container * container object */ public void addContainer(ContainerId containerId, Container container) { this.containers.putIfAbsent(containerId, container); } @Override public void onContainerStopped(ContainerId containerId) { LOG.info("Succeeded to stop Container {}", containerId); this.containers.remove(containerId); } @Override public void onContainerStatusReceived(ContainerId containerId, ContainerStatus containerStatus) { LOG.info("Container Status: id={}, status={}", containerId, containerStatus); } @Override public void onContainerStarted(ContainerId containerId, Map<String, ByteBuffer> allServiceResponse) { LOG.info("Succeeded to start Container {}", containerId); Container container = this.containers.get(containerId); if (container != null) { getNmClientAsync().getContainerStatusAsync(containerId, container.getNodeId()); } } @Override public void onStartContainerError(ContainerId containerId, Throwable t) { LOG.error(String.format("Failed to start Container %s", containerId), t); this.containers.remove(containerId); } @Override public void onGetContainerStatusError(ContainerId containerId, Throwable t) { LOG.error(String.format("Failed to query the status of Container %s", containerId), t); } @Override public void onStopContainerError(ContainerId containerId, Throwable t) { LOG.error(String.format("Failed to stop Container %s", containerId), t); this.containers.remove(containerId); } } /** * Lazily compose the map of jar and file names to LocalResource records for inclusion in GuaguaYarnTask container * requests. Can re-use the same map as guagua tasks need identical HDFS-based resources (jars etc.) to run. * * @return the resource map for a ContainerLaunchContext */ private synchronized Map<String, LocalResource> getTaskResourceMap() { // Set the local resources: just send the copies already in HDFS if (null == localResources) { localResources = Maps.newHashMap(); try { // if you have to update the Conf for export to tasks, do it now // updateGuaguaConfForExport(); localResources = YarnUtils.getLocalResourceMap(getYarnConf(), getAppId()); } catch (IOException ioe) { // fail fast, this container will never launch. throw new IllegalStateException( "Could not configure the container launch context for GuaguaYarnTask.", ioe); } } // else, return the prepopulated copy to reuse for each GuaguaYarkTask return localResources; } public ContainerId getContainerId() { return masterContainerId; } public void setContainerId(ContainerId containerId) { this.masterContainerId = containerId; } public ApplicationAttemptId getAppAttemptId() { return appAttemptId; } public void setAppAttemptId(ApplicationAttemptId appAttemptId) { this.appAttemptId = appAttemptId; } public boolean isDone() { return done; } public void setDone(boolean done) { this.done = done; } public Configuration getYarnConf() { return yarnConf; } public void setYarnConf(YarnConfiguration yarnConf) { this.yarnConf = yarnConf; } public AtomicInteger getCompletedCount() { return completedCount; } public void setCompletedCount(AtomicInteger completedCount) { this.completedCount = completedCount; } public AtomicInteger getFailedCount() { return failedCount; } public void setFailedCount(AtomicInteger failedCount) { this.failedCount = failedCount; } public AtomicInteger getAllocatedCount() { return allocatedCount; } public void setAllocatedCount(AtomicInteger allocatedCount) { this.allocatedCount = allocatedCount; } public AtomicInteger getSuccessfulCount() { return successfulCount; } public void setSuccessfulCount(AtomicInteger successfulCount) { this.successfulCount = successfulCount; } public int getContainersToLaunch() { return containersToLaunch; } public void setContainersToLaunch(int containersToLaunch) { this.containersToLaunch = containersToLaunch; } public ExecutorService getExecutor() { return executor; } public void setExecutor(ExecutorService executor) { this.executor = executor; } public int getHeapPerContainer() { return heapPerContainer; } public void setHeapPerContainer(int heapPerContainer) { this.heapPerContainer = heapPerContainer; } public AMRMClientAsync<ContainerRequest> getAmRMClient() { return amRmClient; } public void setAmRMClient(AMRMClientAsync<ContainerRequest> amRMClient) { this.amRmClient = amRMClient; } public NMClientAsync getNmClientAsync() { return nmClientAsync; } public void setNmClientAsync(NMClientAsync nmClientAsync) { this.nmClientAsync = nmClientAsync; } public NMCallbackHandler getContainerListener() { return containerListener; } public void setContainerListener(NMCallbackHandler containerListener) { this.containerListener = containerListener; } public String getContainerArgs() { return containerArgs; } public void setContainerArgs(String containerArgs) { this.containerArgs = containerArgs; } public ApplicationId getAppId() { return appId; } public void setAppId(ApplicationId appId) { this.appId = appId; } /** * Application entry point * * @param args * command-line args (set by GuaguaYarnClient, if any) */ public static void main(final String[] args) { LOG.info("Starting GuaguaAppMaster. "); String containerIdString = System.getenv().get(Environment.CONTAINER_ID.name()); if (containerIdString == null) { // container id should always be set in the env by the framework throw new IllegalArgumentException("ContainerId not found in env vars."); } ContainerId containerId = ConverterUtils.toContainerId(containerIdString); ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); Configuration conf = new YarnConfiguration(); String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name()); conf.set(MRJobConfig.USER_NAME, jobUserName); try { UserGroupInformation.setConfiguration(conf); // Security framework already loaded the tokens into current UGI, just use them Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); LOG.info("Executing with tokens:"); for (Token<?> token : credentials.getAllTokens()) { LOG.info(token.toString()); } UserGroupInformation appMasterUgi = UserGroupInformation.createRemoteUser(jobUserName); appMasterUgi.addCredentials(credentials); // Now remove the AM->RM token so tasks don't have it Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } final GuaguaAppMaster appMaster = new GuaguaAppMaster(containerId, appAttemptId, conf); appMasterUgi.doAs(new PrivilegedAction<Void>() { @Override public Void run() { boolean result = false; try { result = appMaster.run(); } catch (Throwable t) { LOG.error("GuaguaAppMaster caught a top-level exception in main.", t); System.exit(1); } if (result) { LOG.info("Guagua Application Master completed successfully. exiting"); System.exit(0); } else { LOG.info("Guagua Application Master failed. exiting"); System.exit(2); } return null; } }); } catch (Throwable t) { LOG.error("GuaguaAppMaster caught a top-level exception in main.", t); System.exit(1); } } }