Java tutorial
/* * Copyright 2015 Apache Software Foundation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.distributedloadsimulator.sls.appmaster; /** * * @author sri */ import java.io.IOException; import java.net.InetSocketAddress; import java.rmi.RemoteException; import java.security.PrivilegedExceptionAction; import java.text.MessageFormat; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.apache.hadoop.distributedloadsimulator.sls.AMNMCommonObject; import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.distributedloadsimulator.sls.scheduler.ContainerSimulator; import org.apache.hadoop.distributedloadsimulator.sls.SLSRunner; import org.apache.hadoop.distributedloadsimulator.sls.conf.SLSConfiguration; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.security.SaslRpcServer; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.log4j.Logger; public class MRAMSimulator extends AMSimulator { /* * Vocabulary Used: * pending -> requests which are NOT yet sent to RM * scheduled -> requests which are sent to RM but not yet assigned * assigned -> requests which are assigned to a container * completed -> request corresponding to which container has completed * * Maps are scheduled as soon as their requests are received. Reduces are * scheduled when all maps have finished (not support slow-start currently). */ private static final int PRIORITY_REDUCE = 10; private static final int PRIORITY_MAP = 20; // pending maps private LinkedList<ContainerSimulator> pendingMaps = new LinkedList<ContainerSimulator>(); // pending failed maps private LinkedList<ContainerSimulator> pendingFailedMaps = new LinkedList<ContainerSimulator>(); // scheduled maps private LinkedList<ContainerSimulator> scheduledMaps = new LinkedList<ContainerSimulator>(); // assigned maps private Map<ContainerId, ContainerSimulator> assignedMaps = new HashMap<ContainerId, ContainerSimulator>(); // reduces which are not yet scheduled private LinkedList<ContainerSimulator> pendingReduces = new LinkedList<ContainerSimulator>(); // pending failed reduces private LinkedList<ContainerSimulator> pendingFailedReduces = new LinkedList<ContainerSimulator>(); // scheduled reduces private LinkedList<ContainerSimulator> scheduledReduces = new LinkedList<ContainerSimulator>(); // assigned reduces private Map<ContainerId, ContainerSimulator> assignedReduces = new HashMap<ContainerId, ContainerSimulator>(); // all maps & reduces private LinkedList<ContainerSimulator> allMaps = new LinkedList<ContainerSimulator>(); private LinkedList<ContainerSimulator> allReduces = new LinkedList<ContainerSimulator>(); // counters private int mapFinished = 0; private int mapTotal = 0; private int reduceFinished = 0; private int reduceTotal = 0; // waiting for AM container private boolean isAMContainerRunning = false; private Container amContainer; // finished private boolean isFinished = false; // resource for AM container private static int MR_AM_CONTAINER_RESOURCE_MEMORY_MB; private static int MR_AM_CONTAINER_RESOURCE_VCORES; private static final Log LOG = LogFactory.getLog(MRAMSimulator.class); private long applicationMasterWaitTime; private List<Long> containerAllocationWaitTime = new ArrayList<Long>(); private List<Long> containerStartWaitTime = new ArrayList<Long>(); private long startRequestingContainers; private boolean firstRequest = true; private List tasks; public void init(int id, int heartbeatInterval, List tasks, ResourceManager rm, SLSRunner se, long traceStartTime, long traceFinishTime, String user, String queue, boolean isTracked, String oldAppId, String[] remoteSimIp, int rmiPort, YarnClient rmClient, Configuration conf) throws IOException { super.init(id, heartbeatInterval, tasks, rm, se, traceStartTime, traceFinishTime, user, queue, isTracked, oldAppId, remoteSimIp, rmiPort, rmClient, conf); amtype = "mapreduce"; this.tasks = tasks; conf.addResource("sls-runner.xml"); MR_AM_CONTAINER_RESOURCE_MEMORY_MB = conf.getInt(SLSConfiguration.CONTAINER_MEMORY_MB, SLSConfiguration.CONTAINER_MEMORY_MB_DEFAULT); MR_AM_CONTAINER_RESOURCE_VCORES = conf.getInt(SLSConfiguration.CONTAINER_VCORES, SLSConfiguration.CONTAINER_VCORES_DEFAULT); } @Override public void firstStep() throws YarnException, IOException, InterruptedException { int containerVCores = conf.getInt(SLSConfiguration.CONTAINER_VCORES, SLSConfiguration.CONTAINER_VCORES_DEFAULT); int containerMemoryMB = conf.getInt(SLSConfiguration.CONTAINER_MEMORY_MB, SLSConfiguration.CONTAINER_MEMORY_MB_DEFAULT); Resource containerResource = BuilderUtils.newResource(containerMemoryMB, containerVCores); List<ContainerSimulator> containerList = new ArrayList<ContainerSimulator>(); for (Object o : tasks) { Map jsonTask = (Map) o; String hostname = jsonTask.get("container.host").toString(); long taskStart = Long.parseLong(jsonTask.get("container.start.ms").toString()); long taskFinish = Long.parseLong(jsonTask.get("container.end.ms").toString()); long lifeTime = taskFinish - taskStart; int priority = Integer.parseInt(jsonTask.get("container.priority").toString()); String type = jsonTask.get("container.type").toString(); containerList.add(new ContainerSimulator(containerResource, lifeTime, hostname, priority, type)); } // get map/reduce tasks for (ContainerSimulator cs : containerList) { if (cs.getType().equals("map")) { cs.setPriority(PRIORITY_MAP); pendingMaps.add(cs); } else if (cs.getType().equals("reduce")) { cs.setPriority(PRIORITY_REDUCE); pendingReduces.add(cs); } } allMaps.addAll(pendingMaps); allReduces.addAll(pendingReduces); mapTotal = pendingMaps.size(); reduceTotal = pendingReduces.size(); totalContainers = mapTotal + reduceTotal; super.firstStep(); if (submited) { requestAMContainer(); } } Map<String, Long> ContainersStartTimes = new HashMap<String, Long>(); /** * send out request for AM container * * @throws org.apache.hadoop.yarn.exceptions.YarnException * @throws java.io.IOException * @throws java.lang.InterruptedException */ protected void requestAMContainer() throws YarnException, IOException, InterruptedException { List<ResourceRequest> ask = new ArrayList<ResourceRequest>(); ResourceRequest amRequest = createResourceRequest( BuilderUtils.newResource(MR_AM_CONTAINER_RESOURCE_MEMORY_MB, MR_AM_CONTAINER_RESOURCE_VCORES), ResourceRequest.ANY, 1, 1); ask.add(amRequest); final AllocateRequest request = this.createAllocateRequest(ask); AllocateResponse response = null; UserGroupInformation ugi = UserGroupInformation.createProxyUser(appAttemptId.toString(), UserGroupInformation.getCurrentUser()); ugi.setAuthenticationMethod(SaslRpcServer.AuthMethod.TOKEN); ugi.addCredentials(credentials); ugi.addToken(amRMToken); ugi.addTokenIdentifier(amRMToken.decodeIdentifier()); response = ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { UserGroupInformation.getCurrentUser().addToken(amRMToken); InetSocketAddress resourceManagerAddress = conf.getSocketAddr( YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); SecurityUtil.setTokenService(amRMToken, resourceManagerAddress); ApplicationMasterProtocol appMasterProtocol = ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class, true); AllocateResponse response = appMasterProtocol.allocate(request); RPC.stopProxy(appMasterProtocol); return response; } }); // waiting until the AM container is allocated while (true) { if (response != null && !response.getAllocatedContainers().isEmpty()) { // get AM container Container container = response.getAllocatedContainers().get(0); for (AMNMCommonObject remoteConnection : RemoteConnections) { if (remoteConnection.isNodeExist(container.getNodeId().toString())) { remoteConnection.addNewContainer(container.getId().toString(), container.getNodeId().toString(), container.getNodeHttpAddress(), container.getResource().getMemory(), container.getResource().getVirtualCores(), container.getPriority().getPriority(), -1L); } } // start AM container ContainersStartTimes.put(container.getId().toString(), System.currentTimeMillis()); amContainer = container; isAMContainerRunning = true; applicationMasterWaitTime = System.currentTimeMillis() - applicationStartTime; try { primaryRemoteConnection.addApplicationMasterWaitTime(applicationMasterWaitTime); } catch (RemoteException e) { LOG.error(e, e); } break; } // this sleep time is different from HeartBeat Thread.sleep(1000); // send out empty request sendContainerRequest(); response = responseQueue.take(); } } @Override @SuppressWarnings("unchecked") protected void processResponseQueue() throws InterruptedException, YarnException, IOException { while (!responseQueue.isEmpty()) { AllocateResponse response = responseQueue.take(); // check completed containers if (!response.getCompletedContainersStatuses().isEmpty()) { for (ContainerStatus cs : response.getCompletedContainersStatuses()) { ContainerId containerId = cs.getContainerId(); if (cs.getExitStatus() == ContainerExitStatus.SUCCESS) { totalContainersDuration = totalContainersDuration + System.currentTimeMillis() - ContainersStartTimes.get(cs.getContainerId().toString()); if (assignedMaps.containsKey(containerId)) { assignedMaps.remove(containerId); mapFinished++; finishedContainers++; } else if (assignedReduces.containsKey(containerId)) { assignedReduces.remove(containerId); reduceFinished++; finishedContainers++; } else { // am container released event isFinished = true; } } else { // container to be killed if (assignedMaps.containsKey(containerId)) { pendingFailedMaps.add(assignedMaps.remove(containerId)); } else if (assignedReduces.containsKey(containerId)) { pendingFailedReduces.add(assignedReduces.remove(containerId)); } else { restart(); } } } } // check finished if (isAMContainerRunning && (mapFinished == mapTotal) && (reduceFinished == reduceTotal)) { // to release the AM container for (AMNMCommonObject remoteConnection : RemoteConnections) { if (remoteConnection.isNodeExist(amContainer.getNodeId().toString())) { remoteConnection.cleanupContainer(amContainer.getId().toString(), amContainer.getNodeId().toString()); } } isAMContainerRunning = false; isFinished = true; } // check allocated containers for (Container container : response.getAllocatedContainers()) { ContainersStartTimes.put(container.getId().toString(), System.currentTimeMillis()); if (!scheduledMaps.isEmpty()) { ContainerSimulator cs = scheduledMaps.remove(); assignedMaps.put(container.getId(), cs); containerAllocationWaitTime.add(System.currentTimeMillis() - startRequestingContainers); containerStartWaitTime.add(System.currentTimeMillis() - applicationStartTime); try { primaryRemoteConnection.addContainerAllocationWaitTime( System.currentTimeMillis() - startRequestingContainers); primaryRemoteConnection .addContainerStartWaitTime(System.currentTimeMillis() - applicationStartTime); } catch (RemoteException e) { LOG.error(e, e); } for (AMNMCommonObject remoteConnection : RemoteConnections) { if (remoteConnection.isNodeExist(container.getNodeId().toString())) { remoteConnection.addNewContainer(container.getId().toString(), container.getNodeId().toString(), container.getNodeHttpAddress(), container.getResource().getMemory(), container.getResource().getVirtualCores(), container.getPriority().getPriority(), cs.getLifeTime()); } } } else if (!this.scheduledReduces.isEmpty()) { ContainerSimulator cs = scheduledReduces.remove(); assignedReduces.put(container.getId(), cs); for (AMNMCommonObject remoteConnection : RemoteConnections) { if (remoteConnection.isNodeExist(container.getNodeId().toString())) { remoteConnection.addNewContainer(container.getId().toString(), container.getNodeId().toString(), container.getNodeHttpAddress(), container.getResource().getMemory(), container.getResource().getVirtualCores(), container.getPriority().getPriority(), cs.getLifeTime()); } } } } } } /** * restart running because of the am container killed */ private void restart() throws YarnException, IOException, InterruptedException { // clear finishedContainers = 0; isFinished = false; mapFinished = 0; reduceFinished = 0; pendingFailedMaps.clear(); pendingMaps.clear(); pendingReduces.clear(); pendingFailedReduces.clear(); pendingMaps.addAll(allMaps); pendingReduces.addAll(pendingReduces); isAMContainerRunning = false; amContainer = null; // resent am container request requestAMContainer(); } @Override protected void sendContainerRequest() throws YarnException, IOException, InterruptedException { if (isFinished) { return; } //LOG.info("HOP :: Send container request "); // send out request List<ResourceRequest> ask = null; if (isAMContainerRunning) { if (mapFinished != mapTotal) { // map phase if (!pendingMaps.isEmpty()) { ask = packageRequests(pendingMaps, PRIORITY_MAP); scheduledMaps.addAll(pendingMaps); pendingMaps.clear(); } else if (!pendingFailedMaps.isEmpty() && scheduledMaps.isEmpty()) { ask = packageRequests(pendingFailedMaps, PRIORITY_MAP); // pendingFailedMaps.size())); scheduledMaps.addAll(pendingFailedMaps); pendingFailedMaps.clear(); } } else if (reduceFinished != reduceTotal) { // reduce phase if (!pendingReduces.isEmpty()) { ask = packageRequests(pendingReduces, PRIORITY_REDUCE); scheduledReduces.addAll(pendingReduces); pendingReduces.clear(); } else if (!pendingFailedReduces.isEmpty() && scheduledReduces.isEmpty()) { ask = packageRequests(pendingFailedReduces, PRIORITY_REDUCE); // pendingFailedReduces.size())); scheduledReduces.addAll(pendingFailedReduces); pendingFailedReduces.clear(); } } if (firstRequest) { firstRequest = false; startRequestingContainers = System.currentTimeMillis(); } } if (ask == null) { ask = new ArrayList<ResourceRequest>(); } final AllocateRequest request = createAllocateRequest(ask); if (totalContainers == 0) { request.setProgress(1.0f); } else { request.setProgress((float) finishedContainers / totalContainers); } if (ask.size() > 0) { int nbContainers = 0; for (ResourceRequest r : ask) { nbContainers += r.getNumContainers(); } LOG.info("application " + appId + " requesting containers " + nbContainers); } AllocateResponse response = null; UserGroupInformation ugi = UserGroupInformation.createProxyUser(appAttemptId.toString(), UserGroupInformation.getCurrentUser()); ugi.setAuthenticationMethod(SaslRpcServer.AuthMethod.TOKEN); ugi.addCredentials(credentials); ugi.addToken(amRMToken); ugi.addTokenIdentifier(amRMToken.decodeIdentifier()); response = ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { UserGroupInformation.getCurrentUser().addToken(amRMToken); InetSocketAddress resourceManagerAddress = conf.getSocketAddr( YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); SecurityUtil.setTokenService(amRMToken, resourceManagerAddress); ApplicationMasterProtocol appMasterProtocol = ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class, true); AllocateResponse response = appMasterProtocol.allocate(request); RPC.stopProxy(appMasterProtocol); return response; } }); if (response != null) { responseQueue.put(response); } } @Override protected void checkStop() { if (isFinished) { super.setEndTime(System.currentTimeMillis()); } } @Override public void lastStep() throws YarnException { //LOG.info(MessageFormat.format("Application reaching to laststep {0}.", appId)); super.lastStep(); // clear data structures allMaps.clear(); allReduces.clear(); assignedMaps.clear(); assignedReduces.clear(); pendingFailedMaps.clear(); pendingFailedReduces.clear(); pendingMaps.clear(); pendingReduces.clear(); scheduledMaps.clear(); scheduledReduces.clear(); responseQueue.clear(); } }