org.apache.tez.dag.app.web.AMWebController.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.tez.dag.app.web.AMWebController.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.tez.dag.app.web;

import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.TreeMap;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import com.google.inject.Inject;
import com.google.inject.name.Named;

import org.apache.commons.lang.StringEscapeUtils;
import org.apache.tez.common.counters.CounterGroup;
import org.apache.tez.common.counters.LimitExceededException;
import org.apache.tez.common.counters.TezCounter;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.client.ProgressBuilder;
import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.TaskAttempt;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.webapp.Controller;
import org.apache.hadoop.yarn.webapp.MimeType;
import org.apache.hadoop.yarn.webapp.View;
import org.apache.hadoop.yarn.webapp.WebAppException;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.Vertex;
import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezVertexID;

public class AMWebController extends Controller {

    private final static Logger LOG = LoggerFactory.getLogger(AMWebController.class);

    // HTTP CORS Request Headers
    static final String ORIGIN = "Origin";

    // HTTP CORS Response Headers
    static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin";
    static final String ACCESS_CONTROL_ALLOW_CREDENTIALS = "Access-Control-Allow-Credentials";
    static final String ACCESS_CONTROL_ALLOW_METHODS = "Access-Control-Allow-Methods";
    static final String ACCESS_CONTROL_ALLOW_HEADERS = "Access-Control-Allow-Headers";
    static final String ACCESS_CONTROL_MAX_AGE = "Access-Control-Max-Age";

    // CORS default responses.
    static final String ALLOWED_METHODS = "GET, HEAD";
    static final String ALLOWED_HEADERS = "X-Requested-With,Content-Type,Accept,Origin";

    static final String DAG_PROGRESS = "dagProgress";
    static final String VERTEX_PROGRESS = "vertexProgress";
    static final String VERTEX_PROGRESSES = "vertexProgresses";

    static final int MAX_QUERIED = 100;
    public static final String VERSION = "2";

    private AppContext appContext;
    private String historyUrl;

    @Inject
    public AMWebController(RequestContext requestContext, AppContext appContext,
            @Named("TezUIHistoryURL") String historyUrl) {
        super(requestContext);
        this.appContext = appContext;
        this.historyUrl = historyUrl;
    }

    @Override
    public void index() {
        ui();
    }

    public void ui() {
        render(StaticAMView.class);
    }

    public void main() {
        ui();
    }

    public void about() {
        renderJSON("Tez AM UI WebServices");
    }

    static String encodeHeader(final String header) {
        if (header == null) {
            return null;
        }
        // Protect against HTTP response splitting vulnerability
        // since value is written as part of the response header
        // Ensure this header only has one header by removing
        // CRs and LFs
        return header.split("\n|\r")[0].trim();
    }

    @VisibleForTesting
    public void setCorsHeaders() {
        final HttpServletResponse res = response();

        /*
         * ideally the Origin and other CORS headers should be checked and response headers set only
         * if it matches the allowed origins. however rm does not forward these headers.
         */
        String historyUrlBase = appContext.getAMConf().get(TezConfiguration.TEZ_HISTORY_URL_BASE, "");
        String origin = request().getHeader(ORIGIN);
        if (origin == null) {
            try {
                URL url = new URL(historyUrlBase);
                origin = url.getProtocol() + "://" + url.getAuthority();
            } catch (MalformedURLException e) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Invalid url set for tez history url base: " + historyUrlBase, e);
                }
            }
        }

        if (origin != null) {
            origin = encodeHeader(origin);
            res.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, origin);
        }
        res.setHeader(ACCESS_CONTROL_ALLOW_METHODS, ALLOWED_METHODS);
        res.setHeader(ACCESS_CONTROL_ALLOW_CREDENTIALS, Boolean.TRUE.toString());
        res.setHeader(ACCESS_CONTROL_ALLOW_HEADERS, ALLOWED_HEADERS);
        res.setHeader(ACCESS_CONTROL_MAX_AGE, "1800");
    }

    void sendErrorResponse(int sc, String msg, Exception e) {
        if (LOG.isDebugEnabled()) {
            LOG.debug(msg, e);
        }

        try {
            response().sendError(sc, msg);
        } catch (IOException e1) {
            throw new WebAppException(e);
        }
    }

    @VisibleForTesting
    static boolean _hasAccess(UserGroupInformation callerUGI, AppContext appContext) {
        if (callerUGI == null) {
            // Allow anonymous access iff acls disabled
            return !appContext.getAMACLManager().isAclsEnabled();
        }
        return appContext.getAMACLManager().checkDAGViewAccess(callerUGI);
    }

    public boolean hasAccess() {
        String remoteUser = request().getRemoteUser();
        UserGroupInformation callerUGI = null;
        if (remoteUser != null && !remoteUser.isEmpty()) {
            callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
        }

        return _hasAccess(callerUGI, appContext);
    }

    public void getDagProgress() {

        setCorsHeaders();

        if (!hasAccess()) {
            sendErrorResponse(HttpServletResponse.SC_UNAUTHORIZED,
                    "Access denied for user: " + request().getRemoteUser(), null);
            return;
        }

        int dagID;
        try {
            dagID = getQueryParamInt(WebUIService.DAG_ID);
        } catch (NumberFormatException e) {
            sendErrorResponse(HttpServletResponse.SC_BAD_REQUEST, "Invalid dag id:", e);
            return;
        }

        DAG currentDAG = appContext.getCurrentDAG();

        if (currentDAG == null || dagID != currentDAG.getID().getId()) {
            sendErrorResponse(HttpServletResponse.SC_NOT_FOUND, "Not current Dag: " + dagID, null);
            return;
        }

        Map<String, ProgressInfo> result = new HashMap<String, ProgressInfo>();
        result.put(DAG_PROGRESS,
                new ProgressInfo(currentDAG.getID().toString(), currentDAG.getCompletedTaskProgress()));
        renderJSON(result);
    }

    public void getVertexProgress() {
        int dagID;
        int vertexID;

        setCorsHeaders();

        if (!hasAccess()) {
            sendErrorResponse(HttpServletResponse.SC_UNAUTHORIZED,
                    "Access denied for user: " + request().getRemoteUser(), null);
            return;
        }

        try {
            dagID = getQueryParamInt(WebUIService.DAG_ID);
            vertexID = getQueryParamInt(WebUIService.VERTEX_ID);
        } catch (NumberFormatException e) {
            sendErrorResponse(HttpServletResponse.SC_BAD_REQUEST, "Invalid dag or vertex id", e);
            return;
        }

        DAG currentDAG = appContext.getCurrentDAG();

        if (currentDAG == null || currentDAG.getID().getId() != dagID) {
            sendErrorResponse(HttpServletResponse.SC_NOT_FOUND, "Not current Dag: " + dagID, null);
            return;
        }

        final TezVertexID tezVertexID = TezVertexID.getInstance(currentDAG.getID(), vertexID);
        Vertex vertex = currentDAG.getVertex(tezVertexID);
        if (vertex == null) {
            sendErrorResponse(HttpServletResponse.SC_NOT_FOUND, "vertex not found: " + vertexID, null);
            return;
        }

        Map<String, ProgressInfo> result = new HashMap<String, ProgressInfo>();
        result.put(VERTEX_PROGRESS, new ProgressInfo(tezVertexID.toString(), vertex.getCompletedTaskProgress()));
        renderJSON(result);
    }

    Collection<Vertex> getVerticesByIdx(DAG dag, Collection<Integer> indexes) {
        Collection<Vertex> vertices = new ArrayList<Vertex>(indexes.size());
        final TezDAGID tezDAGID = dag.getID();

        for (Integer idx : indexes) {
            final TezVertexID tezVertexID = TezVertexID.getInstance(tezDAGID, idx);
            if (tezVertexID == null) {
                continue;
            }
            final Vertex vertex = dag.getVertex(tezVertexID);
            if (vertex != null) {
                vertices.add(vertex);
            }
        }

        return vertices;
    }

    int getQueryParamInt(String name) throws NumberFormatException {
        final String valueStr = $(name).trim();

        return Integer.parseInt(valueStr);
    }

    public void getVertexProgresses() {
        int dagID;

        setCorsHeaders();
        if (!hasAccess()) {
            sendErrorResponse(HttpServletResponse.SC_UNAUTHORIZED,
                    "Access denied for user: " + request().getRemoteUser(), null);
            return;
        }

        List<Integer> vertexIDs = new ArrayList<Integer>();
        try {
            dagID = getQueryParamInt(WebUIService.DAG_ID);
            for (String vertexIDStr : $(WebUIService.VERTEX_ID).trim().split(",", MAX_QUERIED)) {
                vertexIDs.add(Integer.parseInt(vertexIDStr));
            }
        } catch (NumberFormatException e) {
            sendErrorResponse(HttpServletResponse.SC_BAD_REQUEST, "Invalid dag or vertices id", e);
            return;
        }

        DAG currentDAG = appContext.getCurrentDAG();
        if (currentDAG == null || currentDAG.getID().getId() != dagID) {
            sendErrorResponse(HttpServletResponse.SC_NOT_FOUND, "Not current Dag: " + dagID, null);
            return;
        }

        Collection<Vertex> vertices;
        if (vertexIDs.isEmpty()) {
            vertices = currentDAG.getVertices().values();
        } else {
            vertices = getVerticesByIdx(currentDAG, vertexIDs);
        }

        Collection<ProgressInfo> progresses = new ArrayList<ProgressInfo>(vertices.size());
        for (Vertex vertex : vertices) {
            progresses.add(new ProgressInfo(vertex.getVertexId().toString(), vertex.getCompletedTaskProgress()));
        }

        Map<String, Collection<ProgressInfo>> result = new HashMap<String, Collection<ProgressInfo>>();
        result.put(VERTEX_PROGRESSES, progresses);
        renderJSON(result);
    }

    // AM WebApi V2.
    @VisibleForTesting
    protected boolean setupResponse() {
        setCorsHeaders();

        if (!hasAccess()) {
            sendErrorResponse(HttpServletResponse.SC_UNAUTHORIZED,
                    "Access denied for user: " + request().getRemoteUser(), null);
            return false;
        }

        return true;
    }

    DAG checkAndGetDAGFromRequest() {
        DAG dag = null;
        int errorCode = HttpServletResponse.SC_OK;
        String message = null;
        Exception ex = null;
        try {
            int dagID = getQueryParamInt(WebUIService.DAG_ID);
            dag = appContext.getCurrentDAG();
            if (dag == null || dag.getID().getId() != dagID) {
                errorCode = HttpServletResponse.SC_NOT_FOUND;
                message = "Not current Dag: " + dagID;
            }
        } catch (NumberFormatException e) {
            errorCode = HttpServletResponse.SC_BAD_REQUEST;
            message = "Invalid dag id";
            ex = e;
        }

        if (errorCode != HttpServletResponse.SC_OK) {
            dag = null;
            sendErrorResponse(errorCode, message, ex);
        }

        return dag;
    }

    Collection<Integer> getVertexIDsFromRequest() {
        final String valueStr = $(WebUIService.VERTEX_ID).trim();

        List<Integer> vertexIDs = new ArrayList<Integer>();
        if (!valueStr.equals("")) {
            String[] vertexIdsStr = valueStr.split(",", MAX_QUERIED);

            try {
                for (String vertexIdStr : vertexIdsStr) {
                    int vertexId = Integer.parseInt(vertexIdStr);
                    vertexIDs.add(vertexId);
                }
            } catch (NumberFormatException nfe) {
                sendErrorResponse(HttpServletResponse.SC_BAD_REQUEST, "invalid vertex ID passed in as parameter",
                        nfe);
                vertexIDs = null;
            }
        }

        return vertexIDs;
    }

    /**
     * Parse a params list in the format: CtrGroup/CtrName1,CtrName2;CtrGroup2;
     * @return nested structure of counter groups and names. Null if nothing specified.
     */
    Map<String, Set<String>> getCounterListFromRequest() {
        final String counterStr = $(WebUIService.COUNTERS).trim();
        if (counterStr == null || counterStr.isEmpty()) {
            return null;
        }

        String delimiter = ";";
        String groupDelimiter = "/";
        String counterDelimiter = ",";

        StringTokenizer tokenizer = new StringTokenizer(counterStr, delimiter);

        Map<String, Set<String>> counterList = new TreeMap<String, Set<String>>();
        while (tokenizer.hasMoreElements()) {
            String token = tokenizer.nextToken().trim();
            int pos = token.indexOf(groupDelimiter);
            if (pos == -1) {
                counterList.put(token, Collections.<String>emptySet());
                continue;
            }
            String counterGroup = token.substring(0, pos);
            Set<String> counters = Collections.<String>emptySet();
            if (pos < token.length() - 1) {
                String counterNames = token.substring(pos + 1, token.length());
                counters = Sets.newHashSet(
                        Splitter.on(counterDelimiter).omitEmptyStrings().trimResults().split(counterNames));
            }
            counterList.put(counterGroup, counters);
        }
        return counterList;
    }

    List<String> splitString(String str, String delimiter, Integer limit) {
        List<String> items = new ArrayList<String>();

        StringTokenizer tokenizer = new StringTokenizer(str, delimiter);
        for (int count = 0; tokenizer.hasMoreElements() && count < limit; count++) {
            items.add(tokenizer.nextToken());
        }

        return items;
    }

    /**
     * getIntegersFromRequest
     * Parses a query parameter with comma separated values and returns an array of integers.
     * The function returns null if any of the value is not an integer
     *
     * @param paramName {String}
     * @param limit {Integer} Maximum number of values to be taken
     *
     * @return {List<Integer>} List of parsed values
     */
    List<Integer> getIntegersFromRequest(String paramName, Integer limit) {
        String valuesStr = $(paramName).trim();

        List<Integer> values = new ArrayList<Integer>();
        if (!valuesStr.equals("")) {
            try {
                for (String valueStr : splitString(valuesStr, ",", limit)) {
                    int value = Integer.parseInt(valueStr);
                    values.add(value);
                }
            } catch (NumberFormatException nfe) {
                sendErrorResponse(HttpServletResponse.SC_BAD_REQUEST,
                        String.format("invalid %s passed in as parameter", paramName), nfe);
                values = null;
            }
        }

        return values;
    }

    /**
     * getTaskIDsFromRequest
     * Takes in "1_0,1_3" and returns [[1,0],[1,3]]
     * Mainly to parse a query parameter with comma separated indexes. For vertex its the index,
     * for task its vertexIndex_taskIndex and for attempts its vertexIndex_taskIndex_attemptNo
     * The function returns null if any of the value is not an integer
     *
     * @param paramName {String}
     * @param limit {Integer} Maximum number of values to be taken
     *
     * @return {List<List<Integer>>} List of parsed values
     */
    List<List<Integer>> getIDsFromRequest(String paramName, Integer limit, Integer count) {
        String valuesStr = $(paramName).trim();

        List<List<Integer>> values = new ArrayList<List<Integer>>();
        if (!valuesStr.equals("")) {
            try {
                for (String valueStr : splitString(valuesStr, ",", limit)) {
                    List<Integer> innerValues = new ArrayList<Integer>();
                    String innerValueStrs[] = valueStr.split("_");
                    if (innerValueStrs.length == count) {
                        for (String innerValueStr : innerValueStrs) {
                            int value = Integer.parseInt(innerValueStr);
                            innerValues.add(value);
                        }
                        values.add(innerValues);
                    }
                }
            } catch (NumberFormatException nfe) {
                sendErrorResponse(HttpServletResponse.SC_BAD_REQUEST,
                        String.format("invalid %s passed in as parameter", paramName), nfe);
                values = null;
            }
        }

        return values;
    }

    public void getDagInfo() {
        if (!setupResponse()) {
            return;
        }

        DAG dag = checkAndGetDAGFromRequest();
        if (dag == null) {
            return;
        }

        Map<String, Set<String>> counterNames = getCounterListFromRequest();

        Map<String, Object> dagInfo = new HashMap<String, Object>();
        dagInfo.put("id", dag.getID().toString());
        dagInfo.put("progress", Float.toString(dag.getCompletedTaskProgress()));
        dagInfo.put("status", dag.getState().toString());

        try {
            if (counterNames != null && !counterNames.isEmpty()) {
                TezCounters counters = dag.getCachedCounters();
                Map<String, Map<String, Long>> counterMap = constructCounterMapInfo(counters, counterNames);
                if (counterMap != null && !counterMap.isEmpty()) {
                    dagInfo.put("counters", counterMap);
                }
            }
        } catch (LimitExceededException e) {
            // Ignore
            // TODO: add an error message instead for counter key
        }
        renderJSON(ImmutableMap.of("dag", dagInfo));
    }

    Map<String, Map<String, Long>> constructCounterMapInfo(TezCounters counters,
            Map<String, Set<String>> counterNames) {
        if (counterNames == null || counterNames.isEmpty()) {
            return null;
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("Requested counter names=" + counterNames.entrySet());
            LOG.debug("actual counters=" + counters);
        }

        Map<String, Map<String, Long>> counterInfo = new TreeMap<String, Map<String, Long>>();

        if (counterNames.containsKey("*")) {
            for (CounterGroup grpCounters : counters) {
                Map<String, Long> matchedCounters = new HashMap<String, Long>();
                for (TezCounter counter : grpCounters) {
                    matchedCounters.put(counter.getName(), counter.getValue());
                }
                counterInfo.put(grpCounters.getName(), matchedCounters);
            }
        } else {
            for (Entry<String, Set<String>> entry : counterNames.entrySet()) {
                Map<String, Long> matchedCounters = new HashMap<String, Long>();
                CounterGroup grpCounters = counters.getGroup(entry.getKey());
                for (TezCounter counter : grpCounters) {
                    if (entry.getValue().isEmpty() || entry.getValue().contains(counter.getName())) {
                        matchedCounters.put(counter.getName(), counter.getValue());
                    }
                }
                counterInfo.put(entry.getKey(), matchedCounters);
            }
        }

        return counterInfo;
    }

    private Map<String, Object> getVertexInfoMap(Vertex vertex, Map<String, Set<String>> counterNames) {
        Map<String, Object> vertexInfo = new HashMap<String, Object>();
        vertexInfo.put("id", vertex.getVertexId().toString());
        vertexInfo.put("status", vertex.getState().toString());
        vertexInfo.put("progress", Float.toString(vertex.getCompletedTaskProgress()));

        vertexInfo.put("initTime", Long.toString(vertex.getInitTime()));
        vertexInfo.put("startTime", Long.toString(vertex.getStartTime()));
        vertexInfo.put("finishTime", Long.toString(vertex.getFinishTime()));
        vertexInfo.put("firstTaskStartTime", Long.toString(vertex.getFirstTaskStartTime()));
        vertexInfo.put("lastTaskFinishTime", Long.toString(vertex.getLastTaskFinishTime()));

        ProgressBuilder vertexProgress = vertex.getVertexProgress();
        vertexInfo.put("totalTasks", Integer.toString(vertexProgress.getTotalTaskCount()));
        vertexInfo.put("runningTasks", Integer.toString(vertexProgress.getRunningTaskCount()));
        vertexInfo.put("succeededTasks", Integer.toString(vertexProgress.getSucceededTaskCount()));

        vertexInfo.put("failedTaskAttempts", Integer.toString(vertexProgress.getFailedTaskAttemptCount()));
        vertexInfo.put("killedTaskAttempts", Integer.toString(vertexProgress.getKilledTaskAttemptCount()));

        try {
            if (counterNames != null && !counterNames.isEmpty()) {
                TezCounters counters = vertex.getCachedCounters();
                Map<String, Map<String, Long>> counterMap = constructCounterMapInfo(counters, counterNames);
                if (counterMap != null && !counterMap.isEmpty()) {
                    vertexInfo.put("counters", counterMap);
                }
            }
        } catch (LimitExceededException e) {
            // Ignore
            // TODO: add an error message instead for counter key
        }

        return vertexInfo;
    }

    public void getVerticesInfo() {
        if (!setupResponse()) {
            return;
        }

        DAG dag = checkAndGetDAGFromRequest();
        if (dag == null) {
            return;
        }

        Collection<Integer> requestedIDs = getVertexIDsFromRequest();

        if (requestedIDs == null) {
            return;
        }

        Map<String, Set<String>> counterNames = getCounterListFromRequest();

        Collection<Vertex> vertexList;
        if (requestedIDs.isEmpty()) {
            // no ids specified return all.
            vertexList = dag.getVertices().values();
        } else {
            vertexList = getVerticesByIdx(dag, requestedIDs);
        }

        ArrayList<Map<String, Object>> verticesInfo = new ArrayList<Map<String, Object>>();
        for (Vertex v : vertexList) {
            verticesInfo.add(getVertexInfoMap(v, counterNames));
        }

        renderJSON(ImmutableMap.of("vertices", verticesInfo));
    }

    Vertex getVertexFromIndex(DAG dag, Integer vertexIndex) {
        final TezVertexID tezVertexID = TezVertexID.getInstance(dag.getID(), vertexIndex);
        Vertex vertex = dag.getVertex(tezVertexID);
        return vertex;
    }

    /**
     * getRequestedTasks
     * Heart of getTasksInfo. Given a dag and a limit, based on the incoming query parameters
     * returns a list of task instances
     *
     * @param dag {DAG}
     * @param limit {Integer}
     */
    List<Task> getRequestedTasks(DAG dag, Integer limit) {
        List<Task> tasks = new ArrayList<Task>();

        List<List<Integer>> taskIDs = getIDsFromRequest(WebUIService.TASK_ID, limit, 2);
        if (taskIDs == null) {
            return null;
        } else if (!taskIDs.isEmpty()) {
            for (List<Integer> indexes : taskIDs) {
                Vertex vertex = getVertexFromIndex(dag, indexes.get(0));
                if (vertex == null) {
                    continue;
                }
                Task task = vertex.getTask(indexes.get(1));
                if (task == null) {
                    continue;
                } else {
                    tasks.add(task);
                }

                if (tasks.size() >= limit) {
                    break;
                }
            }
        } else {
            List<Integer> vertexIDs = getIntegersFromRequest(WebUIService.VERTEX_ID, limit);
            if (vertexIDs == null) {
                return null;
            } else if (!vertexIDs.isEmpty()) {
                for (Integer vertexID : vertexIDs) {
                    Vertex vertex = getVertexFromIndex(dag, vertexID);
                    if (vertex == null) {
                        continue;
                    }
                    List<Task> vertexTasks = new ArrayList<Task>(vertex.getTasks().values());
                    tasks.addAll(vertexTasks.subList(0, Math.min(vertexTasks.size(), limit - tasks.size())));

                    if (tasks.size() >= limit) {
                        break;
                    }
                }
            } else {
                Collection<Vertex> vertices = dag.getVertices().values();
                for (Vertex vertex : vertices) {
                    List<Task> vertexTasks = new ArrayList<Task>(vertex.getTasks().values());
                    tasks.addAll(vertexTasks.subList(0, Math.min(vertexTasks.size(), limit - tasks.size())));

                    if (tasks.size() >= limit) {
                        break;
                    }
                }
            }
        }

        return tasks;
    }

    /**
     * Renders the response JSON for tasksInfo API
     * The JSON will have an array of task objects under the key tasks.
     */
    public void getTasksInfo() {
        if (!setupResponse()) {
            return;
        }

        DAG dag = checkAndGetDAGFromRequest();
        if (dag == null) {
            return;
        }

        int limit = MAX_QUERIED;
        try {
            limit = getQueryParamInt(WebUIService.LIMIT);
        } catch (NumberFormatException e) {
            //Ignore
        }

        List<Task> tasks = getRequestedTasks(dag, limit);
        if (tasks == null) {
            return;
        }

        Map<String, Set<String>> counterNames = getCounterListFromRequest();

        ArrayList<Map<String, Object>> tasksInfo = new ArrayList<Map<String, Object>>();
        for (Task t : tasks) {
            Map<String, Object> taskInfo = new HashMap<String, Object>();
            taskInfo.put("id", t.getTaskId().toString());
            taskInfo.put("progress", Float.toString(t.getProgress()));
            taskInfo.put("status", t.getState().toString());

            try {
                TezCounters counters = t.getCounters();
                Map<String, Map<String, Long>> counterMap = constructCounterMapInfo(counters, counterNames);
                if (counterMap != null && !counterMap.isEmpty()) {
                    taskInfo.put("counters", counterMap);
                }
            } catch (LimitExceededException e) {
                // Ignore
                // TODO: add an error message instead for counter key
            }

            tasksInfo.add(taskInfo);
        }

        renderJSON(ImmutableMap.of("tasks", tasksInfo));
    }

    /**
     * getRequestedAttempts
     * Given a dag and a limit, based on the incoming query parameters. Used by getAttemptsInfo
     * returns a list of task instances
     *
     * @param dag {DAG}
     * @param limit {Integer}
     */
    List<TaskAttempt> getRequestedAttempts(DAG dag, Integer limit) {
        List<TaskAttempt> attempts = new ArrayList<TaskAttempt>();

        List<List<Integer>> attemptIDs = getIDsFromRequest(WebUIService.ATTEMPT_ID, limit, 3);
        if (attemptIDs == null) {
            return null;
        } else if (!attemptIDs.isEmpty()) {
            for (List<Integer> indexes : attemptIDs) {
                Vertex vertex = getVertexFromIndex(dag, indexes.get(0));
                if (vertex == null) {
                    continue;
                }
                Task task = vertex.getTask(indexes.get(1));
                if (task == null) {
                    continue;
                }

                TaskAttempt attempt = task
                        .getAttempt(TezTaskAttemptID.getInstance(task.getTaskId(), indexes.get(2)));
                if (attempt == null) {
                    continue;
                } else {
                    attempts.add(attempt);
                }

                if (attempts.size() >= limit) {
                    break;
                }
            }
        }

        return attempts;
    }

    /**
     * Renders the response JSON for attemptsInfo API
     * The JSON will have an array of attempt objects under the key attempts.
     */
    public void getAttemptsInfo() {
        if (!setupResponse()) {
            return;
        }

        DAG dag = checkAndGetDAGFromRequest();
        if (dag == null) {
            return;
        }

        int limit = MAX_QUERIED;
        try {
            limit = getQueryParamInt(WebUIService.LIMIT);
        } catch (NumberFormatException e) {
            //Ignore
        }

        List<TaskAttempt> attempts = getRequestedAttempts(dag, limit);
        if (attempts == null) {
            return;
        }

        Map<String, Set<String>> counterNames = getCounterListFromRequest();

        ArrayList<Map<String, Object>> attemptsInfo = new ArrayList<Map<String, Object>>();
        for (TaskAttempt a : attempts) {
            Map<String, Object> attemptInfo = new HashMap<String, Object>();
            attemptInfo.put("id", a.getID().toString());
            attemptInfo.put("progress", Float.toString(a.getProgress()));
            attemptInfo.put("status", a.getState().toString());

            try {
                TezCounters counters = a.getCounters();
                Map<String, Map<String, Long>> counterMap = constructCounterMapInfo(counters, counterNames);
                if (counterMap != null && !counterMap.isEmpty()) {
                    attemptInfo.put("counters", counterMap);
                }
            } catch (LimitExceededException e) {
                // Ignore
                // TODO: add an error message instead for counter key
            }
            attemptsInfo.add(attemptInfo);
        }

        renderJSON(ImmutableMap.of("attempts", attemptsInfo));
    }

    @Override
    @VisibleForTesting
    public void renderJSON(Object object) {
        super.renderJSON(object);
    }

    public static class StaticAMView extends View {
        @Inject
        AppContext appContext;
        @Inject
        @Named("TezUIHistoryURL")
        String historyUrl;

        @Override
        public void render() {
            response().setContentType(MimeType.HTML);
            PrintWriter pw = writer();
            pw.write("<html>");
            pw.write("<head>");
            pw.write("<meta charset=\"utf-8\">");
            pw.write("<title>Redirecting to Tez UI</title>");
            pw.write("</head>");
            pw.write("<body>");
            if (historyUrl == null || historyUrl.isEmpty()) {
                pw.write("<h1>Tez UI Url is not defined.</h1>"
                        + "<p>To enable tracking url pointing to Tez UI, set the config <b>"
                        + TezConfiguration.TEZ_HISTORY_URL_BASE + "</b> in the tez-site.xml.</p>");
            } else {
                pw.write("<h1>Redirecting to Tez UI</h1>. <p>If you are not redirected shortly, click "
                        + "<a href='" + historyUrl + "'><b>here</b></a></p>");
                pw.write("<script type='text/javascript'>setTimeout(function() { " + "window.location.replace('"
                        + historyUrl + "');" + "}, 0); </script>");
            }
            pw.write("</body>");
            pw.write("</html>");
            pw.flush();
        }
    }

    @VisibleForTesting
    static class ProgressInfo {
        private String id;

        public float getProgress() {
            return progress;
        }

        public String getId() {
            return id;
        }

        private float progress;

        public ProgressInfo(String id, float progress) {
            this.id = id;
            this.progress = progress;
        }
    }
}