org.openflamingo.remote.thrift.mapred.ThriftJobTrackerPlugin.java Source code

Java tutorial

Introduction

Here is the source code for org.openflamingo.remote.thrift.mapred.ThriftJobTrackerPlugin.java

Source

/**
 * Licensed to Cloudera, Inc. under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  Cloudera, Inc. licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.openflamingo.remote.thrift.mapred;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
import java.util.*;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.mapred.Counters.Counter;
import org.apache.hadoop.mapred.Counters.Group;
import org.apache.hadoop.mapred.JobTracker.State;
import org.apache.hadoop.mapred.TaskStatus.Phase;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.thriftfs.*;
import org.apache.hadoop.thriftfs.api.IOException;
import org.apache.hadoop.thriftfs.api.RequestContext;
import org.apache.hadoop.thriftfs.api.ThriftDelegationToken;
import org.apache.hadoop.thriftfs.jobtracker.api.*;
import org.apache.thrift.TException;
import org.apache.thrift.TProcessor;
import org.apache.thrift.TProcessorFactory;
import org.apache.thrift.transport.TTransport;

/**
 * Exposes JobTracker APIs as a Thrift service, running by default on
 * DEFAULT_THRIFT_ADDRESS.
 */
@SuppressWarnings("deprecation")
public class ThriftJobTrackerPlugin extends JobTrackerPlugin implements Configurable {

    private static final int DEFAULT_NUM_TASKS_TO_SEND = 10;
    private static final int DEFAULT_NUM_FAILURES_TO_SEND = 5;

    /**
     * Provides lots of methods for mapping mapred objects onto their
     * Thrift equivalents, dispatched through the magic of polymorphism.
     */
    private static class JTThriftUtils {
        static ThriftJobPriority toThrift(JobPriority priority) {
            switch (priority) {
            case VERY_HIGH:
                return ThriftJobPriority.VERY_HIGH;
            case HIGH:
                return ThriftJobPriority.HIGH;
            case NORMAL:
                return ThriftJobPriority.NORMAL;
            case LOW:
                return ThriftJobPriority.LOW;
            case VERY_LOW:
                return ThriftJobPriority.VERY_LOW;
            }
            LOG.info("Unexpected priority in toThrift(JobPriority) - defaulting to NORMAL");
            return ThriftJobPriority.NORMAL;
        }

        public static ThriftJobID toThrift(JobID jobId) {
            ThriftJobID ret = new ThriftJobID();
            ret.setJobID(jobId.getId());
            ret.setJobTrackerID(jobId.getJtIdentifier());
            ret.setAsString(jobId.toString());
            return ret;
        }

        public static JobID fromThrift(ThriftJobID jobId) {
            return new JobID(jobId.getJobTrackerID(), jobId.getJobID());
        }

        public static TaskID fromThrift(ThriftTaskID taskId) {
            return new TaskID(fromThrift(taskId.getJobID()), taskId.getTaskType() == ThriftTaskType.MAP,
                    taskId.getTaskID());
        }

        public static TaskAttemptID fromThrift(ThriftTaskAttemptID taskId) {
            JobID jid = fromThrift(taskId.getTaskID().jobID);
            String ident = jid.getJtIdentifier();
            boolean isMap = taskId.taskID.getTaskType() == ThriftTaskType.MAP;
            TaskAttemptID id = new TaskAttemptID(ident, jid.getId(), isMap, taskId.getTaskID().getTaskID(),
                    taskId.getAttemptID());
            return id;
        }

        public static ThriftJobState jobRunStateToThrift(int state) {
            switch (state) {
            case JobStatus.RUNNING:
                return ThriftJobState.RUNNING;
            case JobStatus.SUCCEEDED:
                return ThriftJobState.SUCCEEDED;
            case JobStatus.FAILED:
                return ThriftJobState.FAILED;
            case JobStatus.PREP:
                return ThriftJobState.PREP;
            case JobStatus.KILLED:
                return ThriftJobState.KILLED;
            default:
                return null; // signify unknown
            }
        }

        public static ThriftJobStatus toThrift(JobStatus job) {
            ThriftJobStatus ret = new ThriftJobStatus();
            ret.setCleanupProgress(job.cleanupProgress());
            ret.setMapProgress(job.mapProgress());
            ret.setReduceProgress(job.reduceProgress());
            ret.setPriority(toThrift(job.getJobPriority()));
            ret.setRunState(jobRunStateToThrift(job.getRunState()));
            ret.setSchedulingInfo(job.getSchedulingInfo());
            ret.setSetupProgress(job.setupProgress());
            ret.setStartTime(job.getStartTime());
            ret.setUser(job.getUsername());
            ret.setJobID(toThrift(job.getJobID()));
            return ret;
        }

        /**
         * Convert a section of an array of TaskInProgress to a ThriftTaskInProgressList.
         * The returned list contains tasks in the range of [fromIdx, toIdx).
         * Callers should make sure that the indices are valid, and that toIdx
         * is not smaller than fromIdx.
         * @param tasks         An array of TaskInProgress objects.
         * @param tracker       The JobTracker.
         * @param fromIdx       The inclusive starting range to convert.
         * @param toIdx         The exclusive ending range to convert, i.e. [fromIdx, toIdx)
         */
        public static ThriftTaskInProgressList toThrift(TaskInProgress[] tasks, JobTracker tracker, int fromIdx,
                int toIdx) {
            ThriftTaskInProgressList ret = new ThriftTaskInProgressList();

            if (toIdx > tasks.length)
                toIdx = tasks.length;
            if (fromIdx > toIdx) {
                assert false; // Internal callers should not pass in bogus args
                fromIdx = toIdx;
            }

            ArrayList<ThriftTaskInProgress> taskArr = new ArrayList<ThriftTaskInProgress>(toIdx - fromIdx);
            for (int i = fromIdx; i < toIdx; ++i)
                taskArr.add(toThrift(tasks[i], tracker));

            ret.setTasks(taskArr);
            ret.setNumTotalTasks(tasks.length);
            return ret;
        }

        /**
         * Converts a JobInProgress object to its corresponding Thrift representation.
         * @param job Input JobInProgress object
         * @param includeTasks Include task information iff true
         */
        public static ThriftJobInProgress toThrift(JobInProgress job, boolean includeTasks, JobTracker tracker) {
            ThriftJobInProgress ret = new ThriftJobInProgress();

            // Take the lock so we can do an atomic copy
            synchronized (job) {
                ret.setDesiredMaps(job.desiredMaps());
                ret.setDesiredReduces(job.desiredReduces());
                ret.setFinishedMaps(job.finishedMaps());
                ret.setFinishedReduces(job.finishedReduces());

                ret.setJobID(toThrift(job.getJobID()));
                ret.setPriority(toThrift(job.getPriority()));
                ret.setProfile(toThrift(job.getProfile()));

                // Status lock is taken here
                ret.setStatus(toThrift(job.getStatus()));

                ret.setStartTime(job.getStartTime());
                ret.setFinishTime(job.getFinishTime());
                ret.setLaunchTime(job.getLaunchTime());
            }

            // No need to hang on to job lock now
            // TODO(henry/bc): By releasing the lock above, getInitialViewTaskList
            // may see a different view of the job and its task list. This
            // could cause inconsistency between the values copied above and
            // the tasks themselves, but no deadlocks/CMEs.
            if (includeTasks) {
                ret.setTasks(getInitialViewTaskList(job, tracker));
            }
            return ret;
        }

        /**
         * There are always two setup tasks and two cleanup tasks by default
         * If one succeeds, the other is killed. We choose not to report those
         * to the UI because they are spurious.
         * This method _always_ return a new array.
         */
        public static TaskInProgress[] sanitizeCleanupSetupTask(TaskInProgress[] tasks) {
            assert tasks.length <= 2; // There should be at most 2 of them

            if (tasks.length != 2)
                return tasks.clone();

            TaskInProgress goodTip = (tasks[1].isRunning() || tasks[1].isComplete()) ? tasks[1] : tasks[0];
            return new TaskInProgress[] { goodTip };
        }

        public static ThriftJobInProgress toThrift(JobInProgress job, JobTracker tracker) {
            return toThrift(job, true, tracker);
        }

        public static ThriftJobProfile toThrift(JobProfile profile) {
            // Takes no locks
            ThriftJobProfile ret = new ThriftJobProfile();
            ret.setJobFile(profile.getJobFile());
            ret.setJobID(toThrift(profile.getJobID()));
            ret.setName(profile.getJobName());
            ret.setQueueName(profile.getQueueName());
            ret.setUser(profile.getUser());
            return ret;
        }

        public static List<ThriftCounterGroup> toThrift(Counters jcs) {
            Collection<String> groupNames = null;
            List<ThriftCounterGroup> ret = null;
            synchronized (jcs) {
                groupNames = new ArrayList<String>(jcs.getGroupNames());
                ret = new ArrayList<ThriftCounterGroup>(groupNames.size());
                for (String s : groupNames) {
                    Group g = jcs.getGroup(s);
                    ThriftCounterGroup tcg = toThrift(g);
                    ret.add(tcg);
                }
            }
            return ret;
        }

        public static ThriftCounterGroup toThrift(Group g) {
            ThriftCounterGroup ret = new ThriftCounterGroup();
            ret.setName(g.getName());
            ret.setDisplayName(g.getDisplayName());
            ret.counters = new HashMap<String, ThriftCounter>();
            for (Counter c : g) {
                ret.counters.put(c.getDisplayName(), toThrift(c));
            }
            return ret;
        }

        public static ThriftCounter toThrift(Counter c) {
            ThriftCounter ret = new ThriftCounter();
            ret.setDisplayName(c.getDisplayName());
            ret.setName(c.getName());
            ret.setValue(c.getValue());
            return ret;
        }

        public static ThriftClusterStatus toThrift(ClusterStatus cs, JobTracker tracker) {
            ThriftClusterStatus tcs = new ThriftClusterStatus();
            tcs.setNumActiveTrackers(cs.getTaskTrackers());
            tcs.setActiveTrackerNames(new ArrayList<String>(cs.getActiveTrackerNames()));
            tcs.setBlacklistedTrackerNames(new ArrayList<String>(cs.getBlacklistedTrackerNames()));
            tcs.setNumBlacklistedTrackers(cs.getBlacklistedTrackers());
            tcs.setNumExcludedNodes(0);
            tcs.setTaskTrackerExpiryInterval(cs.getTTExpiryInterval());
            tcs.setMapTasks(cs.getMapTasks());
            tcs.setReduceTasks(cs.getReduceTasks());
            tcs.setMaxMapTasks(cs.getMaxMapTasks());
            tcs.setMaxReduceTasks(cs.getMaxReduceTasks());
            tcs.setState(cs.getJobTrackerState() == State.INITIALIZING ? JobTrackerState.INITIALIZING
                    : JobTrackerState.RUNNING);
            tcs.setUsedMemory(cs.getUsedMemory());
            tcs.setMaxMemory(cs.getMaxMemory());
            tcs.setTotalSubmissions(tracker.getTotalSubmissions());

            tcs.setHasRecovered(tracker.hasRecovered());
            tcs.setHasRestarted(tracker.hasRestarted());

            tcs.setHostname(tracker.getJobTrackerMachine());
            tcs.setIdentifier(tracker.getTrackerIdentifier());

            tcs.setStartTime(tracker.getStartTime());

            tcs.setHttpPort(tracker.getInfoPort());

            return tcs;
        }

        public static ThriftTaskTrackerStatus toThrift(TaskTrackerStatus t) {
            ThriftTaskTrackerStatus ttts = new ThriftTaskTrackerStatus();
            ttts.setTrackerName(t.getTrackerName());
            ttts.setAvailableSpace(t.getResourceStatus().getAvailableSpace());
            ttts.setFailureCount(t.getFailures());
            ttts.setHost(t.getHost());
            ttts.setHttpPort(t.getHttpPort());
            ttts.setLastSeen(t.getLastSeen());
            ttts.setMapCount(t.countMapTasks());
            ttts.setReduceCount(t.countReduceTasks());
            ttts.setMaxMapTasks(t.getMaxMapSlots());
            ttts.setMaxReduceTasks(t.getMaxReduceSlots());

            ttts.setTotalPhysicalMemory(t.getResourceStatus().getTotalPhysicalMemory());
            ttts.setTotalVirtualMemory(t.getResourceStatus().getTotalVirtualMemory());
            Collection<TaskStatus> tasks = null;
            synchronized (t) {
                tasks = new ArrayList<TaskStatus>(t.getTaskReports());
            }
            for (TaskStatus tr : tasks) {
                ttts.addToTaskReports(toThrift(tr));
            }
            return ttts;
        }

        public static ThriftTaskStatus toThrift(TaskStatus ts) {
            ThriftTaskStatus tts = new ThriftTaskStatus();
            tts.setCounters(new ThriftGroupList(toThrift(ts.getCounters())));
            tts.setDiagnosticInfo(ts.getDiagnosticInfo());
            tts.setFinishTime(ts.getFinishTime());
            tts.setOutputSize(ts.getOutputSize());
            tts.setPhase(toThrift(ts.getPhase()));
            tts.setProgress(ts.getProgress());
            tts.setStartTime(ts.getStartTime());
            tts.setStateString(ts.getStateString());
            tts.setTaskID(toThrift(ts.getTaskID()));
            tts.setTaskTracker(ts.getTaskTracker());
            tts.setState(toThrift(ts.getRunState()));

            if (ts.getIsMap()) {
                // not available in 0.20: tts.setMapFinishTime(ts.getMapFinishTime());
                tts.setShuffleFinishTime(0);
                tts.setSortFinishTime(0);
            } else {
                tts.setMapFinishTime(0);
                tts.setShuffleFinishTime(ts.getShuffleFinishTime());
                tts.setSortFinishTime(ts.getSortFinishTime());
            }
            return tts;
        }

        public static ThriftTaskState toThrift(TaskStatus.State runState) {
            switch (runState) {
            case COMMIT_PENDING:
                return ThriftTaskState.COMMIT_PENDING;
            case RUNNING:
                return ThriftTaskState.RUNNING;
            case SUCCEEDED:
                return ThriftTaskState.SUCCEEDED;
            case FAILED:
                return ThriftTaskState.FAILED;
            case KILLED:
                return ThriftTaskState.KILLED;
            case FAILED_UNCLEAN:
                return ThriftTaskState.FAILED_UNCLEAN;
            case KILLED_UNCLEAN:
                return ThriftTaskState.KILLED_UNCLEAN;
            }
            LOG.info("Unexpected runState in toThrift(TaskStatus.State) - defaulting to FAILED_UNCLEAN");
            return ThriftTaskState.FAILED_UNCLEAN;
        }

        private static ThriftTaskAttemptID toThrift(TaskAttemptID taskID) {
            ThriftTaskAttemptID ret = new ThriftTaskAttemptID();
            ret.setTaskID(toThrift(taskID.getTaskID()));
            ret.setAttemptID(taskID.getId());
            ret.setAsString(taskID.toString());
            return ret;
        }

        private static ThriftTaskID toThrift(TaskID taskID) {
            ThriftTaskID ret = new ThriftTaskID();
            ret.setJobID(toThrift(taskID.getJobID()));
            ret.setTaskID(taskID.getId());
            ret.setTaskType(getThriftTaskType(taskID));
            ret.setAsString(taskID.toString());
            return ret;
        }

        private static ThriftTaskType getThriftTaskType(TaskID task) {
            if (task.isMap()) {
                return ThriftTaskType.MAP;
            } else {
                return ThriftTaskType.REDUCE;
            }
        }

        private static ThriftTaskPhase toThrift(Phase phase) {
            switch (phase) {
            case CLEANUP:
                return ThriftTaskPhase.CLEANUP;
            case STARTING:
                return ThriftTaskPhase.STARTING;
            case MAP:
                return ThriftTaskPhase.MAP;
            case REDUCE:
                return ThriftTaskPhase.REDUCE;
            case SHUFFLE:
                return ThriftTaskPhase.SHUFFLE;
            case SORT:
                return ThriftTaskPhase.SORT;
            }
            LOG.info("Unexpected phase in toThrift(Phase) - defaulting to CLEANUP");
            return ThriftTaskPhase.CLEANUP;
        }

        public static ThriftJobQueueInfo toThrift(JobQueueInfo q) {
            ThriftJobQueueInfo tq = new ThriftJobQueueInfo();
            tq.queueName = q.getQueueName();
            tq.schedulingInfo = q.getSchedulingInfo();
            return tq;
        }

        public static ThriftTaskType getTaskInProgressType(TaskInProgress tip) {
            // Note that the order of the tests are important, since the
            // conditions are not mutually exclusive.
            if (tip.isJobSetupTask())
                return ThriftTaskType.JOB_SETUP;
            if (tip.isJobCleanupTask())
                return ThriftTaskType.JOB_CLEANUP;
            if (tip.isMapTask())
                return ThriftTaskType.MAP;
            else
                return ThriftTaskType.REDUCE;
        }

        /**
         * Returns a TaskList to be presented with the initial view of the JobInProgress.
         * This is unfortunately very much tied to how the JobBrowser UI is presented:
         * - At most 5 most recent tasks, and
         * - At most 5 failed tasks (killed don't count).
         */
        private static ThriftTaskInProgressList getInitialViewTaskList(JobInProgress job, JobTracker jobTracker) {
            List<TaskInProgress> allTips = new ArrayList<TaskInProgress>();
            synchronized (job) {
                allTips.addAll(Arrays.asList(job.getTasks(TaskType.MAP)));
                allTips.addAll(Arrays.asList(job.getTasks(TaskType.REDUCE)));
                allTips.addAll(
                        Arrays.asList(JTThriftUtils.sanitizeCleanupSetupTask(job.getTasks(TaskType.JOB_CLEANUP))));
                allTips.addAll(
                        Arrays.asList(JTThriftUtils.sanitizeCleanupSetupTask(job.getTasks(TaskType.JOB_SETUP))));
            }

            // Sort by reverse time, but put all the genuine failures in front, and the
            // killed tasks at the end. After the sorting, the goal is to have this array:
            //   [ real failures ... others ... failed/killed ]  (all in reverse order)
            // Then we find the boundary between the real failures and the completed, and
            // return a chunk from that boundary, containing the earlier failures and the
            // recent tasks, which is exactly what the UI wants.
            Collections.sort(allTips, new Comparator<TaskInProgress>() {
                public int compare(TaskInProgress foo, TaskInProgress bar) {
                    if (isFailOnError(foo) && !isFailOnError(bar))
                        return -1;
                    if (!isFailOnError(foo) && isFailOnError(bar))
                        return 1;
                    if (foo.isFailed() && !bar.isFailed())
                        return 1;
                    if (!foo.isFailed() && bar.isFailed())
                        return -1;
                    long diff = foo.getExecStartTime() - bar.getExecStartTime();
                    if (diff == 0)
                        return 0;
                    return (diff > 0) ? -1 : 1;
                }
            });

            int offset = 0;
            int count = DEFAULT_NUM_TASKS_TO_SEND;

            // We only want DEFAULT_NUM_FAILURES_TO_SEND number of failures included.
            int nFailures = 0;
            for (TaskInProgress tip : allTips) {
                if (!isFailOnError(tip))
                    break;
                ++nFailures;
            }
            if (nFailures > DEFAULT_NUM_FAILURES_TO_SEND)
                offset = nFailures - DEFAULT_NUM_FAILURES_TO_SEND;

            return JTThriftUtils.toThrift(allTips.toArray(new TaskInProgress[allTips.size()]), jobTracker, offset,
                    offset + count);
        }

        public static ThriftTaskInProgress toThrift(TaskInProgress t, JobTracker tracker) {
            ThriftTaskInProgress ret = new ThriftTaskInProgress();
            TaskStatus[] sts = null;
            ThriftTaskType type = getTaskInProgressType(t);

            synchronized (t) {
                ret.setComplete(t.isComplete());
                ret.setExecFinishTime(t.getExecFinishTime());
                ret.setExecStartTime(t.getExecStartTime());
                ret.setFailed(t.isFailed());
                ret.setProgress(t.getProgress());
                ret.setStartTime(t.getStartTime());
                ret.setTaskID(toThrift(t.getTIPId()));
                // TODO(henry): This can go away when we go on to > 0.20
                ret.taskID.setTaskType(type);

                // getTaskStatuses copies a collection but is not synchronised :(
                sts = t.getTaskStatuses();
            }

            ret.setCounters(new ThriftGroupList(toThrift(t.getCounters())));

            Map<String, ThriftTaskStatus> statusMap = new HashMap<String, ThriftTaskStatus>();
            Map<String, List<String>> dataMap = new HashMap<String, List<String>>();
            for (TaskStatus ts : sts) {
                ThriftTaskAttemptID id = toThrift(ts.getTaskID());
                id.taskID.setTaskType(type);
                statusMap.put(id.getAsString(), toThrift(ts));
                try {
                    // Atomic copy
                    String[] strDiags = tracker.getTaskDiagnostics(ts.getTaskID());
                    // Thrift does not like null values in maps
                    List<String> diag = (strDiags == null ? new ArrayList<String>() : Arrays.asList(strDiags));
                    dataMap.put(id.getAsString(), diag);
                } catch (java.io.IOException e) {
                    // tracker.getTaskDiagnostics is supposed to throw,
                    // but I can't see where it does (and removing the throws clause
                    // doesn't cause a compile failure...), so this is probably
                    // extraneous
                    LOG.warn(e);
                    throw new RuntimeException(e.getMessage());
                }
            }
            ret.setTaskStatuses(statusMap);

            // Takes lock on t
            TaskReport report = t.generateSingleReport();
            ret.setMostRecentState(report.getState());
            ret.setSuccessfulAttempt(toThrift(report.getSuccessfulTaskAttempt()).asString);
            // Because report has a reference to an array from t, we need to synchronize on
            // t to copy it :(
            Collection<TaskAttemptID> attempts = null;
            synchronized (t) {
                attempts = new ArrayList<TaskAttemptID>(report.getRunningTaskAttempts());
            }
            List<String> runningAttempts = new ArrayList<String>(attempts.size());
            for (TaskAttemptID tid : attempts) {
                runningAttempts.add(toThrift(tid).asString);
            }
            ret.setRunningAttempts(runningAttempts);
            ret.setTaskDiagnosticData(dataMap);

            return ret;
        }

        public static ThriftTaskQueryState inferTaskState(TaskInProgress tip) {
            // The ordering of the checks is important
            if (tip.isComplete())
                return ThriftTaskQueryState.SUCCEEDED;
            else if (isFailOnError(tip))
                return ThriftTaskQueryState.FAILED;
            else if (tip.isFailed())
                return ThriftTaskQueryState.KILLED;
            else if (tip.getExecStartTime() == 0)
                return ThriftTaskQueryState.PENDING;
            else
                return ThriftTaskQueryState.RUNNING;
        }

        /**
         * Guess whether the TaskInProgress failed due to genuine error,
         * rather than simply aborted. It checks whether the execution has started
         * for this task. This is not always correct. A running task can still be
         * aborted.
         */
        private static boolean isFailOnError(TaskInProgress tip) {
            return tip.isFailed() && tip.getExecStartTime() != 0;
        }

    }

    public static final Log LOG = LogFactory.getLog(JobTrackerPlugin.class.getName());

    /** Name of the configuration property of the Thrift server address */
    public static final String THRIFT_ADDRESS_PROPERTY = "jobtracker.thrift.address";

    /**
     * Default address and port this server will bind to, in case nothing is found
     * in the configuration object.
     */
    public static final String DEFAULT_THRIFT_ADDRESS = "0.0.0.0:9290";

    private JobTracker jobTracker = null;

    private Configuration conf;

    private ThriftPluginServer thriftServer;

    @Override
    public void start(Object service) {
        LOG.info("Starting ThriftJobTrackerPlugin");
        this.jobTracker = (JobTracker) service;
        try {
            InetSocketAddress address = NetUtils
                    .createSocketAddr(conf.get(THRIFT_ADDRESS_PROPERTY, DEFAULT_THRIFT_ADDRESS));
            this.thriftServer = new ThriftPluginServer(address, new ProcessorFactory());
            thriftServer.setConf(conf);
            thriftServer.start();
            // The port may have been 0, so we update it.
            conf.set(THRIFT_ADDRESS_PROPERTY, address.getHostName() + ":" + thriftServer.getPort());
        } catch (Exception e) {
            LOG.warn("Cannot start Thrift jobtracker plug-in", e);
            throw new RuntimeException("Cannot start Thrift jobtracker plug-in", e);
        }
    }

    @Override
    public void stop() {
        LOG.info("Stopping ThriftJobTrackerPlugin");
        if (thriftServer != null) {
            thriftServer.stop();
        }
    }

    public void close() {
        LOG.info("Closing ThriftJobTrackerPlugin");
        if (thriftServer != null) {
            thriftServer.close();
        }
    }

    /** Java server-side implementation of the 'Jobtracker' Thrift interface. */
    class ThriftHandler extends ThriftHandlerBase implements Jobtracker.Iface {

        public ThriftHandler(ThriftServerContext serverContext) {
            super(serverContext);
        }

        /** Returns the JobTracker's name */
        public String getJobTrackerName(RequestContext ctx) {
            return assumeUserContextAndExecute(ctx, new PrivilegedAction<String>() {
                public String run() {
                    return jobTracker.getJobTrackerMachine();
                }
            });
        }

        /** Returns a large clusterstatus object, augmented with some extra
         * detail from the JobTracker
         */
        public ThriftClusterStatus getClusterStatus(RequestContext ctx) {
            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftClusterStatus>() {
                public ThriftClusterStatus run() {
                    ClusterStatus cs = jobTracker.getClusterStatus(true);
                    return JTThriftUtils.toThrift(cs, jobTracker);
                }
            });
        }

        /** Returns a list of all run-queues available to the JobTracker */
        public ThriftJobQueueList getQueues(RequestContext ctx) throws IOException, TException {
            return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<ThriftJobQueueList>() {
                public ThriftJobQueueList run() throws java.io.IOException {
                    JobQueueInfo queues[] = null;
                    queues = jobTracker.getQueues();

                    ArrayList<ThriftJobQueueInfo> ret = new ArrayList<ThriftJobQueueInfo>(queues.length);

                    for (JobQueueInfo q : queues) {
                        ThriftJobQueueInfo tq = JTThriftUtils.toThrift(q);
                        ret.add(tq);
                    }
                    return new ThriftJobQueueList(ret);
                }
            });
        }

        /** Returns job by id (including task info) */
        public ThriftJobInProgress getJob(RequestContext ctx, ThriftJobID jobID) throws JobNotFoundException {
            final JobID jid = JTThriftUtils.fromThrift(jobID);
            JobInProgress job = assumeUserContextAndExecute(ctx, new PrivilegedAction<JobInProgress>() {
                public JobInProgress run() {
                    return jobTracker.getJob(jid);
                }
            });
            if (job == null) {
                throw new JobNotFoundException();
            }
            return JTThriftUtils.toThrift(job, jobTracker);
        }

        /** Returns all running jobs (does not include task info) */
        public ThriftJobList getRunningJobs(RequestContext ctx) {
            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftJobList>() {
                public ThriftJobList run() {
                    // Atomic copy
                    List<JobInProgress> jobs = jobTracker.getRunningJobs();
                    ArrayList<ThriftJobInProgress> ret = new ArrayList<ThriftJobInProgress>(jobs.size());

                    for (JobInProgress job : jobs) {
                        ret.add(JTThriftUtils.toThrift(job, false, jobTracker));
                    }
                    return new ThriftJobList(ret);
                }
            });
        }

        /** Returns all completed jobs (does not include task info) */
        public ThriftJobList getCompletedJobs(RequestContext ctx) {
            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftJobList>() {
                public ThriftJobList run() {
                    List<JobInProgress> jobs = null;
                    synchronized (jobTracker) {
                        jobs = jobTracker.completedJobs();
                    }
                    ArrayList<ThriftJobInProgress> ret = new ArrayList<ThriftJobInProgress>(jobs.size());

                    for (JobInProgress job : jobs) {
                        ret.add(JTThriftUtils.toThrift(job, false, jobTracker));
                    }
                    return new ThriftJobList(ret);
                }
            });
        }

        /** Returns all failed jobs (does not include task info) */
        public ThriftJobList getFailedJobs(RequestContext ctx) {
            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftJobList>() {
                public ThriftJobList run() {
                    List<JobInProgress> jobs = null;
                    synchronized (jobTracker) {
                        jobs = jobTracker.failedJobs();
                    }
                    List<ThriftJobInProgress> ret = new ArrayList<ThriftJobInProgress>(jobs.size());
                    for (JobInProgress job : jobs) {
                        if (job.getStatus().getRunState() == JobStatus.FAILED) {
                            ret.add(JTThriftUtils.toThrift(job, false, jobTracker));
                        }
                    }
                    return new ThriftJobList(ret);
                }
            });
        }

        /** Returns all killed jobs (does not include task info) */
        public ThriftJobList getKilledJobs(RequestContext ctx) {
            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftJobList>() {
                public ThriftJobList run() {
                    List<JobInProgress> jobs = null;
                    synchronized (jobTracker) {
                        jobs = jobTracker.failedJobs();
                    }
                    List<ThriftJobInProgress> ret = new ArrayList<ThriftJobInProgress>(jobs.size());
                    for (JobInProgress job : jobs) {
                        if (job.getStatus().getRunState() == JobStatus.KILLED) {
                            ret.add(JTThriftUtils.toThrift(job, false, jobTracker));
                        }
                    }
                    return new ThriftJobList(ret);
                }
            });
        }

        /** Returns all running / failed / completed jobs (does not include task info) */
        public ThriftJobList getAllJobs(RequestContext ctx) {
            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftJobList>() {
                public ThriftJobList run() {
                    List<JobInProgress> jobList = new ArrayList<JobInProgress>();
                    jobList.addAll(jobTracker.getRunningJobs());
                    synchronized (jobTracker) {
                        jobList.addAll(jobTracker.failedJobs());
                        jobList.addAll(jobTracker.completedJobs());
                    }
                    List<ThriftJobInProgress> ret = new ArrayList<ThriftJobInProgress>();
                    for (JobInProgress job : jobList) {
                        ret.add(JTThriftUtils.toThrift(job, false, jobTracker));
                    }
                    return new ThriftJobList(ret);
                }
            });
        }

        /**
         * Return the count of jobs, broken down by status, for a given user.
         */
        public ThriftUserJobCounts getUserJobCounts(RequestContext ctx, final String user) {
            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftUserJobCounts>() {
                public ThriftUserJobCounts run() {
                    ThriftUserJobCounts ret = new ThriftUserJobCounts(0, 0, 0, 0, 0);

                    JobStatus[] allJobs = jobTracker.getAllJobs();
                    for (JobStatus js : allJobs) {
                        if (!js.getUsername().equals(user))
                            continue;
                        switch (js.getRunState()) {
                        case JobStatus.PREP:
                            ++ret.nPrep;
                            break;
                        case JobStatus.RUNNING:
                            ++ret.nRunning;
                            break;
                        case JobStatus.SUCCEEDED:
                            ++ret.nSucceeded;
                            break;
                        case JobStatus.FAILED:
                            ++ret.nFailed;
                            break;
                        case JobStatus.KILLED:
                            ++ret.nKilled;
                            break;
                        default:
                            LOG.error("Unknown JobStatus " + js.getRunState() + " for job id "
                                    + js.getJobID().getId());
                        }
                    }
                    return ret;
                }
            });
        }

        /**
         * Return a (possibly incomplete) list of tasks.
         */
        public ThriftTaskInProgressList getTaskList(RequestContext ctx, ThriftJobID thriftJobID,
                Set<ThriftTaskType> types, Set<ThriftTaskQueryState> states, String text, int count, int offset)
                throws JobNotFoundException {
            final JobID jid = JTThriftUtils.fromThrift(thriftJobID);
            JobInProgress job = assumeUserContextAndExecute(ctx, new PrivilegedAction<JobInProgress>() {
                public JobInProgress run() {
                    return jobTracker.getJob(jid);
                }
            });

            if (job == null)
                throw new JobNotFoundException();

            // Gather all the tasks of the matching type
            List<TaskInProgress> allTips = new ArrayList<TaskInProgress>();
            synchronized (job) {
                if (types.contains(ThriftTaskType.MAP))
                    allTips.addAll(Arrays.asList(job.getTasks(TaskType.MAP)));
                if (types.contains(ThriftTaskType.REDUCE))
                    allTips.addAll(Arrays.asList(job.getTasks(TaskType.REDUCE)));
                if (types.contains(ThriftTaskType.JOB_CLEANUP))
                    allTips.addAll(Arrays
                            .asList(JTThriftUtils.sanitizeCleanupSetupTask(job.getTasks(TaskType.JOB_CLEANUP))));
                if (types.contains(ThriftTaskType.JOB_SETUP))
                    allTips.addAll(Arrays
                            .asList(JTThriftUtils.sanitizeCleanupSetupTask(job.getTasks(TaskType.JOB_SETUP))));
            }

            // Are the arguments out of bound?
            if (count < 0 || offset < 0 || offset >= allTips.size()) {
                LOG.error("Bad arguments to getTaskList(): count " + count + "; offset " + offset
                        + "; while total tasks count is " + allTips.size());
                return JTThriftUtils.toThrift(new TaskInProgress[0], jobTracker, 0, 0);
            }

            List<TaskInProgress> matches = null;
            if (text == null)
                text = "";
            else
                text = text.trim();

            boolean doFilterStates = (states.size() != ThriftTaskQueryState.class.getEnumConstants().length);
            boolean doFilterText = !text.isEmpty();

            if (doFilterStates || doFilterText) {
                text = text.toUpperCase();
                matches = new ArrayList<TaskInProgress>();

                // Note that it's important to finish all matching, regardless
                // of the requested count, because we need to report the total
                // number of matches.
                for (TaskInProgress tip : allTips) {
                    ThriftTaskQueryState qstate = null;

                    if (doFilterStates) {
                        // Do filter by states
                        qstate = JTThriftUtils.inferTaskState(tip);
                        if (!states.contains(qstate))
                            continue;
                    }

                    if (doFilterText) {
                        // Match against (1) state, (2) most recent state, (3) ID
                        if (qstate == null)
                            qstate = JTThriftUtils.inferTaskState(tip);
                        String qstateStr = qstate.toString();
                        if (!qstateStr.contains(text) && !tip.getTIPId().toString().toUpperCase().contains(text)
                                && !tip.generateSingleReport().getState().toUpperCase().contains(text))
                            continue;
                    }

                    matches.add(tip);
                }
            } else {
                // If not filtering, we just do offset/limit into the full list
                matches = allTips;
            }

            return JTThriftUtils.toThrift(matches.toArray(new TaskInProgress[matches.size()]), jobTracker, offset,
                    offset + count);
        }

        /** Returns the task identified by the id */
        public ThriftTaskInProgress getTask(RequestContext ctx, ThriftTaskID ttaskId)
                throws JobNotFoundException, TaskNotFoundException {
            final TaskID taskId = JTThriftUtils.fromThrift(ttaskId);
            final JobID jobId = JTThriftUtils.fromThrift(ttaskId.getJobID());
            final JobInProgress job = assumeUserContextAndExecute(ctx, new PrivilegedAction<JobInProgress>() {
                public JobInProgress run() {
                    return jobTracker.getJob(jobId);
                }
            });
            if (job == null)
                throw new JobNotFoundException();
            TaskInProgress tip = assumeUserContextAndExecute(ctx, new PrivilegedAction<TaskInProgress>() {
                public TaskInProgress run() {
                    return job.getTaskInProgress(taskId);
                }
            });
            if (tip == null)
                throw new TaskNotFoundException();
            return JTThriftUtils.toThrift(tip, jobTracker);
        }

        /** Returns the set of counters associated with a given job */
        public ThriftGroupList getJobCounters(RequestContext ctx, final ThriftJobID jobID)
                throws JobNotFoundException {
            Counters jcs;
            try {
                jcs = assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Counters>() {
                    public Counters run() throws java.io.IOException {
                        return jobTracker.getJobCounters(JTThriftUtils.fromThrift(jobID));
                    }
                });
            } catch (IOException e) {
                throw new JobNotFoundException();
            }
            if (jcs == null) {
                throw new JobNotFoundException();
            }
            return new ThriftGroupList(JTThriftUtils.toThrift(jcs));
        }

        public ThriftJobCounterRollups getJobCounterRollups(RequestContext ctx, final ThriftJobID jobID)
                throws JobNotFoundException {
            JobInProgress jip = assumeUserContextAndExecute(ctx, new PrivilegedAction<JobInProgress>() {
                public JobInProgress run() {
                    return jobTracker.getJob(JTThriftUtils.fromThrift(jobID));
                }
            });
            if (jip == null) {
                throw new JobNotFoundException();
            }

            ThriftJobCounterRollups ret = new ThriftJobCounterRollups();
            Counters mc = new Counters();
            jip.getMapCounters(mc);
            Counters rc = new Counters();
            jip.getReduceCounters(mc);
            ret.mapCounters = new ThriftGroupList(JTThriftUtils.toThrift(mc));
            ret.reduceCounters = new ThriftGroupList(JTThriftUtils.toThrift(rc));
            ret.jobCounters = new ThriftGroupList(JTThriftUtils.toThrift(jip.getJobCounters()));

            return ret;
        }

        /** Returns only active TaskTrackerStatus objects */
        public ThriftTaskTrackerStatusList getActiveTrackers(RequestContext ctx) {
            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftTaskTrackerStatusList>() {
                public ThriftTaskTrackerStatusList run() {
                    Collection<TaskTrackerStatus> active = jobTracker.activeTaskTrackers();
                    List<ThriftTaskTrackerStatus> trackers = new ArrayList<ThriftTaskTrackerStatus>(active.size());
                    for (TaskTrackerStatus t : active) {
                        trackers.add(JTThriftUtils.toThrift(t));
                    }
                    return new ThriftTaskTrackerStatusList(trackers);
                }
            });
        }

        /** Returns only blacklisted TaskTrackerStatus objects */
        public ThriftTaskTrackerStatusList getBlacklistedTrackers(RequestContext ctx) {
            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftTaskTrackerStatusList>() {
                public ThriftTaskTrackerStatusList run() {
                    Collection<TaskTrackerStatus> black = jobTracker.blacklistedTaskTrackers();
                    List<ThriftTaskTrackerStatus> trackers = new ArrayList<ThriftTaskTrackerStatus>(black.size());
                    for (TaskTrackerStatus t : black) {
                        trackers.add(JTThriftUtils.toThrift(t));
                    }
                    return new ThriftTaskTrackerStatusList(trackers);
                }
            });
        }

        /** Returns all TaskTrackerStatus objects */
        public ThriftTaskTrackerStatusList getAllTrackers(RequestContext ctx) {
            return assumeUserContextAndExecute(ctx, new PrivilegedAction<ThriftTaskTrackerStatusList>() {
                public ThriftTaskTrackerStatusList run() {
                    Collection<TaskTrackerStatus> all = jobTracker.taskTrackers();
                    List<ThriftTaskTrackerStatus> trackers = new ArrayList<ThriftTaskTrackerStatus>(all.size());
                    for (TaskTrackerStatus t : all) {
                        trackers.add(JTThriftUtils.toThrift(t));
                    }
                    return new ThriftTaskTrackerStatusList(trackers);
                }
            });
        }

        /** Returns a single TaskTrackerStatus object by name */
        public ThriftTaskTrackerStatus getTracker(RequestContext ctx, final String name)
                throws TaskTrackerNotFoundException {
            ThriftTaskTrackerStatus ret = assumeUserContextAndExecute(ctx,
                    new PrivilegedAction<ThriftTaskTrackerStatus>() {
                        public ThriftTaskTrackerStatus run() {
                            Collection<TaskTrackerStatus> all = jobTracker.taskTrackers();
                            for (TaskTrackerStatus t : all) {
                                if (t.getTrackerName().equals(name))
                                    return JTThriftUtils.toThrift(t);
                            }
                            return null;
                        }
                    });
            if (ret != null)
                return ret;
            else
                throw new TaskTrackerNotFoundException();
        }

        /** Returns the current time in ms on this machine */
        public long getCurrentTime(RequestContext ctx) {
            // This is the call that the JT uses to determine the current time
            return System.currentTimeMillis();
        }

        /** Reads the local jobconf XML file for a given job */
        public String getJobConfXML(RequestContext ctx, final ThriftJobID jobID) throws IOException {
            return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<String>() {
                public String run() throws java.io.IOException {
                    /* This always returns a filename of hadoop.log.dir + "/" + jobid + "_conf.xml"
                     * Better check that jobid doesn't contain anything nasty.
                     */
                    JobID jid = JTThriftUtils.fromThrift(jobID);
                    String jidstring = jid.toString();
                    if (jidstring.contains(File.separator) || jidstring.contains(File.pathSeparator)) {
                        throw new IllegalArgumentException("jobConf arguments can't contain path separators");
                    }
                    String jobFilePath = JobTracker.getLocalJobFilePath(jid);

                    StringBuffer fileData = new StringBuffer(1000);
                    BufferedReader reader;
                    reader = new BufferedReader(new FileReader(jobFilePath));
                    char[] buf = new char[1024];
                    int numRead = 0;
                    while ((numRead = reader.read(buf)) > 0) {
                        fileData.append(buf, 0, numRead);
                    }
                    reader.close();
                    return fileData.toString();
                }
            });
        }

        /** Kill a job by jobid */
        public void killJob(final RequestContext ctx, final ThriftJobID jobID)
                throws IOException, JobNotFoundException {
            ThriftJobInProgress job = assumeUserContextAndExecute(ctx,
                    new PrivilegedExceptionAction<ThriftJobInProgress>() {
                        public ThriftJobInProgress run() throws JobNotFoundException {
                            return getJob(ctx, jobID);
                        }
                    });
            if (job == null) {
                throw new JobNotFoundException();
            }
            JobID jid = JTThriftUtils.fromThrift(jobID);
            try {
                jobTracker.killJob(jid);
            } catch (Throwable t) {
                LOG.info("killJob failed", t);
                throw ThriftUtils.toThrift(t);
            }
        }

        /** Kill a task attempt by taskattemptid */
        public void killTaskAttempt(RequestContext ctx, ThriftTaskAttemptID attemptID)
                throws IOException, TaskAttemptNotFoundException, JobNotFoundException {
            final TaskAttemptID taskid = JTThriftUtils.fromThrift(attemptID);
            final JobID jid = JTThriftUtils.fromThrift(attemptID.taskID.jobID);

            final JobInProgress job = assumeUserContextAndExecute(ctx, new PrivilegedAction<JobInProgress>() {
                public JobInProgress run() {
                    return jobTracker.getJob(jid);
                }
            });
            if (job == null) {
                throw new JobNotFoundException();
            }

            final TaskInProgress tip = assumeUserContextAndExecute(ctx, new PrivilegedAction<TaskInProgress>() {
                public TaskInProgress run() {
                    return job.getTaskInProgress(taskid.getTaskID());
                }
            });
            if (tip == null) {
                throw new TaskAttemptNotFoundException();
            }

            TaskStatus status = assumeUserContextAndExecute(ctx, new PrivilegedAction<TaskStatus>() {
                public TaskStatus run() {
                    return tip.getTaskStatus(taskid);
                }
            });
            if (status == null) {
                throw new TaskAttemptNotFoundException();
            }

            assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Void>() {
                public Void run() throws java.io.IOException {
                    // Second parameter means always kill, don't fail
                    if (!jobTracker.killTask(taskid, true)) {
                        throw new RuntimeException();
                    }
                    return null;
                }
            });
        }

        /** Set a job's priority */
        public void setJobPriority(RequestContext ctx, final ThriftJobID jobID, final ThriftJobPriority priority)
                throws IOException, JobNotFoundException {
            final JobID jid = JTThriftUtils.fromThrift(jobID);
            JobInProgress job = assumeUserContextAndExecute(ctx, new PrivilegedAction<JobInProgress>() {
                public JobInProgress run() {
                    return jobTracker.getJob(jid);
                }
            });
            if (job == null) {
                throw new JobNotFoundException();
            }
            assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<Void>() {
                public Void run() throws java.io.IOException {
                    jobTracker.setJobPriority(jid, priority.toString());
                    return null;
                }
            });
        }

        @Override
        public ThriftDelegationToken getDelegationToken(RequestContext ctx, final String renewer)
                throws IOException, TException {
            return assumeUserContextAndExecute(ctx, new PrivilegedExceptionAction<ThriftDelegationToken>() {
                public ThriftDelegationToken run() throws java.io.IOException {
                    Token<DelegationTokenIdentifier> delegationToken;
                    try {
                        delegationToken = jobTracker.getDelegationToken(new Text(renewer));
                    } catch (InterruptedException e) {
                        throw new java.io.IOException(e);
                    }

                    return ThriftUtils.toThrift(delegationToken, JobTracker.getAddress(conf));
                }
            });
        }
    }

    /** Implementation of configurable interface */
    public Configuration getConf() {
        return conf;
    }

    /** Implementation of configurable interface */
    public void setConf(Configuration conf) {
        this.conf = conf;
    }

    /** Creates Thrift processors to handle incoming requests */
    class ProcessorFactory extends TProcessorFactory {

        ProcessorFactory() {
            super(null);
        }

        @Override
        public TProcessor getProcessor(TTransport t) {
            ThriftServerContext context = new ThriftServerContext(t);
            Jobtracker.Iface impl = ThriftUtils.SecurityCheckingProxy.create(conf, new ThriftHandler(context),
                    Jobtracker.Iface.class);
            return new Jobtracker.Processor(impl);
        }
    }
}