Example usage for org.apache.hadoop.mapreduce ClusterMetrics getReduceSlotCapacity

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce ClusterMetrics getReduceSlotCapacity.

Prototype

public int getReduceSlotCapacity() 

Source Link

Document

Get the total number of reduce slots in the cluster.

Usage

From source file:skewtune.mapreduce.STJobTracker.java

License:Apache License

PartitionPlanner.ClusterInfo getClusterAvailability(ReactionContext context, long now)
        throws IOException, InterruptedException {
    ClusterMetrics metrics = cluster.getClusterStatus();
    TaskAttemptID attemptId = context.getTargetAttemptID();
    TaskType type = attemptId == null ? context.getTaskID().getTaskType() : attemptId.getTaskType();

    int maxSlots = type == TaskType.MAP ? metrics.getMapSlotCapacity() : metrics.getReduceSlotCapacity();
    int runningSlots = type == TaskType.MAP ? metrics.getRunningMaps() : metrics.getRunningReduces();
    int runningSkewTune = 0;
    double[] remainingTimes = new double[maxSlots];
    int from = maxSlots;

    // if this is a speculative REDUCE, the original slot becomes available. We should make it available.
    boolean availRightNow = attemptId != null && type == TaskType.REDUCE && context.getTimePerByte() == 0.f;

    synchronized (this) {
        // FIXME this only involves tasks that are scheduled and running
        // we should keep an expected information as well.

        // on planning, we should add the planned tasks and getClusterAvailability should
        // incorporate any planned stuffs in it.

        // the information required:
        // Map<JobID, [long planned at, for tasks -- estimated runtime]>
        // on first heartbeat from each task, we remove each information.

        for (Map.Entry<TaskAttemptID, TaskInProgress> e : taskidToTIP.entrySet()) {
            TaskAttemptID taskid = e.getKey();
            if (taskid.getTaskType() == type) {
                // extra check
                if (availRightNow && taskid.equals(attemptId))
                    continue; // this will become available immediately

                TaskInProgress tip = e.getValue();
                double t = tip.getRemainingTime(taskid, now);
                if (t > 0.) {
                    remainingTimes[--from] = tip.getRemainingTime(taskid, now);
                    ++runningSkewTune;// ww w . j  a  v  a2  s .co  m
                    if (from == 0)
                        break;
                }
            }
        }
        if (from > 0) {
            synchronized (plannedJobs) {
                for (Map.Entry<JobID, PlannedJob> e : this.plannedJobs.entrySet()) {
                    PlannedJob plan = e.getValue();
                    from = plan.fillCompletionTime(type, now, remainingTimes, from);
                    if (from == 0)
                        break;
                }
            }
        }
    }
    Arrays.sort(remainingTimes, from, maxSlots);

    if (LOG.isDebugEnabled()) {
        LOG.debug("cluster availability = " + Arrays.toString(remainingTimes));
    }

    // FIXME incorporate other tasks that are not SkewTune

    return new PartitionPlanner.ClusterInfo(type, maxSlots, runningSlots, runningSkewTune, remainingTimes,
            maxSlots);
}

---