com.addthis.hydra.job.HostFailWorker.java Source code

Java tutorial

Introduction

Here is the source code for com.addthis.hydra.job.HostFailWorker.java

Source

/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.addthis.hydra.job;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

import com.addthis.basis.util.Parameter;

import com.addthis.hydra.job.mq.HostState;
import com.addthis.hydra.job.mq.JobKey;
import com.addthis.hydra.job.spawn.HostManager;
import com.addthis.hydra.job.spawn.Spawn;
import com.addthis.maljson.JSONException;
import com.addthis.maljson.JSONObject;

import com.google.common.collect.ImmutableSet;

import com.yammer.metrics.Metrics;
import com.yammer.metrics.core.Counter;

import org.apache.commons.lang3.tuple.Pair;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class HostFailWorker {

    private static final Logger log = LoggerFactory.getLogger(HostFailWorker.class);
    private final AtomicBoolean newAdditions = new AtomicBoolean(false); // True if a host has been recently added to the queue
    private final AtomicBoolean obeyTaskSlots = new AtomicBoolean(true); // Whether spawn should honor the max task slots when moving tasks to fail hosts
    private final HostFailState hostFailState;
    private final Spawn spawn;
    private final HostManager hostManager;

    // Perform host-failure related operations at a given interval
    private static final long hostFailDelayMillis = Parameter.longValue("host.fail.delay", 15_000);
    // Quiet period between when host is failed in UI and when Spawn begins failure-related operations
    private static final long hostFailQuietPeriod = Parameter.longValue("host.fail.quiet.period", 20_000);

    // Don't rebalance additional tasks if spawn is already rebalancing at least this many.
    private static final int maxMovingTasks = Parameter.intValue("host.fail.maxMovingTasks", 6);
    // Use a smaller max when a disk is being failed, to avoid a 'thundering herds' scenario
    private static final int maxMovingTasksDiskFull = Parameter.intValue("host.fail.maxMovingTasksDiskFull", 2);

    private static final Counter failHostCount = Metrics.newCounter(Spawn.class, "failHostCount");

    // Various keys used to make JSON objects to send to the UI
    private static final String infoHostsKey = "uuids";
    private static final String infoDeadFsKey = "deadFs";
    private static final String infoWarningKey = "warning";
    private static final String infoPrefailCapacityKey = "prefail";
    private static final String infoPostfailCapacityKey = "postfail";
    private static final String infoFatalWarningKey = "fatal";
    private final ScheduledExecutorService executorService;

    public HostFailWorker(Spawn spawn, HostManager hostManager, ScheduledExecutorService executorService) {
        this.spawn = spawn;
        this.hostManager = hostManager;
        this.executorService = executorService;
        hostFailState = new HostFailState(spawn);
        hostFailState.loadState();
    }

    /** Initializes scheduled execution of fail host task **/
    public void initFailHostTaskSchedule() {
        if (executorService != null) {
            if (hostFailState.nextHostToFail() != null) {
                queueFailNextHost();
            }
            executorService.scheduleWithFixedDelay(new FailHostTask(true), hostFailDelayMillis, hostFailDelayMillis,
                    TimeUnit.MILLISECONDS);
        }
    }

    /**
     * Mark a series of hosts for failure
     *
     * @param hostIds        A comma-separated list of host uuids
     * @param state      The state of the hosts being failed
     */
    public void markHostsToFail(String hostIds, FailState state) {
        if (hostIds != null) {
            for (String host : hostIds.split(",")) {
                hostFailState.putHost(host, state);
                spawn.sendHostUpdateEvent(spawn.hostManager.getHostState(host));
            }
            queueFailNextHost();
        }
    }

    /**
     * Retrieve an enum describing whether/how a host has been failed (for programmatic purposes)
     *
     * @param hostId A host uuid to check
     * @return A FailState object describing whether the host has been failed
     */
    public FailState getFailureState(String hostId) {
        return hostFailState.getState(hostId);
    }

    public boolean shouldKickTasks(String hostId) {
        FailState failState = getFailureState(hostId);
        // A Failing_Fs_Okay host is nominally fine for the time being. It should be allowed to run tasks.
        return failState == FailState.ALIVE || failState == FailState.FAILING_FS_OKAY;
    }

    /**
     * Retrieve a human-readable string describing whether/how a host has been failed
     *
     * @param hostId A host uuid to check
     * @param up     Whether the host is up
     * @return A String describing the host's failure state (mainly for the UI)
     */
    public String getFailureStateString(String hostId, boolean up) {
        FailState failState = getFailureState(hostId);
        switch (failState) {
        case ALIVE:
            return up ? "up" : "down";
        case FAILING_FS_DEAD:
            return "queued to fail (fs dead)";
        case FAILING_FS_OKAY:
            return "queued to fail (fs okay)";
        case DISK_FULL:
            return "disk near full; moving tasks off";
        default:
            return "UNKNOWN";
        }
    }

    /**
     * Cancel the failure of one or more hosts
     *
     * @param hostIds A comma-separated list of host uuids
     */
    public void removeHostsForFailure(String hostIds) {
        if (hostIds != null) {
            for (String host : hostIds.split(",")) {
                hostFailState.removeHost(host);
                spawn.sendHostUpdateEvent(spawn.hostManager.getHostState(host));
            }
        }
    }

    /**
     * Decide whether a given host can be failed based on whether other minions in the cluster are up
     *
     * @param failedHostUuid The host to be failed
     * @return True only if there are no down hosts that would need to be up in order to correctly fail the host
     */
    protected boolean checkHostStatesForFailure(String failedHostUuid) {
        Collection<HostState> hostStates = spawn.hostManager.listHostStatus(null);
        for (HostState hostState : hostStates) {
            if (!failedHostUuid.equals(hostState.getHostUuid())
                    && shouldBlockHostFailure(ImmutableSet.of(failedHostUuid), hostState)) {
                log.warn("Unable to fail host: " + failedHostUuid + " because one of the minions ("
                        + hostState.getHostUuid() + ") on " + hostState.getHost()
                        + " is currently down.  Retry when all minions are available");
                return false;
            }
        }
        return true;
    }

    /**
     * Fail a host. For any tasks with replicas on that host, move these replicas elsewhere. For any tasks with live copies on the host,
     * promote a replica, then make a new replica somewhere else.
     */
    private void markHostDead(String failedHostUuid) {
        if (failedHostUuid == null || !checkHostStatesForFailure(failedHostUuid)) {
            return;
        }
        spawn.markHostStateDead(failedHostUuid);
        hostFailState.removeHost(failedHostUuid);
        failHostCount.inc();
    }

    /**
     * Before failing host(s), check if a different host needs to be up to perform the failure operation.
     *
     * @param failedHostUUIDs The hosts being failed
     * @param hostState       The host to check
     * @return True if the host is down
     */
    private boolean shouldBlockHostFailure(Set<String> failedHostUUIDs, HostState hostState) {
        if (hostState == null || hostState.isDead() || hostState.isUp()) {
            return false;
        }
        for (JobKey jobKey : hostState.allJobKeys()) // never null due to implementation
        {
            JobTask task = spawn.getTask(jobKey);
            if (task != null
                    && (failedHostUUIDs.contains(task.getHostUUID()) || task.hasReplicaOnHosts(failedHostUUIDs))) {
                // There is a task on a to-be-failed host that has a copy on a host that is down. We cannot fail for now.
                return true;
            }
        }
        return false;
    }

    /**
     * After receiving a host failure request, queue an event to fail that host after a quiet period
     */
    private void queueFailNextHost() {
        if (newAdditions.compareAndSet(false, true)) {
            executorService.schedule(new FailHostTask(false), hostFailQuietPeriod, TimeUnit.MILLISECONDS);
        }
    }

    /**
     * Find the next host on the fail queue, considering filesystem-dead hosts first. Perform the correct actions and remove from the queue if appropriate.
     */
    private void failNextHost() {
        Pair<String, FailState> hostToFail = hostFailState.nextHostToFail();
        if (hostToFail != null) {
            String failedHostUuid = hostToFail.getLeft();
            FailState failState = hostToFail.getRight();
            if (failState == FailState.FAILING_FS_DEAD) {
                // File system is dead. Relocate all tasks ASAP.
                markHostDead(failedHostUuid);
                spawn.getSpawnBalancer().fixTasksForFailedHost(spawn.hostManager.listHostStatus(null),
                        failedHostUuid);
            } else {
                HostState host = spawn.hostManager.getHostState(failedHostUuid);
                if (host == null) {
                    // Host is gone or has no more tasks. Simply mark it as failed.
                    markHostDead(failedHostUuid);
                    return;
                }
                boolean diskFull = (failState == FailState.DISK_FULL);
                if (!diskFull && spawn.getSystemManager().isQuiesced()) {
                    // If filesystem is okay, don't do any moves while spawn is quiesced.
                    return;
                }
                int taskMovingMax = diskFull ? maxMovingTasksDiskFull : maxMovingTasks;
                int tasksToMove = taskMovingMax - countRebalancingTasks();
                if (tasksToMove <= 0) {
                    // Spawn is already moving enough tasks; hold off until later
                    return;
                }
                List<JobTaskMoveAssignment> assignments = spawn.getSpawnBalancer()
                        .pushTasksOffDiskForFilesystemOkayFailure(host, tasksToMove);
                // Use available task slots to push tasks off the host in question. Not all of these assignments will necessarily be moved.
                spawn.executeReallocationAssignments(assignments, !diskFull && obeyTaskSlots.get());
                if (failState == FailState.FAILING_FS_OKAY && assignments.isEmpty() && host.countTotalLive() == 0) {
                    // Found no tasks on the failed host, so fail it for real.
                    markHostDead(failedHostUuid);
                    spawn.getSpawnBalancer().fixTasksForFailedHost(
                            spawn.hostManager.listHostStatus(host.getMinionTypes()), failedHostUuid);
                }
            }
        }
    }

    public void setObeyTaskSlots(boolean obey) {
        obeyTaskSlots.set(obey);
    }

    /**
     * Retrieve information about the implications of failing a host, to inform/warn a user in the UI
     *
     * @param hostsToFail The hosts that will be failed
     * @return A JSONObject with various data about the implications of the failure
     */
    public JSONObject getInfoForHostFailure(String hostsToFail, boolean deadFilesystem) throws JSONException {
        if (hostsToFail == null) {
            return new JSONObject();
        }
        HashSet<String> ids = new HashSet<>(Arrays.asList(hostsToFail.split(",")));
        long totalClusterAvail = 0, totalClusterUsed = 0, hostAvail = 0;
        List<String> hostsDown = new ArrayList<>();
        for (HostState host : spawn.hostManager.listHostStatus(null)) {
            // Sum up disk availability across the entire cluster and across the specified hosts
            if (host.getMax() != null && host.getUsed() != null) {
                if (getFailureState(host.getHostUuid()) == FailState.ALIVE) {
                    totalClusterAvail += host.getMax().getDisk();
                    totalClusterUsed += host.getUsed().getDisk();
                }
                if (ids.contains(host.getHostUuid())) {
                    hostAvail += host.getMax().getDisk();
                }
            }
            if (!ids.contains(host.getHostUuid()) && shouldBlockHostFailure(ids, host)) {
                hostsDown.add(host.getHostUuid() + " on " + host.getHost());
            }
        }
        // Guard against division by zero in the case of unexpected values
        totalClusterAvail = Math.max(1, totalClusterAvail);
        hostAvail = Math.min(totalClusterAvail - 1, hostAvail);
        return constructInfoMessage(hostsToFail, deadFilesystem, (double) (totalClusterUsed) / totalClusterAvail,
                (double) (totalClusterUsed) / (totalClusterAvail - hostAvail), hostsDown);
    }

    /**
     * Create the info message about host message using some raw values
     *
     * @param prefailCapacity  The capacity the cluster had before the failure
     * @param postfailCapacity The capacity the cluster would have after the failure
     * @param hostsDown        Any hosts that are down that might temporarily prevent failure
     * @return A JSONObject encapsulating the above information.
     * @throws JSONException
     */
    private JSONObject constructInfoMessage(String hostsToFail, boolean deadFilesystem, double prefailCapacity,
            double postfailCapacity, List<String> hostsDown) throws JSONException {
        JSONObject obj = new JSONObject();
        obj.put(infoHostsKey, hostsToFail);
        obj.put(infoDeadFsKey, deadFilesystem);
        obj.put(infoPrefailCapacityKey, prefailCapacity);
        obj.put(infoPostfailCapacityKey, postfailCapacity);
        if (Double.isNaN(postfailCapacity)) {
            obj.put(infoFatalWarningKey, "Cannot fail all hosts from a cluster");
        } else if (postfailCapacity >= 1) {
            obj.put(infoFatalWarningKey, "Insufficient cluster disk capacity");
        }
        if (!hostsDown.isEmpty()) {
            obj.put(infoWarningKey,
                    "Some hosts are down. Host failure could be delayed until they return: " + hostsDown);
        }
        return obj;
    }

    /**
     * A simple wrapper around failNextHost that is run by the failExecutor.
     */
    private class FailHostTask implements Runnable {

        private final boolean skipIfNewAdditions;

        public FailHostTask(boolean skipIfNewAdditions) {
            this.skipIfNewAdditions = skipIfNewAdditions;
        }

        @Override
        public void run() {
            updateFullMinions();
            if (skipIfNewAdditions && newAdditions.get()) {
                return;
            }
            try {
                failNextHost();
            } catch (Exception e) {
                log.warn("Exception while failing host: {}", e.getMessage(), e);
            } finally {
                newAdditions.set(false);
            }
        }
    }

    public void updateFullMinions() {
        for (HostState hostState : spawn.hostManager.listHostStatus(null)) {
            if (hostState == null || hostState.isDead() || !hostState.isUp()) {
                continue;
            }
            String hostId = hostState.getHostUuid();
            if (spawn.getSpawnBalancer().isDiskFull(hostState)) {
                markHostsToFail(hostId, HostFailWorker.FailState.DISK_FULL);
            } else if (getFailureState(hostId) == HostFailWorker.FailState.DISK_FULL) {
                // Host was previously full, but isn't anymore. Take it off the disk_full list.
                hostFailState.removeHost(hostId);
            }
        }
    }

    private int countRebalancingTasks() {
        int count = 0;
        for (Job job : spawn.listJobs()) {
            if (job.getState() == JobState.REBALANCE) {
                for (JobTask task : job.getCopyOfTasks()) {
                    if (task.getState() == JobTaskState.REBALANCE) {
                        count++;
                    }
                }
            }
        }
        return count;
    }

    /**
     * This enum tracks HostFailWorker's ideas of Host State. Options are:
     * - ALIVE: host is normal
     * - FAILING_FS_DEAD: User has requested that the host be failed immediately. There is a quiet period to allow the
     * queue logic to exit gracefully and to allow user to cancel if there was a mistake.
     * - FAILING_FS_OKAY: User has requested that the host be failed eventually, after safely migrating each task off.
     * - DISK_FULL: HostFailWorker detects hosts that are nearly full on disk, and moves tasks off automatically. Once
     * they return to safer levels, they will go back to ALIVE status.
     */
    public enum FailState {
        ALIVE, FAILING_FS_DEAD, FAILING_FS_OKAY, DISK_FULL
    }

}