Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hms.controller; import java.io.IOException; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hms.common.conf.CommonConfigurationKeys; import org.apache.hms.common.entity.Status; import org.apache.hms.common.entity.action.Action; import org.apache.hms.common.entity.action.ActionDependency; import org.apache.hms.common.entity.action.ActionStatus; import org.apache.hms.common.entity.action.PackageAction; import org.apache.hms.common.entity.cluster.MachineState; import org.apache.hms.common.entity.cluster.MachineState.StateEntry; import org.apache.hms.common.entity.command.ClusterCommand; import org.apache.hms.common.entity.command.Command; import org.apache.hms.common.entity.command.CommandStatus; import org.apache.hms.common.entity.command.CreateClusterCommand; import org.apache.hms.common.entity.command.DeleteClusterCommand; import org.apache.hms.common.entity.command.UpgradeClusterCommand; import org.apache.hms.common.entity.command.CommandStatus.ActionEntry; import org.apache.hms.common.entity.command.CommandStatus.HostStatusPair; import org.apache.hms.common.entity.command.DeleteCommand; import org.apache.hms.common.entity.manifest.ClusterHistory; import org.apache.hms.common.entity.manifest.ClusterManifest; import org.apache.hms.common.entity.manifest.ConfigManifest; import org.apache.hms.common.entity.manifest.NodesManifest; import org.apache.hms.common.entity.manifest.PackageInfo; import org.apache.hms.common.entity.manifest.Role; import org.apache.hms.common.entity.manifest.SoftwareManifest; import org.apache.hms.common.util.ExceptionUtil; import org.apache.hms.common.util.JAXBUtil; import org.apache.hms.common.util.ZookeeperUtil; import org.apache.zookeeper.AsyncCallback.Children2Callback; import org.apache.zookeeper.AsyncCallback.VoidCallback; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.ZooDefs.Ids; import org.apache.zookeeper.data.Stat; public class CommandHandler implements Children2Callback, VoidCallback, Watcher { private static Log LOG = LogFactory.getLog(CommandHandler.class); private static long SEVEN_DAYS = 1000L * 60 * 1440 * 7; private static String AGENT_ACTION = "/action"; private static String AGENT_STATUS = "/status"; private static String AGENT_WORKLOG = "/worklog"; public static String COMMAND_STATUS = "/status"; private static AtomicInteger actionCount = new AtomicInteger(); private final ZooKeeper zk; private final int handlerCount; private final LinkedBlockingQueue<String> tasks = new LinkedBlockingQueue<String>(); // access to watchedMachineNodes needs to be synchronized on itself private final Map<String, Set<String>> watchedMachineNodes = new HashMap<String, Set<String>>(); private Handler handlers[]; private volatile boolean running = true; // true while controller runs public CommandHandler(ZooKeeper zk, int handlerCount) throws KeeperException, InterruptedException { this.zk = zk; this.handlerCount = handlerCount; zk.getChildren(CommonConfigurationKeys.ZOOKEEPER_COMMAND_QUEUE_PATH_DEFAULT, this); zk.getChildren(CommonConfigurationKeys.ZOOKEEPER_LIVE_CONTROLLER_PATH_DEFAULT, this); zk.getChildren(CommonConfigurationKeys.ZOOKEEPER_STATUS_QUEUE_PATH_DEFAULT, this); } @Override public void processResult(int rc, String path, Object ctx) { } @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) { for (String child : children) { tasks.add(path + "/" + child); } } @Override public void process(WatchedEvent event) { String path = event.getPath(); LOG.info("Triggered path: " + path); if (event.getType() == Event.EventType.NodeChildrenChanged) { if (path.equals(CommonConfigurationKeys.ZOOKEEPER_LIVE_CONTROLLER_PATH_DEFAULT)) { zk.getChildren(CommonConfigurationKeys.ZOOKEEPER_COMMAND_QUEUE_PATH_DEFAULT, this, this, null); } else { zk.getChildren(path, this, this, null); } } else if (event.getType() == Event.EventType.NodeDataChanged) { tasks.add(path); } } private boolean isMachineNode(String path) { if (path.startsWith(CommonConfigurationKeys.ZOOKEEPER_CLUSTER_ROOT_DEFAULT) && path.split("/").length == 4) { return true; } return false; } private void checkCmds(String taskPath) throws IOException, KeeperException, InterruptedException { Set<String> cmdStatusPaths = null; synchronized (watchedMachineNodes) { cmdStatusPaths = watchedMachineNodes.remove(taskPath); } /* * current controller must already own the locks to these cmds. Otherwise, * these cmds wouldn't have been put into watchedMachineNodes. */ if (cmdStatusPaths != null) { for (String path : cmdStatusPaths) { queueActions(path); } } } private void handle() throws InterruptedException, KeeperException, IOException { boolean workOnIt = true; String taskPath = tasks.take(); if (isMachineNode(taskPath)) { // machine state change LOG.info("machine state changed: " + taskPath); checkCmds(taskPath); return; } try { // trying to acquire the lock zk.create(CommonConfigurationKeys.ZOOKEEPER_LOCK_QUEUE_PATH_DEFAULT + "/" + taskPath.replace('/', '.'), new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL); } catch (KeeperException.NodeExistsException e) { // client cmd or action status has been processed return; } // got the lock if (taskPath.startsWith(CommonConfigurationKeys.ZOOKEEPER_COMMAND_QUEUE_PATH_DEFAULT)) { String cmd = taskPath.substring(taskPath.lastIndexOf('/') + 1); if (cmd.indexOf('-') < 0) { throw new IOException("Unknown command: " + cmd); } byte[] data = zk.getData(taskPath, false, null); Command command = JAXBUtil.read(data, Command.class); String commandStatusPath = ZookeeperUtil.getCommandStatusPath(taskPath); Stat stat = zk.exists(commandStatusPath, false); if (stat != null) { byte[] test = zk.getData(commandStatusPath, false, stat); CommandStatus status = JAXBUtil.read(test, CommandStatus.class); if (status.getStatus() == Status.SUCCEEDED || status.getStatus() == Status.FAILED) { workOnIt = false; // Delete command, if it has been completed and older than // 7 days if (System.currentTimeMillis() > stat.getMtime() + (SEVEN_DAYS)) { LOG.info("Clean up deployment history: " + taskPath); deleteIfExists(taskPath); } } } if (workOnIt) { try { if (command instanceof DeleteCommand) { deleteCluster(taskPath, (DeleteCommand) command); } else if (command instanceof CreateClusterCommand) { createCluster(taskPath, (CreateClusterCommand) command); } else if (command instanceof DeleteClusterCommand) { deleteCluster(taskPath, (DeleteClusterCommand) command); } else if (command instanceof UpgradeClusterCommand) { updateCluster(taskPath, (ClusterCommand) command); } else { throw new IOException("Unknown command: " + command); } } catch (KeeperException e) { unlockCommand(taskPath); // Look for other command to work. zk.getChildren(CommonConfigurationKeys.ZOOKEEPER_COMMAND_QUEUE_PATH_DEFAULT, this); } } else { unlockCommand(taskPath); // Look for other command to work. zk.getChildren(CommonConfigurationKeys.ZOOKEEPER_COMMAND_QUEUE_PATH_DEFAULT, this); } } else if (taskPath.startsWith(CommonConfigurationKeys.ZOOKEEPER_STATUS_QUEUE_PATH_DEFAULT)) { updateSystemState(taskPath); } else if (taskPath.startsWith(CommonConfigurationKeys.ZOOKEEPER_CLUSTER_ROOT_DEFAULT)) { String queueNode = taskPath.substring(0, taskPath.lastIndexOf('/')); LOG.info("queueNode is " + queueNode); if (queueNode.endsWith(AGENT_STATUS)) { // agent status event updateSystemState(taskPath); } else if (queueNode.endsWith(AGENT_ACTION)) { // action being queued runFakeAgent(taskPath); } else { throw new IOException("Unknown event: " + taskPath); } } else { throw new IOException("Unexpected request: " + taskPath); } } /** * Simulate Agent Status */ private void runFakeAgent(String actionPath) throws KeeperException, InterruptedException, IOException { LOG.info("Fake agent received action event at " + actionPath); Thread.sleep(1000); Stat stat = zk.exists(actionPath, false); if (stat == null) { // action has been worked on return; } Action action = JAXBUtil.read(zk.getData(actionPath, false, null), Action.class); // create worklog node zk.create(actionPath + AGENT_WORKLOG, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); ActionStatus status = new ActionStatus(); status.setStatus(Status.SUCCEEDED); status.setError("Failure is unavoidable"); status.setCmdPath(action.getCmdPath()); status.setActionId(action.getActionId()); status.setActionPath(actionPath); String actionQueue = actionPath.substring(0, actionPath.lastIndexOf('/')); String hostNode = actionQueue.substring(0, actionQueue.lastIndexOf('/')); status.setHost(hostNode); String statusNode = zk.create(hostNode + AGENT_STATUS + "/" + "status-", JAXBUtil.write(status), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT_SEQUENTIAL); LOG.info("Fake agent queued status object at " + statusNode); } private void updateSystemState(String statusPath) throws InterruptedException, KeeperException, IOException { LOG.info("status path is: " + statusPath); Stat stat = zk.exists(statusPath, false); if (stat == null) { /* status has been previously processed by either this or another controller * delete the status lock if it exists */ LOG.info("status has been previously processed: " + statusPath); statusCleanup(statusPath, null); return; } ActionStatus actionStat = JAXBUtil.read(zk.getData(statusPath, false, null), ActionStatus.class); if (actionStat.getStatus() != Status.SUCCEEDED && actionStat.getStatus() != Status.FAILED) throw new IOException("Invalid action status: " + actionStat.getStatus() + " from action " + actionStat.getActionPath()); String actionPath = actionStat.getActionPath(); stat = zk.exists(actionPath, false); if (stat == null) { /* status has been previously processed by either this or another controller * delete the status znode, plus action and status locks */ statusCleanup(statusPath, actionPath); return; } String actionQueue = actionPath.substring(0, actionPath.lastIndexOf('/')); String hostNode = actionQueue.substring(0, actionQueue.lastIndexOf('/')); // update system status if (actionStat.getStatus() == Status.SUCCEEDED) { Action action = JAXBUtil.read(zk.getData(actionPath, false, null), Action.class); MachineState machineState = JAXBUtil.read(zk.getData(hostNode, false, stat), MachineState.class); boolean retry = true; while (retry) { retry = false; Set<StateEntry> states = machineState.getStates(); if (states == null) { states = new HashSet<StateEntry>(); } if (action.getExpectedResults() != null) { states.addAll(action.getExpectedResults()); } machineState.setStates(states); try { stat = zk.setData(hostNode, JAXBUtil.write(machineState), stat.getVersion()); } catch (KeeperException.BadVersionException e) { LOG.info("version mismatch: expected=" + stat.getVersion() + " msg: " + e.getMessage()); machineState = JAXBUtil.read(zk.getData(hostNode, false, stat), MachineState.class); LOG.info("new version is " + stat.getVersion()); retry = true; } } } // update cmd status if (actionStat.getStatus() != Status.SUCCEEDED) { try { zk.create( actionStat.getCmdPath() + COMMAND_STATUS + "/" + actionStat.getHost().replace('/', '.') + "-" + actionStat.getActionId(), JAXBUtil.write(actionStat), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } catch (KeeperException.NodeExistsException e) { } } String host = hostNode.substring(hostNode.lastIndexOf('/') + 1); String cmdStatusPath = actionStat.getCmdPath() + COMMAND_STATUS; CommandStatus cmdStatus = JAXBUtil.read(zk.getData(cmdStatusPath, false, stat), CommandStatus.class); boolean retry = true; while (retry) { retry = false; boolean found = false; boolean needUpdate = false; for (ActionEntry actionEntry : cmdStatus.getActionEntries()) { if (actionEntry.getAction().getActionId() == actionStat.getActionId()) { int failCount = 0; if (actionStat.getStatus() == Status.FAILED) { // Count current action status if it has failed. failCount++; } for (HostStatusPair hsp : actionEntry.getHostStatus()) { if (hsp.getStatus() == Status.FAILED) { // Walk through existing hosts, and count number of failed // actions. failCount++; } if (host.equals(hsp.getHost())) { found = true; Status status = hsp.getStatus(); if (status == Status.UNQUEUED || status == Status.QUEUED || status == Status.STARTED) { hsp.setStatus(actionStat.getStatus()); cmdStatus.setCompletedActions(cmdStatus.getCompletedActions() + 1); if (cmdStatus.getCompletedActions() == cmdStatus.getTotalActions()) { Status overallStatus = Status.SUCCEEDED; for (ActionEntry aEntry : cmdStatus.getActionEntries()) { boolean shouldBreak = false; for (HostStatusPair hspair : aEntry.getHostStatus()) { if (hspair.getStatus() != Status.SUCCEEDED) { overallStatus = Status.FAILED; shouldBreak = true; break; } } if (shouldBreak) break; } cmdStatus.setStatus(overallStatus); cmdStatus.setEndTime(new Date(System.currentTimeMillis()).toString()); updateClusterStatus(actionStat.getCmdPath()); } else if (failCount == actionEntry.getHostStatus().size()) { // If all nodes failed the action, set the command to fail. cmdStatus.setStatus(Status.FAILED); cmdStatus.setEndTime(new Date(System.currentTimeMillis()).toString()); updateClusterStatus(actionStat.getCmdPath()); } needUpdate = true; LOG.info("Fail count:" + failCount); break; } else if (status == actionStat.getStatus()) { // duplicate status update, nothing to be done } else { throw new IOException("UNEXPECTED action status: " + actionStat.getStatus() + " from action " + actionPath + ", current host status is " + status); } } } if (found) { break; } } } if (!found) { throw new IOException("UNEXPECTED: can't find action " + actionPath); } if (needUpdate) { try { stat = zk.setData(cmdStatusPath, JAXBUtil.write(cmdStatus), stat.getVersion()); if (cmdStatus.getStatus() == Status.SUCCEEDED || cmdStatus.getStatus() == Status.FAILED) { unlockCommand(actionStat.getCmdPath()); } } catch (KeeperException.BadVersionException e) { LOG.info("version mismatch: expected=" + stat.getVersion() + " msg: " + e.getMessage()); cmdStatus = JAXBUtil.read(zk.getData(cmdStatusPath, false, stat), CommandStatus.class); LOG.info("new version is " + stat.getVersion()); retry = true; } } } statusCleanup(statusPath, actionPath); LOG.info("Deleted action:" + actionPath + ", status:" + statusPath); } public void unlockCommand(String cmdPath) { String cmdLock = cmdPath.replace('/', '.'); try { deleteIfExists(CommonConfigurationKeys.ZOOKEEPER_LOCK_QUEUE_PATH_DEFAULT + "/" + cmdLock); } catch (InterruptedException e) { LOG.warn("Unable to unlock:" + cmdPath); } catch (KeeperException e) { LOG.warn("Unable to unlock:" + cmdPath); } } private void updateClusterStatus(String cmdPath) throws IOException, KeeperException, InterruptedException { Stat current = zk.exists(cmdPath, false); ClusterCommand cmd = JAXBUtil.read(zk.getData(cmdPath, false, current), ClusterCommand.class); String clusterPath = ZookeeperUtil.getClusterPath(cmd.getClusterManifest().getClusterName()); boolean retry = true; while (retry) { retry = false; try { if (cmd instanceof DeleteClusterCommand) { deleteIfExists(clusterPath); } unlockCluster(cmd.getClusterManifest().getClusterName()); } catch (KeeperException.BadVersionException e) { retry = true; LOG.warn(ExceptionUtil.getStackTrace(e)); LOG.warn("version mismatch: expected=" + current.getVersion()); zk.getData(clusterPath, false, current); LOG.warn("Cluster status update failed. Cluster ID:" + cmd.getClusterManifest().getClusterName() + " state: " + current.getVersion()); } } } private void statusCleanup(String statusPath, String actionPath) throws InterruptedException, KeeperException { deleteIfExists(actionPath); deleteIfExists(statusPath); // delete action lock for fake agent if (actionPath != null && actionPath.length() > 0) { deleteIfExists( CommonConfigurationKeys.ZOOKEEPER_LOCK_QUEUE_PATH_DEFAULT + "/" + actionPath.replace('/', '.')); } // delete status lock if (statusPath != null && statusPath.length() > 0) { deleteIfExists( CommonConfigurationKeys.ZOOKEEPER_LOCK_QUEUE_PATH_DEFAULT + "/" + statusPath.replace('/', '.')); } } private void deleteIfExists(String path) throws InterruptedException, KeeperException { if (path == null || path.length() == 0) { return; } Stat stat = zk.exists(path, false); if (stat == null) { return; } List<String> children = null; try { children = zk.getChildren(path, null); if (children != null) { for (String child : children) { deleteIfExists(path + "/" + child); } } zk.delete(path, -1); } catch (KeeperException.NoNodeException e) { LOG.info(ExceptionUtil.getStackTrace(e)); } } private void createNodeIfNecessary(String path, byte[] data) throws KeeperException, InterruptedException { try { zk.create(path, data, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } catch (KeeperException.NodeExistsException e) { } } private void createAndWatchClusterNodes(ClusterManifest cm, String cmdStatusPath) throws KeeperException, InterruptedException, IOException { CommandStatus cmdStatus = JAXBUtil.read(zk.getData(cmdStatusPath, false, null), CommandStatus.class); List<ActionEntry> list = cmdStatus.getActionEntries(); HashSet<String> hosts = new HashSet<String>(); for (ActionEntry a : list) { List<HostStatusPair> hsList = a.getHostStatus(); for (HostStatusPair hsp : hsList) { hosts.add(hsp.getHost()); } } ClusterHistory history; try { String path = ZookeeperUtil.getClusterPath(cm.getClusterName()); Stat stat = zk.exists(path, false); byte[] buffer = zk.getData(path, false, stat); history = JAXBUtil.read(buffer, ClusterHistory.class); } catch (KeeperException.NoNodeException e) { history = new ClusterHistory(); ArrayList<ClusterManifest> manifests = new ArrayList<ClusterManifest>(); manifests.add(cm); history.setHistory(manifests); } createAndWatchClusterNodes(cmdStatus.getClusterName(), hosts, history); } private void createAndWatchClusterNodes(String cluster, Set<String> hosts, ClusterHistory ch) throws KeeperException, InterruptedException, IOException { byte[] empty = new byte[0]; String clusterNode = ZookeeperUtil.getClusterPath(cluster); createNodeIfNecessary(clusterNode, JAXBUtil.write(ch)); // create host nodes and queues for (String host : hosts) { String hostNode = clusterNode + "/" + host; createNodeIfNecessary(hostNode, JAXBUtil.write(new MachineState())); String actionQueue = hostNode + AGENT_ACTION; createNodeIfNecessary(actionQueue, empty); String statusQueue = hostNode + AGENT_STATUS; createNodeIfNecessary(statusQueue, empty); // watch on agent status queue zk.getChildren(statusQueue, this); // to run fake agent, watch on action queue also //zk.getChildren(actionQueue, this); } } private void commitCommandPlan(String cmdStatusPath, CommandStatus cmdStatus, List<ActionEntry> actionEntries) throws KeeperException, InterruptedException, IOException { cmdStatus.setActionEntries(actionEntries); int totalActions = 0; for (ActionEntry a : actionEntries) { totalActions += a.getHostStatus().size(); } cmdStatus.setTotalActions(totalActions); // write out the plan that is captured in cmdStatus zk.create(cmdStatusPath, JAXBUtil.write(cmdStatus), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } private Set<String> convertRolesToHosts(NodesManifest nodesManifest, Set<String> roles) { Set<String> hosts = new HashSet<String>(); if (roles == null) { // If roles are unspecified, expand the unique host list for (Role collectRole : nodesManifest.getRoles()) { String[] hostList = collectRole.getHosts(); for (String host : hostList) { hosts.add(host); } } } else { for (String role : roles) { for (Role testRole : nodesManifest.getRoles()) { if (role.equals(testRole.getName())) { String[] hostList = testRole.getHosts(); for (String host : hostList) { hosts.add(host); } } } } } return hosts; } private PackageInfo[] convertRolesToPackages(SoftwareManifest softwareManifest, String role) { Set<PackageInfo> packages = new LinkedHashSet<PackageInfo>(); for (Role tmp : softwareManifest.getRoles()) { if (role == null || tmp.equals(role)) { for (PackageInfo p : tmp.getPackages()) { packages.add(p); } } } return packages.toArray(new PackageInfo[packages.size()]); } private List<HostStatusPair> setHostStatus(Set<String> hosts, Status status) { List<HostStatusPair> nodesList = new ArrayList<HostStatusPair>(); for (String node : hosts) { HostStatusPair hsp = new HostStatusPair(node, status); nodesList.add(hsp); } return nodesList; } /** * Create a lock for serializing cluster related commands. * @param clusterName * @throws KeeperException * @throws InterruptedException * @throws IOException */ private void lockCluster(String taskPath, String clusterName) throws KeeperException, InterruptedException, IOException { StringBuilder path = new StringBuilder(); path.append(CommonConfigurationKeys.ZOOKEEPER_LOCK_QUEUE_PATH_DEFAULT); path.append("/"); path.append("cluster."); path.append(clusterName); try { zk.create(path.toString(), new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL); } catch (KeeperException e) { tasks.add(taskPath); throw e; } } /** * Unlock cluster lock for operating next cluster related commands. * @param clusterName * @return Stat which cluster is unlocked * @throws KeeperException * @throws InterruptedException * @throws IOException */ public void unlockCluster(String clusterName) throws KeeperException, InterruptedException, IOException { try { StringBuilder path = new StringBuilder(); path.append(CommonConfigurationKeys.ZOOKEEPER_LOCK_QUEUE_PATH_DEFAULT); path.append("/"); path.append("cluster."); path.append(clusterName); Stat stat = zk.exists(path.toString(), false); zk.delete(path.toString(), stat.getVersion()); } catch (KeeperException.NoNodeException e) { } } /** * Check all cluster for duplicated nodes in use. * @param nm * @return true if node is already used by another cluster. * @throws InterruptedException * @throws KeeperException * @throws IOException */ private boolean checkNodesInUse(NodesManifest nm) throws KeeperException, InterruptedException { Set<String> hosts = convertRolesToHosts(nm, null); List<String> children = zk.getChildren(CommonConfigurationKeys.ZOOKEEPER_CLUSTER_ROOT_DEFAULT, null); Stat stat = new Stat(); boolean result = false; for (String cluster : children) { try { LOG.info("Check " + cluster); String path = ZookeeperUtil.getClusterPath(cluster); byte[] data = zk.getData(path, false, stat); ClusterHistory ch = JAXBUtil.read(data, ClusterHistory.class); int index = ch.getHistory().size() - 1; ClusterManifest cm = ch.getHistory().get(index); Set<String> test = convertRolesToHosts(cm.getNodes(), null); hosts.retainAll(test); if (!hosts.isEmpty()) { result = true; break; } } catch (Exception e) { LOG.error(ExceptionUtil.getStackTrace(e)); } } return result; } /** * Update Zookeeper for failed command status * @param path * @param cmd * @throws IOException * @throws KeeperException * @throws InterruptedException */ public void failCommand(String path, Command cmd) throws IOException, KeeperException, InterruptedException { String cmdStatusPath = ZookeeperUtil.getCommandStatusPath(path); CommandStatus cmdStatus = new CommandStatus(); String currentTime = new Date(System.currentTimeMillis()).toString(); cmdStatus.setStartTime(currentTime); cmdStatus.setEndTime(currentTime); cmdStatus.setStatus(Status.FAILED); cmdStatus.setTotalActions(0); boolean retry = true; while (retry) { try { Stat stat = zk.exists(cmdStatusPath, false); if (stat == null) { zk.create(cmdStatusPath, JAXBUtil.write(cmdStatus), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } else { zk.setData(cmdStatusPath, JAXBUtil.write(cmdStatus), stat.getVersion()); } retry = false; } catch (KeeperException.BadVersionException e) { retry = true; } } if (cmd instanceof ClusterCommand) { try { String clusterName = ((ClusterCommand) cmd).getClusterManifest().getClusterName(); unlockCluster(clusterName); } catch (NullPointerException e) { // Ignore if the cluster has not been locked. } } String cmdPath = CommonConfigurationKeys.ZOOKEEPER_COMMAND_QUEUE_PATH_DEFAULT + "/" + cmd.getId(); unlockCommand(cmdPath); } private void createCluster(String taskPath, ClusterCommand command) throws KeeperException, InterruptedException, IOException { lockCluster(taskPath, command.getClusterManifest().getClusterName()); if (checkNodesInUse(command.getClusterManifest().getNodes())) { failCommand(taskPath, command); LOG.error("Duplicated nodes detected in existing cluster."); unlockCluster(command.getClusterManifest().getClusterName()); return; } generateClusterPlan(taskPath, (ClusterCommand) command); String cmdStatusPath = ZookeeperUtil.getCommandStatusPath(taskPath); runClusterActions(command.getClusterManifest(), cmdStatusPath); } private void updateCluster(String taskPath, ClusterCommand command) throws KeeperException, InterruptedException, IOException { lockCluster(taskPath, command.getClusterManifest().getClusterName()); generateClusterPlan(taskPath, (ClusterCommand) command); String cmdStatusPath = ZookeeperUtil.getCommandStatusPath(taskPath); runClusterActions(command.getClusterManifest(), cmdStatusPath); } private void deleteCluster(String taskPath, DeleteClusterCommand command) throws KeeperException, InterruptedException, IOException { lockCluster(taskPath, command.getClusterManifest().getClusterName()); ClusterManifest cm = command.getClusterManifest(); String path = ZookeeperUtil.getClusterPath(cm.getClusterName()); byte[] data = zk.getData(path, null, null); ClusterHistory history = JAXBUtil.read(data, ClusterHistory.class); int index = history.getHistory().size() - 1; ClusterManifest currentCluster = history.getHistory().get(index); cm.setNodes(currentCluster.getNodes()); generateClusterPlan(taskPath, (ClusterCommand) command); String cmdStatusPath = ZookeeperUtil.getCommandStatusPath(taskPath); runClusterActions(command.getClusterManifest(), cmdStatusPath); } private void generateClusterPlan(String cmdPath, ClusterCommand cmd) throws KeeperException, InterruptedException, IOException { String cmdStatusPath = ZookeeperUtil.getCommandStatusPath(cmdPath); Stat stat = zk.exists(cmdStatusPath, false); if (stat != null) { // plan already exists, let's pick up what's left from another controller return; } // new create command LOG.info("Generate command plan: " + cmdPath); String startTime = new Date(System.currentTimeMillis()).toString(); CommandStatus cmdStatus = new CommandStatus(Status.STARTED, startTime); // Setup actions List<ActionEntry> actionEntries = new LinkedList<ActionEntry>(); ClusterManifest cm = ((ClusterCommand) cmd).getClusterManifest(); cmdStatus.setClusterName(cm.getClusterName()); NodesManifest nm = cm.getNodes(); ConfigManifest configM = cm.getConfig(); for (Action action : configM.getActions()) { // Find the host list for this action Set<String> hosts; if (action.getRole() == null) { hosts = convertRolesToHosts(nm, null); } else { Set<String> role = new HashSet<String>(); role.add(action.getRole()); hosts = convertRolesToHosts(nm, role); } List<HostStatusPair> nodesList = setHostStatus(hosts, Status.UNQUEUED); ActionEntry ae = new ActionEntry(); action.setCmdPath(cmdPath); action.setActionId(actionCount.incrementAndGet()); ae.setHostStatus(nodesList); List<ActionDependency> adList = action.getDependencies(); if (adList != null) { for (ActionDependency ad : adList) { Set<String> roles = ad.getRoles(); Set<String> dependentHosts = convertRolesToHosts(nm, roles); StringBuilder sb = new StringBuilder(); List<String> myhosts = new ArrayList<String>(); for (String host : dependentHosts) { sb.append(CommonConfigurationKeys.ZOOKEEPER_CLUSTER_ROOT_DEFAULT); sb.append("/"); sb.append(cm.getClusterName()); sb.append("/"); sb.append(host); myhosts.add(sb.toString()); sb.delete(0, sb.length()); } ad.setHosts(myhosts); } } // If the action is package action resolve the action from software manifest if (action instanceof PackageAction) { SoftwareManifest sm = cm.getSoftware(); if (action.getRole() == null) { // If no role is defined, install all the software in the software manifest PackageInfo[] packages = convertRolesToPackages(sm, null); ((PackageAction) action).setPackages(packages); } else { for (Role role : sm.getRoles()) { if (role.getName().equals(action.getRole())) { PackageInfo[] packages = convertRolesToPackages(sm, action.getRole()); ((PackageAction) action).setPackages(packages); } } } } ae.setAction(action); actionEntries.add(ae); } commitCommandPlan(cmdStatusPath, cmdStatus, actionEntries); } private void runClusterActions(ClusterManifest cm, String cmdStatusPath) throws KeeperException, InterruptedException, IOException { CommandStatus cmdStatus = JAXBUtil.read(zk.getData(cmdStatusPath, false, null), CommandStatus.class); String cluster = cmdStatus.getClusterName(); String clusterNode = ZookeeperUtil.getClusterPath(cluster); try { createAndWatchClusterNodes(cm, cmdStatusPath); } catch (KeeperException e) { LOG.debug(ExceptionUtil.getStackTrace(e)); } queueActions(cmdStatusPath); LOG.info("Issued actions for cluster [" + cluster + "] with " + zk.exists(clusterNode, null).getNumChildren() + " cluster nodes"); return; } private boolean isDependencySatisfied(String cmdStatusPath, List<ActionDependency> dependencies) throws KeeperException, InterruptedException, IOException { if (dependencies == null) { return true; } Stat stat = new Stat(); for (ActionDependency dep : dependencies) { List<String> hosts = dep.getHosts(); List<StateEntry> deps = dep.getStates(); if (hosts == null || hosts.size() == 0 || deps == null || deps.size() == 0) { continue; } int satisfied = 0; for (String host : hosts) { MachineState state = JAXBUtil.read(zk.getData(host, this, stat), MachineState.class); if (state == null || state.getStates() == null || !state.getStates().containsAll(deps)) { /* * Adding the cmd to watchedMachineNodes and return. We only add the * cmd once. Note that whenever the watch is triggered we remove the * mapping from watchedMachineNodes. This ensures that at any time * there is at most one mapping in watchedMachineNodes containing this * cmd as its value. Hence, at any time there will be at most one * handler thread executing queueActions() for this cmd (i.e., when * watch is triggered). */ synchronized (watchedMachineNodes) { Set<String> cmdStatusPaths = watchedMachineNodes.get(host); if (cmdStatusPaths == null) { cmdStatusPaths = new HashSet<String>(); } cmdStatusPaths.add(cmdStatusPath); watchedMachineNodes.put(host, cmdStatusPaths); } } else { satisfied++; } } float confidenceLevel = satisfied / hosts.size(); if (confidenceLevel < 0.5) { return false; } } return true; } private void queueActions(String cmdStatusPath) throws IOException, KeeperException, InterruptedException { LOG.info("try to queue actions for cmd " + cmdStatusPath); Stat stat = new Stat(); CommandStatus cmdStatus = JAXBUtil.read(zk.getData(cmdStatusPath, false, stat), CommandStatus.class); // we queue actions and update their status one at a time. After each // action is queued, we try to update its status (retry if necessary). // If retry happens, we start over again and try to find actions that // need to be issued. boolean startOver = true; while (startOver) { startOver = false; for (ActionEntry actionEntry : cmdStatus.getActionEntries()) { //TODO needs to check if an actionEntry is already done if (!isDependencySatisfied(cmdStatusPath, actionEntry.getAction().getDependencies())) { LOG.info("dependency is not satified for actionId=" + actionEntry.getAction().getActionId()); return; } int actionId = actionEntry.getAction().getActionId(); for (HostStatusPair hsp : actionEntry.getHostStatus()) { if (hsp.getStatus() == Status.UNQUEUED) { // queue action String actionNode = CommonConfigurationKeys.ZOOKEEPER_CLUSTER_ROOT_DEFAULT + "/" + cmdStatus.getClusterName() + "/" + hsp.getHost() + AGENT_ACTION + "/" + "action-"; actionNode = zk.create(actionNode, JAXBUtil.write(actionEntry.getAction()), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT_SEQUENTIAL); // update status for queued action String host = hsp.getHost(); hsp.setStatus(Status.QUEUED); boolean retry = true; while (retry) { retry = false; try { stat = zk.setData(cmdStatusPath, JAXBUtil.write(cmdStatus), stat.getVersion()); } catch (KeeperException.BadVersionException e) { LOG.info("version mismatch: expected=" + stat.getVersion() + " msg: " + e.getMessage()); // our copy is stale, we need to start over again after // updating the current status startOver = true; cmdStatus = JAXBUtil.read(zk.getData(cmdStatusPath, false, stat), CommandStatus.class); LOG.info("new version is " + stat.getVersion()); // find the item we want to update and check if it needs to be // updated boolean found = false; for (ActionEntry actEntry : cmdStatus.getActionEntries()) { if (actEntry.getAction().getActionId() == actionId) { for (HostStatusPair hostStat : actEntry.getHostStatus()) { if (hostStat.getHost().equals(host)) { // only update the status when we are in unqueued // state if (hostStat.getStatus() == Status.UNQUEUED) { hostStat.setStatus(Status.QUEUED); retry = true; } found = true; break; } } if (found) break; } } } } LOG.info("Queued action " + actionNode); if (startOver) break; } } if (startOver) break; } } } private void recursiveDelete(String path) throws KeeperException, InterruptedException { List<String> children = zk.getChildren(path, null); if (children.size() > 0) { for (String child : children) { recursiveDelete(path + "/" + child); } } zk.delete(path, -1); } private void deleteClusterInZookeeper(String cmdPath, DeleteClusterCommand cmd) throws KeeperException, InterruptedException, IOException { String clusterName = cmd.getClusterManifest().getClusterName(); LOG.info("Starting COMMAND: " + cmd + " on " + cmdPath); String cmdStatusPath = cmdPath + COMMAND_STATUS; try { String startTime = new Date(System.currentTimeMillis()).toString(); zk.create(cmdStatusPath, JAXBUtil.write(new CommandStatus(Status.STARTED, startTime)), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } catch (KeeperException.NodeExistsException e) { // cmd has been worked on } Stat stat = new Stat(); byte[] data = zk.getData(cmdStatusPath, false, stat); CommandStatus cmdStatus = JAXBUtil.read(data, CommandStatus.class); if (cmdStatus.getStatus() == Status.SUCCEEDED) { return; } String clusterPath = CommonConfigurationKeys.ZOOKEEPER_CLUSTER_ROOT_DEFAULT + "/" + clusterName; deleteIfExists(clusterPath); cmdStatus.setEndTime(new Date(System.currentTimeMillis()).toString()); cmdStatus.setStatus(Status.SUCCEEDED); zk.setData(cmdStatusPath, JAXBUtil.write(cmdStatus), stat.getVersion()); LOG.info("Deleted cluster " + clusterName); } private void deleteCluster(String cmdPath, DeleteCommand cmd) throws KeeperException, InterruptedException, IOException { DeleteClusterCommand delete = new DeleteClusterCommand(); ClusterManifest cm = new ClusterManifest(); cm.setClusterName(cmd.getClusterName()); delete.setClusterManifest(cm); deleteClusterInZookeeper(cmdPath, delete); } public Command getCommand(String cmdPath) throws KeeperException, InterruptedException, IOException { Stat stat = new Stat(); Command cmd = JAXBUtil.read(zk.getData(cmdPath, false, stat), Command.class); return cmd; } public synchronized void start() { handlers = new Handler[handlerCount]; for (int i = 0; i < handlerCount; i++) { handlers[i] = new Handler(i); handlers[i].start(); } } /** Stops the service. */ public synchronized void stop() { LOG.info("Stopping command handler"); running = false; if (handlers != null) { for (int i = 0; i < handlerCount; i++) { if (handlers[i] != null) { handlers[i].interrupt(); } } } notifyAll(); } /** Wait for the server to be stopped. * Does not wait for all subthreads to finish. * See {@link #stop()}. */ public synchronized void join() throws InterruptedException { while (running) { wait(); } } /** Handles queued commands . */ private class Handler extends Thread { public Handler(int instanceNumber) { this.setDaemon(true); this.setName("Command handler " + instanceNumber); } @Override public void run() { LOG.info(getName() + ": starting"); while (running) { try { handle(); } catch (InterruptedException e) { if (running) { // unexpected -- log it LOG.warn(getName() + " caught: " + ExceptionUtil.getStackTrace(e)); } } catch (Exception e) { LOG.warn(getName() + " caught: " + ExceptionUtil.getStackTrace(e)); } } LOG.info(getName() + ": exiting"); } } }