Java tutorial
// Licensed to the Apacohe Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. package com.cloud.vm; import java.net.URI; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.TimeZone; import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.naming.ConfigurationException; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo; import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.framework.ca.Certificate; import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.AsyncJob; import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.cloudstack.framework.jobs.Outcome; import org.apache.cloudstack.framework.jobs.dao.VmWorkJobDao; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.framework.jobs.impl.JobSerializerHelper; import org.apache.cloudstack.framework.jobs.impl.OutcomeImpl; import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.MessageDispatcher; import org.apache.cloudstack.framework.messagebus.MessageHandler; import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.Listener; import com.cloud.agent.api.AgentControlAnswer; import com.cloud.agent.api.AgentControlCommand; import com.cloud.agent.api.Answer; import com.cloud.agent.api.AttachOrDettachConfigDriveCommand; import com.cloud.agent.api.CheckVirtualMachineAnswer; import com.cloud.agent.api.CheckVirtualMachineCommand; import com.cloud.agent.api.ClusterVMMetaDataSyncAnswer; import com.cloud.agent.api.ClusterVMMetaDataSyncCommand; import com.cloud.agent.api.Command; import com.cloud.agent.api.MigrateCommand; import com.cloud.agent.api.ModifyTargetsCommand; import com.cloud.agent.api.PingRoutingCommand; import com.cloud.agent.api.PlugNicAnswer; import com.cloud.agent.api.PlugNicCommand; import com.cloud.agent.api.PrepareForMigrationCommand; import com.cloud.agent.api.RebootAnswer; import com.cloud.agent.api.RebootCommand; import com.cloud.agent.api.ReplugNicAnswer; import com.cloud.agent.api.ReplugNicCommand; import com.cloud.agent.api.RestoreVMSnapshotAnswer; import com.cloud.agent.api.RestoreVMSnapshotCommand; import com.cloud.agent.api.ScaleVmCommand; import com.cloud.agent.api.StartAnswer; import com.cloud.agent.api.StartCommand; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupRoutingCommand; import com.cloud.agent.api.StopAnswer; import com.cloud.agent.api.StopCommand; import com.cloud.agent.api.UnPlugNicAnswer; import com.cloud.agent.api.UnPlugNicCommand; import com.cloud.agent.api.UnregisterVMCommand; import com.cloud.agent.api.routing.NetworkElementCommand; import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.GPUDeviceTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.agent.manager.Commands; import com.cloud.agent.manager.allocator.HostAllocator; import com.cloud.alert.AlertManager; import com.cloud.capacity.CapacityManager; import com.cloud.configuration.Config; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; import com.cloud.dc.Pod; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.HostPodDao; import com.cloud.deploy.DataCenterDeployment; import com.cloud.deploy.DeployDestination; import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.deploy.DeploymentPlanningManager; import com.cloud.domain.dao.DomainDao; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; import com.cloud.exception.AffinityConflictException; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.ConnectionException; import com.cloud.exception.InsufficientAddressCapacityException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.exception.InsufficientVirtualNetworkCapacityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.exception.StorageUnavailableException; import com.cloud.gpu.dao.VGPUTypesDao; import com.cloud.ha.HighAvailabilityManager; import com.cloud.ha.HighAvailabilityManager.WorkType; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.HypervisorGuru; import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.network.Network; import com.cloud.network.NetworkModel; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.router.VirtualRouter; import com.cloud.network.rules.RulesManager; import com.cloud.offering.DiskOffering; import com.cloud.offering.DiskOfferingInfo; import com.cloud.offering.ServiceOffering; import com.cloud.org.Cluster; import com.cloud.resource.ResourceManager; import com.cloud.resource.ResourceState; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.ScopeType; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.Volume.Type; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.user.User; import com.cloud.utils.DateUtil; import com.cloud.utils.Journal; import com.cloud.utils.Pair; import com.cloud.utils.Predicate; import com.cloud.utils.ReflectionUse; import com.cloud.utils.StringUtils; import com.cloud.utils.Ternary; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.ExecutionException; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.utils.fsm.StateMachine2; import com.cloud.vm.ItWorkVO.Step; import com.cloud.vm.VirtualMachine.Event; import com.cloud.vm.VirtualMachine.PowerState; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.snapshot.VMSnapshotManager; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; import com.google.common.base.Strings; public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMachineManager, VmWorkJobHandler, Listener, Configurable { private static final Logger s_logger = Logger.getLogger(VirtualMachineManagerImpl.class); public static final String VM_WORK_JOB_HANDLER = VirtualMachineManagerImpl.class.getSimpleName(); private static final String VM_SYNC_ALERT_SUBJECT = "VM state sync alert"; @Inject DataStoreManager dataStoreMgr; @Inject protected NetworkOrchestrationService _networkMgr; @Inject protected NetworkModel _networkModel; @Inject protected AgentManager _agentMgr; @Inject protected VMInstanceDao _vmDao; @Inject protected ServiceOfferingDao _offeringDao; @Inject protected DiskOfferingDao _diskOfferingDao; @Inject protected VMTemplateDao _templateDao; @Inject protected DomainDao _domainDao; @Inject protected ItWorkDao _workDao; @Inject protected UserVmDao _userVmDao; @Inject protected UserVmService _userVmService; @Inject protected CapacityManager _capacityMgr; @Inject protected NicDao _nicsDao; @Inject protected HostDao _hostDao; @Inject protected AlertManager _alertMgr; @Inject protected GuestOSCategoryDao _guestOsCategoryDao; @Inject protected GuestOSDao _guestOsDao; @Inject protected VolumeDao _volsDao; @Inject protected HighAvailabilityManager _haMgr; @Inject protected HostPodDao _podDao; @Inject protected DataCenterDao _dcDao; @Inject protected ClusterDao _clusterDao; @Inject protected PrimaryDataStoreDao _storagePoolDao; @Inject protected HypervisorGuruManager _hvGuruMgr; @Inject protected NetworkDao _networkDao; @Inject protected StoragePoolHostDao _poolHostDao; @Inject protected VMSnapshotDao _vmSnapshotDao; @Inject protected RulesManager rulesMgr; @Inject protected AffinityGroupVMMapDao _affinityGroupVMMapDao; @Inject protected VGPUTypesDao _vgpuTypesDao; @Inject protected EntityManager _entityMgr; @Inject protected GuestOSCategoryDao _guestOSCategoryDao; @Inject protected GuestOSDao _guestOSDao = null; @Inject protected UserVmDetailsDao _vmDetailsDao; @Inject protected ServiceOfferingDao _serviceOfferingDao = null; @Inject protected CAManager caManager; @Inject ConfigDepot _configDepot; protected List<HostAllocator> hostAllocators; public List<HostAllocator> getHostAllocators() { return hostAllocators; } public void setHostAllocators(final List<HostAllocator> hostAllocators) { this.hostAllocators = hostAllocators; } protected List<StoragePoolAllocator> _storagePoolAllocators; @Inject protected ResourceManager _resourceMgr; @Inject protected VMSnapshotManager _vmSnapshotMgr = null; @Inject protected ClusterDetailsDao _clusterDetailsDao; @Inject protected UserVmDetailsDao _uservmDetailsDao; @Inject protected ConfigurationDao _configDao; @Inject VolumeOrchestrationService volumeMgr; @Inject DeploymentPlanningManager _dpMgr; @Inject protected MessageBus _messageBus; @Inject protected VirtualMachinePowerStateSync _syncMgr; @Inject protected VmWorkJobDao _workJobDao; @Inject protected AsyncJobManager _jobMgr; VmWorkJobHandlerProxy _jobHandlerProxy = new VmWorkJobHandlerProxy(this); Map<VirtualMachine.Type, VirtualMachineGuru> _vmGurus = new HashMap<VirtualMachine.Type, VirtualMachineGuru>(); protected StateMachine2<State, VirtualMachine.Event, VirtualMachine> _stateMachine; static final ConfigKey<Integer> StartRetry = new ConfigKey<Integer>("Advanced", Integer.class, "start.retry", "10", "Number of times to retry create and start commands", true); static final ConfigKey<Integer> VmOpWaitInterval = new ConfigKey<Integer>("Advanced", Integer.class, "vm.op.wait.interval", "120", "Time (in seconds) to wait before checking if a previous operation has succeeded", true); static final ConfigKey<Integer> VmOpLockStateRetry = new ConfigKey<Integer>("Advanced", Integer.class, "vm.op.lock.state.retry", "5", "Times to retry locking the state of a VM for operations, -1 means forever", true); static final ConfigKey<Long> VmOpCleanupInterval = new ConfigKey<Long>("Advanced", Long.class, "vm.op.cleanup.interval", "86400", "Interval to run the thread that cleans up the vm operations (in seconds)", false); static final ConfigKey<Long> VmOpCleanupWait = new ConfigKey<Long>("Advanced", Long.class, "vm.op.cleanup.wait", "3600", "Time (in seconds) to wait before cleanuping up any vm work items", true); static final ConfigKey<Long> VmOpCancelInterval = new ConfigKey<Long>("Advanced", Long.class, "vm.op.cancel.interval", "3600", "Time (in seconds) to wait before cancelling a operation", false); static final ConfigKey<Boolean> VmDestroyForcestop = new ConfigKey<Boolean>("Advanced", Boolean.class, "vm.destroy.forcestop", "false", "On destroy, force-stop takes this value ", true); static final ConfigKey<Integer> ClusterDeltaSyncInterval = new ConfigKey<Integer>("Advanced", Integer.class, "sync.interval", "60", "Cluster Delta sync interval in seconds", false); static final ConfigKey<Integer> ClusterVMMetaDataSyncInterval = new ConfigKey<Integer>("Advanced", Integer.class, "vmmetadata.sync.interval", "180", "Cluster VM metadata sync interval in seconds", false); static final ConfigKey<Long> VmJobCheckInterval = new ConfigKey<Long>("Advanced", Long.class, "vm.job.check.interval", "3000", "Interval in milliseconds to check if the job is complete", false); static final ConfigKey<Long> VmJobTimeout = new ConfigKey<Long>("Advanced", Long.class, "vm.job.timeout", "600000", "Time in milliseconds to wait before attempting to cancel a job", false); static final ConfigKey<Integer> VmJobStateReportInterval = new ConfigKey<Integer>("Advanced", Integer.class, "vm.job.report.interval", "60", "Interval to send application level pings to make sure the connection is still working", false); ScheduledExecutorService _executor = null; protected long _nodeId; @Override public void registerGuru(final VirtualMachine.Type type, final VirtualMachineGuru guru) { synchronized (_vmGurus) { _vmGurus.put(type, guru); } } @Override @DB public void allocate(final String vmInstanceName, final VirtualMachineTemplate template, final ServiceOffering serviceOffering, final DiskOfferingInfo rootDiskOfferingInfo, final List<DiskOfferingInfo> dataDiskOfferings, final LinkedHashMap<? extends Network, List<? extends NicProfile>> auxiliaryNetworks, final DeploymentPlan plan, final HypervisorType hyperType, final Map<String, Map<Integer, String>> extraDhcpOptions, final Map<Long, DiskOffering> datadiskTemplateToDiskOfferingMap) throws InsufficientCapacityException { final VMInstanceVO vm = _vmDao.findVMByInstanceName(vmInstanceName); final Account owner = _entityMgr.findById(Account.class, vm.getAccountId()); if (s_logger.isDebugEnabled()) { s_logger.debug("Allocating entries for VM: " + vm); } vm.setDataCenterId(plan.getDataCenterId()); if (plan.getPodId() != null) { vm.setPodIdToDeployIn(plan.getPodId()); } assert plan.getClusterId() == null && plan.getPoolId() == null : "We currently don't support cluster and pool preset yet"; final VMInstanceVO vmFinal = _vmDao.persist(vm); final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vmFinal, template, serviceOffering, null, null); Transaction.execute(new TransactionCallbackWithExceptionNoReturn<InsufficientCapacityException>() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) throws InsufficientCapacityException { if (s_logger.isDebugEnabled()) { s_logger.debug("Allocating nics for " + vmFinal); } try { if (!vmProfile.getBootArgs().contains("ExternalLoadBalancerVm")) { _networkMgr.allocate(vmProfile, auxiliaryNetworks, extraDhcpOptions); } } catch (final ConcurrentOperationException e) { throw new CloudRuntimeException( "Concurrent operation while trying to allocate resources for the VM", e); } if (s_logger.isDebugEnabled()) { s_logger.debug("Allocating disks for " + vmFinal); } if (template.getFormat() == ImageFormat.ISO) { volumeMgr.allocateRawVolume(Type.ROOT, "ROOT-" + vmFinal.getId(), rootDiskOfferingInfo.getDiskOffering(), rootDiskOfferingInfo.getSize(), rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), vmFinal, template, owner, null); } else if (template.getFormat() == ImageFormat.BAREMETAL) { // Do nothing } else { volumeMgr.allocateTemplatedVolume(Type.ROOT, "ROOT-" + vmFinal.getId(), rootDiskOfferingInfo.getDiskOffering(), rootDiskOfferingInfo.getSize(), rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), template, vmFinal, owner); } if (dataDiskOfferings != null) { for (final DiskOfferingInfo dataDiskOfferingInfo : dataDiskOfferings) { volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + vmFinal.getId(), dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(), dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), vmFinal, template, owner, null); } } if (datadiskTemplateToDiskOfferingMap != null && !datadiskTemplateToDiskOfferingMap.isEmpty()) { int diskNumber = 1; for (Entry<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap : datadiskTemplateToDiskOfferingMap .entrySet()) { DiskOffering diskOffering = dataDiskTemplateToDiskOfferingMap.getValue(); long diskOfferingSize = diskOffering.getDiskSize() / (1024 * 1024 * 1024); VMTemplateVO dataDiskTemplate = _templateDao .findById(dataDiskTemplateToDiskOfferingMap.getKey()); volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + vmFinal.getId() + "-" + String.valueOf(diskNumber), diskOffering, diskOfferingSize, null, null, vmFinal, dataDiskTemplate, owner, Long.valueOf(diskNumber)); diskNumber++; } } } }); if (s_logger.isDebugEnabled()) { s_logger.debug("Allocation completed for VM: " + vmFinal); } } @Override public void allocate(final String vmInstanceName, final VirtualMachineTemplate template, final ServiceOffering serviceOffering, final LinkedHashMap<? extends Network, List<? extends NicProfile>> networks, final DeploymentPlan plan, final HypervisorType hyperType) throws InsufficientCapacityException { allocate(vmInstanceName, template, serviceOffering, new DiskOfferingInfo(serviceOffering), new ArrayList<DiskOfferingInfo>(), networks, plan, hyperType, null, null); } private VirtualMachineGuru getVmGuru(final VirtualMachine vm) { if (vm != null) { return _vmGurus.get(vm.getType()); } return null; } @Override public void expunge(final String vmUuid) throws ResourceUnavailableException { try { advanceExpunge(vmUuid); } catch (final OperationTimedoutException e) { throw new CloudRuntimeException("Operation timed out", e); } catch (final ConcurrentOperationException e) { throw new CloudRuntimeException("Concurrent operation ", e); } } @Override public void advanceExpunge(final String vmUuid) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); advanceExpunge(vm); } protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException { if (vm == null || vm.getRemoved() != null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Unable to find vm or vm is destroyed: " + vm); } return; } advanceStop(vm.getUuid(), false); vm = _vmDao.findByUuid(vm.getUuid()); try { if (!stateTransitTo(vm, VirtualMachine.Event.ExpungeOperation, vm.getHostId())) { s_logger.debug("Unable to destroy the vm because it is not in the correct state: " + vm); throw new CloudRuntimeException("Unable to destroy " + vm); } } catch (final NoTransitionException e) { s_logger.debug("Unable to destroy the vm because it is not in the correct state: " + vm); throw new CloudRuntimeException("Unable to destroy " + vm, e); } if (s_logger.isDebugEnabled()) { s_logger.debug("Destroying vm " + vm); } final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType()); s_logger.debug("Cleaning up NICS"); final List<Command> nicExpungeCommands = hvGuru.finalizeExpungeNics(vm, profile.getNics()); _networkMgr.cleanupNics(profile); s_logger.debug("Cleaning up hypervisor data structures (ex. SRs in XenServer) for managed storage"); final List<Command> volumeExpungeCommands = hvGuru.finalizeExpungeVolumes(vm); final Long hostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId(); List<Map<String, String>> targets = getTargets(hostId, vm.getId()); if (volumeExpungeCommands != null && volumeExpungeCommands.size() > 0 && hostId != null) { final Commands cmds = new Commands(Command.OnError.Stop); for (final Command volumeExpungeCommand : volumeExpungeCommands) { cmds.addCommand(volumeExpungeCommand); } _agentMgr.send(hostId, cmds); if (!cmds.isSuccessful()) { for (final Answer answer : cmds.getAnswers()) { if (!answer.getResult()) { s_logger.warn("Failed to expunge vm due to: " + answer.getDetails()); throw new CloudRuntimeException( "Unable to expunge " + vm + " due to " + answer.getDetails()); } } } } if (hostId != null) { volumeMgr.revokeAccess(vm.getId(), hostId); } // Clean up volumes based on the vm's instance id volumeMgr.cleanupVolumes(vm.getId()); if (hostId != null && CollectionUtils.isNotEmpty(targets)) { removeDynamicTargets(hostId, targets); } final VirtualMachineGuru guru = getVmGuru(vm); guru.finalizeExpunge(vm); //remove the overcommit detials from the uservm details _uservmDetailsDao.removeDetails(vm.getId()); // send hypervisor-dependent commands before removing final List<Command> finalizeExpungeCommands = hvGuru.finalizeExpunge(vm); if (finalizeExpungeCommands != null && finalizeExpungeCommands.size() > 0) { if (hostId != null) { final Commands cmds = new Commands(Command.OnError.Stop); for (final Command command : finalizeExpungeCommands) { cmds.addCommand(command); } if (nicExpungeCommands != null) { for (final Command command : nicExpungeCommands) { cmds.addCommand(command); } } _agentMgr.send(hostId, cmds); if (!cmds.isSuccessful()) { for (final Answer answer : cmds.getAnswers()) { if (!answer.getResult()) { s_logger.warn("Failed to expunge vm due to: " + answer.getDetails()); throw new CloudRuntimeException( "Unable to expunge " + vm + " due to " + answer.getDetails()); } } } } } if (s_logger.isDebugEnabled()) { s_logger.debug("Expunged " + vm); } } private List<Map<String, String>> getTargets(Long hostId, long vmId) { List<Map<String, String>> targets = new ArrayList<>(); HostVO hostVO = _hostDao.findById(hostId); if (hostVO == null || hostVO.getHypervisorType() != HypervisorType.VMware) { return targets; } List<VolumeVO> volumes = _volsDao.findByInstance(vmId); if (CollectionUtils.isEmpty(volumes)) { return targets; } for (VolumeVO volume : volumes) { StoragePoolVO storagePoolVO = _storagePoolDao.findById(volume.getPoolId()); if (storagePoolVO != null && storagePoolVO.isManaged()) { Map<String, String> target = new HashMap<>(); target.put(ModifyTargetsCommand.STORAGE_HOST, storagePoolVO.getHostAddress()); target.put(ModifyTargetsCommand.STORAGE_PORT, String.valueOf(storagePoolVO.getPort())); target.put(ModifyTargetsCommand.IQN, volume.get_iScsiName()); targets.add(target); } } return targets; } private void removeDynamicTargets(long hostId, List<Map<String, String>> targets) { ModifyTargetsCommand cmd = new ModifyTargetsCommand(); cmd.setTargets(targets); cmd.setApplyToAllHostsInCluster(true); cmd.setAdd(false); cmd.setTargetTypeToRemove(ModifyTargetsCommand.TargetTypeToRemove.DYNAMIC); sendModifyTargetsCommand(cmd, hostId); } private void sendModifyTargetsCommand(ModifyTargetsCommand cmd, long hostId) { Answer answer = _agentMgr.easySend(hostId, cmd); if (answer == null) { String msg = "Unable to get an answer to the modify targets command"; s_logger.warn(msg); } else if (!answer.getResult()) { String msg = "Unable to modify target on the following host: " + hostId; s_logger.warn(msg); } } @Override public boolean start() { // TODO, initial delay is hardcoded _executor.scheduleAtFixedRate(new CleanupTask(), 5, VmJobStateReportInterval.value(), TimeUnit.SECONDS); _executor.scheduleAtFixedRate(new TransitionTask(), VmOpCleanupInterval.value(), VmOpCleanupInterval.value(), TimeUnit.SECONDS); cancelWorkItems(_nodeId); volumeMgr.cleanupStorageJobs(); // cleanup left over place holder works _workJobDao.expungeLeftoverWorkJobs(ManagementServerNode.getManagementServerId()); return true; } @Override public boolean stop() { return true; } @Override public boolean configure(final String name, final Map<String, Object> xmlParams) throws ConfigurationException { ReservationContextImpl.init(_entityMgr); VirtualMachineProfileImpl.init(_entityMgr); VmWorkMigrate.init(_entityMgr); _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Vm-Operations-Cleanup")); _nodeId = ManagementServerNode.getManagementServerId(); _agentMgr.registerForHostEvents(this, true, true, true); _messageBus.subscribe(VirtualMachineManager.Topics.VM_POWER_STATE, MessageDispatcher.getDispatcher(this)); return true; } protected VirtualMachineManagerImpl() { setStateMachine(); } @Override public void start(final String vmUuid, final Map<VirtualMachineProfile.Param, Object> params) { start(vmUuid, params, null, null); } @Override public void start(final String vmUuid, final Map<VirtualMachineProfile.Param, Object> params, final DeploymentPlan planToDeploy, final DeploymentPlanner planner) { try { advanceStart(vmUuid, params, planToDeploy, planner); } catch (final ConcurrentOperationException e) { throw new CloudRuntimeException("Unable to start a VM due to concurrent operation", e) .add(VirtualMachine.class, vmUuid); } catch (final InsufficientCapacityException e) { throw new CloudRuntimeException("Unable to start a VM due to insufficient capacity", e) .add(VirtualMachine.class, vmUuid); } catch (final ResourceUnavailableException e) { if (e.getScope() != null && e.getScope().equals(VirtualRouter.class)) { throw new CloudRuntimeException("Network is unavailable. Please contact administrator", e) .add(VirtualMachine.class, vmUuid); } throw new CloudRuntimeException("Unable to start a VM due to unavailable resources", e) .add(VirtualMachine.class, vmUuid); } } protected boolean checkWorkItems(final VMInstanceVO vm, final State state) throws ConcurrentOperationException { while (true) { final ItWorkVO vo = _workDao.findByOutstandingWork(vm.getId(), state); if (vo == null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Unable to find work for VM: " + vm + " and state: " + state); } return true; } if (vo.getStep() == Step.Done) { if (s_logger.isDebugEnabled()) { s_logger.debug("Work for " + vm + " is " + vo.getStep()); } return true; } // also check DB to get latest VM state to detect vm update from concurrent process before idle waiting to get an early exit final VMInstanceVO instance = _vmDao.findById(vm.getId()); if (instance != null && instance.getState() == State.Running) { if (s_logger.isDebugEnabled()) { s_logger.debug("VM is already started in DB: " + vm); } return true; } if (vo.getSecondsTaskIsInactive() > VmOpCancelInterval.value()) { s_logger.warn( "The task item for vm " + vm + " has been inactive for " + vo.getSecondsTaskIsInactive()); return false; } try { Thread.sleep(VmOpWaitInterval.value() * 1000); } catch (final InterruptedException e) { s_logger.info("Waiting for " + vm + " but is interrupted"); throw new ConcurrentOperationException("Waiting for " + vm + " but is interrupted"); } s_logger.debug("Waiting some more to make sure there's no activity on " + vm); } } @DB protected Ternary<VMInstanceVO, ReservationContext, ItWorkVO> changeToStartState( final VirtualMachineGuru vmGuru, final VMInstanceVO vm, final User caller, final Account account) throws ConcurrentOperationException { final long vmId = vm.getId(); ItWorkVO work = new ItWorkVO(UUID.randomUUID().toString(), _nodeId, State.Starting, vm.getType(), vm.getId()); int retry = VmOpLockStateRetry.value(); while (retry-- != 0) { try { final ItWorkVO workFinal = work; final Ternary<VMInstanceVO, ReservationContext, ItWorkVO> result = Transaction.execute( new TransactionCallbackWithException<Ternary<VMInstanceVO, ReservationContext, ItWorkVO>, NoTransitionException>() { @Override public Ternary<VMInstanceVO, ReservationContext, ItWorkVO> doInTransaction( final TransactionStatus status) throws NoTransitionException { final Journal journal = new Journal.LogJournal("Creating " + vm, s_logger); final ItWorkVO work = _workDao.persist(workFinal); final ReservationContextImpl context = new ReservationContextImpl(work.getId(), journal, caller, account); if (stateTransitTo(vm, Event.StartRequested, null, work.getId())) { if (s_logger.isDebugEnabled()) { s_logger.debug("Successfully transitioned to start state for " + vm + " reservation id = " + work.getId()); } return new Ternary<VMInstanceVO, ReservationContext, ItWorkVO>(vm, context, work); } return new Ternary<VMInstanceVO, ReservationContext, ItWorkVO>(null, null, work); } }); work = result.third(); if (result.first() != null) { return result; } } catch (final NoTransitionException e) { if (s_logger.isDebugEnabled()) { s_logger.debug("Unable to transition into Starting state due to " + e.getMessage()); } } final VMInstanceVO instance = _vmDao.findById(vmId); if (instance == null) { throw new ConcurrentOperationException("Unable to acquire lock on " + vm); } if (s_logger.isDebugEnabled()) { s_logger.debug("Determining why we're unable to update the state to Starting for " + instance + ". Retry=" + retry); } final State state = instance.getState(); if (state == State.Running) { if (s_logger.isDebugEnabled()) { s_logger.debug("VM is already started: " + vm); } return null; } if (state.isTransitional()) { if (!checkWorkItems(vm, state)) { throw new ConcurrentOperationException("There are concurrent operations on " + vm); } else { continue; } } if (state != State.Stopped) { s_logger.debug("VM " + vm + " is not in a state to be started: " + state); return null; } } throw new ConcurrentOperationException("Unable to change the state of " + vm); } protected <T extends VMInstanceVO> boolean changeState(final T vm, final Event event, final Long hostId, final ItWorkVO work, final Step step) throws NoTransitionException { // FIXME: We should do this better. Step previousStep = null; if (work != null) { previousStep = work.getStep(); _workDao.updateStep(work, step); } boolean result = false; try { result = stateTransitTo(vm, event, hostId); return result; } finally { if (!result && work != null) { _workDao.updateStep(work, previousStep); } } } protected boolean areAffinityGroupsAssociated(final VirtualMachineProfile vmProfile) { final VirtualMachine vm = vmProfile.getVirtualMachine(); final long vmGroupCount = _affinityGroupVMMapDao.countAffinityGroupsForVm(vm.getId()); if (vmGroupCount > 0) { return true; } return false; } @Override public void advanceStart(final String vmUuid, final Map<VirtualMachineProfile.Param, Object> params, final DeploymentPlanner planner) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { advanceStart(vmUuid, params, null, planner); } @Override public void advanceStart(final String vmUuid, final Map<VirtualMachineProfile.Param, Object> params, final DeploymentPlan planToDeploy, final DeploymentPlanner planner) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { // avoid re-entrance VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); placeHolder = createPlaceHolderWork(vm.getId()); try { orchestrateStart(vmUuid, params, planToDeploy, planner); } finally { if (placeHolder != null) { _workJobDao.expunge(placeHolder.getId()); } } } else { final Outcome<VirtualMachine> outcome = startVmThroughJobQueue(vmUuid, params, planToDeploy, planner); try { final VirtualMachine vm = outcome.get(); } catch (final InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (final java.util.concurrent.ExecutionException e) { throw new RuntimeException("Execution excetion", e); } final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); if (jobResult != null) { if (jobResult instanceof ConcurrentOperationException) { throw (ConcurrentOperationException) jobResult; } else if (jobResult instanceof ResourceUnavailableException) { throw (ResourceUnavailableException) jobResult; } else if (jobResult instanceof InsufficientCapacityException) { throw (InsufficientCapacityException) jobResult; } else if (jobResult instanceof RuntimeException) { throw (RuntimeException) jobResult; } else if (jobResult instanceof Throwable) { throw new RuntimeException("Unexpected exception", (Throwable) jobResult); } } } } private void setupAgentSecurity(final Host vmHost, final Map<String, String> sshAccessDetails, final VirtualMachine vm) throws AgentUnavailableException, OperationTimedoutException { final String csr = caManager.generateKeyStoreAndCsr(vmHost, sshAccessDetails); if (!Strings.isNullOrEmpty(csr)) { final Map<String, String> ipAddressDetails = new HashMap<>(sshAccessDetails); ipAddressDetails.remove(NetworkElementCommand.ROUTER_NAME); final Certificate certificate = caManager.issueCertificate(csr, Arrays.asList(vm.getHostName(), vm.getInstanceName()), new ArrayList<>(ipAddressDetails.values()), CAManager.CertValidityPeriod.value(), null); final boolean result = caManager.deployCertificate(vmHost, certificate, false, sshAccessDetails); if (!result) { s_logger.error("Failed to setup certificate for system vm: " + vm.getInstanceName()); } } else { s_logger.error("Failed to setup keystore and generate CSR for system vm: " + vm.getInstanceName()); } } @Override public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfile.Param, Object> params, final DeploymentPlan planToDeploy, final DeploymentPlanner planner) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { final CallContext cctxt = CallContext.current(); final Account account = cctxt.getCallingAccount(); final User caller = cctxt.getCallingUser(); VMInstanceVO vm = _vmDao.findByUuid(vmUuid); final VirtualMachineGuru vmGuru = getVmGuru(vm); final Ternary<VMInstanceVO, ReservationContext, ItWorkVO> start = changeToStartState(vmGuru, vm, caller, account); if (start == null) { return; } vm = start.first(); final ReservationContext ctx = start.second(); ItWorkVO work = start.third(); VMInstanceVO startedVm = null; final ServiceOfferingVO offering = _offeringDao.findById(vm.getId(), vm.getServiceOfferingId()); final VirtualMachineTemplate template = _entityMgr.findByIdIncludingRemoved(VirtualMachineTemplate.class, vm.getTemplateId()); DataCenterDeployment plan = new DataCenterDeployment(vm.getDataCenterId(), vm.getPodIdToDeployIn(), null, null, null, null, ctx); if (planToDeploy != null && planToDeploy.getDataCenterId() != 0) { if (s_logger.isDebugEnabled()) { s_logger.debug("advanceStart: DeploymentPlan is provided, using dcId:" + planToDeploy.getDataCenterId() + ", podId: " + planToDeploy.getPodId() + ", clusterId: " + planToDeploy.getClusterId() + ", hostId: " + planToDeploy.getHostId() + ", poolId: " + planToDeploy.getPoolId()); } plan = new DataCenterDeployment(planToDeploy.getDataCenterId(), planToDeploy.getPodId(), planToDeploy.getClusterId(), planToDeploy.getHostId(), planToDeploy.getPoolId(), planToDeploy.getPhysicalNetworkId(), ctx); } final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType()); boolean canRetry = true; ExcludeList avoids = null; try { final Journal journal = start.second().getJournal(); if (planToDeploy != null) { avoids = planToDeploy.getAvoids(); } if (avoids == null) { avoids = new ExcludeList(); } if (s_logger.isDebugEnabled()) { s_logger.debug("Deploy avoids pods: " + avoids.getPodsToAvoid() + ", clusters: " + avoids.getClustersToAvoid() + ", hosts: " + avoids.getHostsToAvoid()); } boolean planChangedByVolume = false; boolean reuseVolume = true; final DataCenterDeployment originalPlan = plan; int retry = StartRetry.value(); while (retry-- != 0) { // It's != so that it can match -1. if (reuseVolume) { // edit plan if this vm's ROOT volume is in READY state already final List<VolumeVO> vols = _volsDao.findReadyRootVolumesByInstance(vm.getId()); for (final VolumeVO vol : vols) { // make sure if the templateId is unchanged. If it is changed, // let planner // reassign pool for the volume even if it ready. final Long volTemplateId = vol.getTemplateId(); if (volTemplateId != null && volTemplateId.longValue() != template.getId()) { if (s_logger.isDebugEnabled()) { s_logger.debug(vol + " of " + vm + " is READY, but template ids don't match, let the planner reassign a new pool"); } continue; } final StoragePool pool = (StoragePool) dataStoreMgr.getPrimaryDataStore(vol.getPoolId()); if (!pool.isInMaintenance()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Root volume is ready, need to place VM in volume's cluster"); } final long rootVolDcId = pool.getDataCenterId(); final Long rootVolPodId = pool.getPodId(); final Long rootVolClusterId = pool.getClusterId(); if (planToDeploy != null && planToDeploy.getDataCenterId() != 0) { final Long clusterIdSpecified = planToDeploy.getClusterId(); if (clusterIdSpecified != null && rootVolClusterId != null) { if (rootVolClusterId.longValue() != clusterIdSpecified.longValue()) { // cannot satisfy the plan passed in to the // planner if (s_logger.isDebugEnabled()) { s_logger.debug( "Cannot satisfy the deployment plan passed in since the ready Root volume is in different cluster. volume's cluster: " + rootVolClusterId + ", cluster specified: " + clusterIdSpecified); } throw new ResourceUnavailableException( "Root volume is ready in different cluster, Deployment plan provided cannot be satisfied, unable to create a deployment for " + vm, Cluster.class, clusterIdSpecified); } } plan = new DataCenterDeployment(planToDeploy.getDataCenterId(), planToDeploy.getPodId(), planToDeploy.getClusterId(), planToDeploy.getHostId(), vol.getPoolId(), null, ctx); } else { plan = new DataCenterDeployment(rootVolDcId, rootVolPodId, rootVolClusterId, null, vol.getPoolId(), null, ctx); if (s_logger.isDebugEnabled()) { s_logger.debug( vol + " is READY, changing deployment plan to use this pool's dcId: " + rootVolDcId + " , podId: " + rootVolPodId + " , and clusterId: " + rootVolClusterId); } planChangedByVolume = true; } } } } final Account owner = _entityMgr.findById(Account.class, vm.getAccountId()); final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vm, template, offering, owner, params); DeployDestination dest = null; try { dest = _dpMgr.planDeployment(vmProfile, plan, avoids, planner); } catch (final AffinityConflictException e2) { s_logger.warn("Unable to create deployment, affinity rules associted to the VM conflict", e2); throw new CloudRuntimeException( "Unable to create deployment, affinity rules associted to the VM conflict"); } if (dest == null) { if (planChangedByVolume) { plan = originalPlan; planChangedByVolume = false; //do not enter volume reuse for next retry, since we want to look for resources outside the volume's cluster reuseVolume = false; continue; } throw new InsufficientServerCapacityException("Unable to create a deployment for " + vmProfile, DataCenter.class, plan.getDataCenterId(), areAffinityGroupsAssociated(vmProfile)); } if (dest != null) { avoids.addHost(dest.getHost().getId()); journal.record("Deployment found ", vmProfile, dest); } long destHostId = dest.getHost().getId(); vm.setPodIdToDeployIn(dest.getPod().getId()); final Long cluster_id = dest.getCluster().getId(); final ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, "cpuOvercommitRatio"); final ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, "memoryOvercommitRatio"); //storing the value of overcommit in the vm_details table for doing a capacity check in case the cluster overcommit ratio is changed. if (_uservmDetailsDao.findDetail(vm.getId(), "cpuOvercommitRatio") == null && (Float.parseFloat(cluster_detail_cpu.getValue()) > 1f || Float.parseFloat(cluster_detail_ram.getValue()) > 1f)) { _uservmDetailsDao.addDetail(vm.getId(), "cpuOvercommitRatio", cluster_detail_cpu.getValue(), true); _uservmDetailsDao.addDetail(vm.getId(), "memoryOvercommitRatio", cluster_detail_ram.getValue(), true); } else if (_uservmDetailsDao.findDetail(vm.getId(), "cpuOvercommitRatio") != null) { _uservmDetailsDao.addDetail(vm.getId(), "cpuOvercommitRatio", cluster_detail_cpu.getValue(), true); _uservmDetailsDao.addDetail(vm.getId(), "memoryOvercommitRatio", cluster_detail_ram.getValue(), true); } vmProfile.setCpuOvercommitRatio(Float.parseFloat(cluster_detail_cpu.getValue())); vmProfile.setMemoryOvercommitRatio(Float.parseFloat(cluster_detail_ram.getValue())); StartAnswer startAnswer = null; try { if (!changeState(vm, Event.OperationRetry, destHostId, work, Step.Prepare)) { throw new ConcurrentOperationException("Unable to update the state of the Virtual Machine " + vm.getUuid() + " oldstate: " + vm.getState() + "Event :" + Event.OperationRetry); } } catch (final NoTransitionException e1) { throw new ConcurrentOperationException(e1.getMessage()); } try { _networkMgr.prepare(vmProfile, new DeployDestination(dest.getDataCenter(), dest.getPod(), null, null), ctx); if (vm.getHypervisorType() != HypervisorType.BareMetal) { volumeMgr.prepare(vmProfile, dest); } //since StorageMgr succeeded in volume creation, reuse Volume for further tries until current cluster has capacity if (!reuseVolume) { reuseVolume = true; } Commands cmds = null; vmGuru.finalizeVirtualMachineProfile(vmProfile, dest, ctx); final VirtualMachineTO vmTO = hvGuru.implement(vmProfile); handlePath(vmTO.getDisks(), vm.getHypervisorType()); cmds = new Commands(Command.OnError.Stop); cmds.addCommand( new StartCommand(vmTO, dest.getHost(), getExecuteInSequence(vm.getHypervisorType()))); vmGuru.finalizeDeployment(cmds, vmProfile, dest, ctx); work = _workDao.findById(work.getId()); if (work == null || work.getStep() != Step.Prepare) { throw new ConcurrentOperationException("Work steps have been changed: " + work); } _workDao.updateStep(work, Step.Starting); _agentMgr.send(destHostId, cmds); _workDao.updateStep(work, Step.Started); startAnswer = cmds.getAnswer(StartAnswer.class); if (startAnswer != null && startAnswer.getResult()) { handlePath(vmTO.getDisks(), startAnswer.getIqnToData()); final String host_guid = startAnswer.getHost_guid(); if (host_guid != null) { final HostVO finalHost = _resourceMgr.findHostByGuid(host_guid); if (finalHost == null) { throw new CloudRuntimeException("Host Guid " + host_guid + " doesn't exist in DB, something went wrong while processing start answer: " + startAnswer); } destHostId = finalHost.getId(); } if (vmGuru.finalizeStart(vmProfile, destHostId, cmds, ctx)) { syncDiskChainChange(startAnswer); if (!changeState(vm, Event.OperationSucceeded, destHostId, work, Step.Done)) { s_logger.error("Unable to transition to a new state. VM uuid: " + vm.getUuid() + "VM oldstate:" + vm.getState() + "Event:" + Event.OperationSucceeded); throw new ConcurrentOperationException("Failed to deploy VM" + vm.getUuid()); } // Update GPU device capacity final GPUDeviceTO gpuDevice = startAnswer.getVirtualMachine().getGpuDevice(); if (gpuDevice != null) { _resourceMgr.updateGPUDetails(destHostId, gpuDevice.getGroupDetails()); } // Remove the information on whether it was a deploy vm request.The deployvm=true information // is set only when the vm is being deployed. When a vm is started from a stop state the // information isn't set, if (_uservmDetailsDao.findDetail(vm.getId(), "deployvm") != null) { _uservmDetailsDao.removeDetail(vm.getId(), "deployvm"); } startedVm = vm; if (s_logger.isDebugEnabled()) { s_logger.debug("Start completed for VM " + vm); } final Host vmHost = _hostDao.findById(destHostId); if (vmHost != null && (VirtualMachine.Type.ConsoleProxy.equals(vm.getType()) || VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) && caManager.canProvisionCertificates()) { final Map<String, String> sshAccessDetails = _networkMgr .getSystemVMAccessDetails(vm); for (int retries = 3; retries > 0; retries--) { try { setupAgentSecurity(vmHost, sshAccessDetails, vm); return; } catch (final Exception e) { s_logger.error( "Retrying after catching exception while trying to secure agent for systemvm id=" + vm.getId(), e); } } throw new CloudRuntimeException( "Failed to setup and secure agent for systemvm id=" + vm.getId()); } return; } else { if (s_logger.isDebugEnabled()) { s_logger.info("The guru did not like the answers so stopping " + vm); } StopCommand stopCmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false); stopCmd.setControlIp(getControlNicIpForVM(vm)); final StopCommand cmd = stopCmd; final Answer answer = _agentMgr.easySend(destHostId, cmd); if (answer != null && answer instanceof StopAnswer) { final StopAnswer stopAns = (StopAnswer) answer; if (vm.getType() == VirtualMachine.Type.User) { final String platform = stopAns.getPlatform(); if (platform != null) { final Map<String, String> vmmetadata = new HashMap<String, String>(); vmmetadata.put(vm.getInstanceName(), platform); syncVMMetaData(vmmetadata); } } } if (answer == null || !answer.getResult()) { s_logger.warn("Unable to stop " + vm + " due to " + (answer != null ? answer.getDetails() : "no answers")); _haMgr.scheduleStop(vm, destHostId, WorkType.ForceStop); throw new ExecutionException("Unable to stop this VM, " + vm.getUuid() + " so we are unable to retry the start operation"); } throw new ExecutionException("Unable to start VM:" + vm.getUuid() + " due to error in finalizeStart, not retrying"); } } s_logger.info("Unable to start VM on " + dest.getHost() + " due to " + (startAnswer == null ? " no start answer" : startAnswer.getDetails())); if (startAnswer != null && startAnswer.getContextParam("stopRetry") != null) { break; } } catch (OperationTimedoutException e) { s_logger.debug("Unable to send the start command to host " + dest.getHost() + " failed to start VM: " + vm.getUuid()); if (e.isActive()) { _haMgr.scheduleStop(vm, destHostId, WorkType.CheckStop); } canRetry = false; throw new AgentUnavailableException("Unable to start " + vm.getHostName(), destHostId, e); } catch (final ResourceUnavailableException e) { s_logger.info("Unable to contact resource.", e); if (!avoids.add(e)) { if (e.getScope() == Volume.class || e.getScope() == Nic.class) { throw e; } else { s_logger.warn("unexpected ResourceUnavailableException : " + e.getScope().getName(), e); throw e; } } } catch (final InsufficientCapacityException e) { s_logger.info("Insufficient capacity ", e); if (!avoids.add(e)) { if (e.getScope() == Volume.class || e.getScope() == Nic.class) { throw e; } else { s_logger.warn("unexpected InsufficientCapacityException : " + e.getScope().getName(), e); } } } catch (final ExecutionException e) { s_logger.error("Failed to start instance " + vm, e); throw new AgentUnavailableException("Unable to start instance due to " + e.getMessage(), destHostId, e); } catch (final NoTransitionException e) { s_logger.error("Failed to start instance " + vm, e); throw new AgentUnavailableException("Unable to start instance due to " + e.getMessage(), destHostId, e); } finally { if (startedVm == null && canRetry) { final Step prevStep = work.getStep(); _workDao.updateStep(work, Step.Release); // If previous step was started/ing && we got a valid answer if ((prevStep == Step.Started || prevStep == Step.Starting) && startAnswer != null && startAnswer.getResult()) { //TODO check the response of cleanup and record it in DB for retry cleanup(vmGuru, vmProfile, work, Event.OperationFailed, false); } else { //if step is not starting/started, send cleanup command with force=true cleanup(vmGuru, vmProfile, work, Event.OperationFailed, true); } } } } } finally { if (startedVm == null) { if (canRetry) { try { changeState(vm, Event.OperationFailed, null, work, Step.Done); } catch (final NoTransitionException e) { throw new ConcurrentOperationException(e.getMessage()); } } } if (planToDeploy != null) { planToDeploy.setAvoids(avoids); } } if (startedVm == null) { throw new CloudRuntimeException("Unable to start instance '" + vm.getHostName() + "' (" + vm.getUuid() + "), see management server log for details"); } } // for managed storage on KVM, need to make sure the path field of the volume in question is populated with the IQN private void handlePath(final DiskTO[] disks, final HypervisorType hypervisorType) { if (hypervisorType != HypervisorType.KVM) { return; } if (disks != null) { for (final DiskTO disk : disks) { final Map<String, String> details = disk.getDetails(); final boolean isManaged = details != null && Boolean.parseBoolean(details.get(DiskTO.MANAGED)); if (isManaged && disk.getPath() == null) { final Long volumeId = disk.getData().getId(); final VolumeVO volume = _volsDao.findById(volumeId); disk.setPath(volume.get_iScsiName()); if (disk.getData() instanceof VolumeObjectTO) { final VolumeObjectTO volTo = (VolumeObjectTO) disk.getData(); volTo.setPath(volume.get_iScsiName()); } volume.setPath(volume.get_iScsiName()); _volsDao.update(volumeId, volume); } } } } // for managed storage on XenServer and VMware, need to update the DB with a path if the VDI/VMDK file was newly created private void handlePath(final DiskTO[] disks, final Map<String, Map<String, String>> iqnToData) { if (disks != null && iqnToData != null) { for (final DiskTO disk : disks) { final Map<String, String> details = disk.getDetails(); final boolean isManaged = details != null && Boolean.parseBoolean(details.get(DiskTO.MANAGED)); if (isManaged) { final Long volumeId = disk.getData().getId(); final VolumeVO volume = _volsDao.findById(volumeId); final String iScsiName = volume.get_iScsiName(); boolean update = false; final Map<String, String> data = iqnToData.get(iScsiName); if (data != null) { final String path = data.get(StartAnswer.PATH); if (path != null) { volume.setPath(path); update = true; } final String imageFormat = data.get(StartAnswer.IMAGE_FORMAT); if (imageFormat != null) { volume.setFormat(ImageFormat.valueOf(imageFormat)); update = true; } if (update) { _volsDao.update(volumeId, volume); } } } } } } private void syncDiskChainChange(final StartAnswer answer) { final VirtualMachineTO vmSpec = answer.getVirtualMachine(); for (final DiskTO disk : vmSpec.getDisks()) { if (disk.getType() != Volume.Type.ISO) { final VolumeObjectTO vol = (VolumeObjectTO) disk.getData(); final VolumeVO volume = _volsDao.findById(vol.getId()); // Use getPath() from VolumeVO to get a fresh copy of what's in the DB. // Before doing this, in a certain situation, getPath() from VolumeObjectTO // returned null instead of an actual path (because it was out of date with the DB). if (vol.getPath() != null) { volumeMgr.updateVolumeDiskChain(vol.getId(), vol.getPath(), vol.getChainInfo()); } else { volumeMgr.updateVolumeDiskChain(vol.getId(), volume.getPath(), vol.getChainInfo()); } } } } @Override public void stop(final String vmUuid) throws ResourceUnavailableException { try { advanceStop(vmUuid, false); } catch (final OperationTimedoutException e) { throw new AgentUnavailableException("Unable to stop vm because the operation to stop timed out", e.getAgentId(), e); } catch (final ConcurrentOperationException e) { throw new CloudRuntimeException("Unable to stop vm because of a concurrent operation", e); } } @Override public void stopForced(String vmUuid) throws ResourceUnavailableException { try { advanceStop(vmUuid, true); } catch (final OperationTimedoutException e) { throw new AgentUnavailableException("Unable to stop vm because the operation to stop timed out", e.getAgentId(), e); } catch (final ConcurrentOperationException e) { throw new CloudRuntimeException("Unable to stop vm because of a concurrent operation", e); } } @Override public boolean getExecuteInSequence(final HypervisorType hypervisorType) { if (HypervisorType.KVM == hypervisorType || HypervisorType.XenServer == hypervisorType || HypervisorType.Hyperv == hypervisorType || HypervisorType.LXC == hypervisorType) { return false; } else if (HypervisorType.VMware == hypervisorType) { final Boolean fullClone = HypervisorGuru.VmwareFullClone.value(); return fullClone; } else { return ExecuteInSequence.value(); } } private List<Map<String, String>> getVolumesToDisconnect(VirtualMachine vm) { List<Map<String, String>> volumesToDisconnect = new ArrayList<>(); List<VolumeVO> volumes = _volsDao.findByInstance(vm.getId()); if (CollectionUtils.isEmpty(volumes)) { return volumesToDisconnect; } for (VolumeVO volume : volumes) { StoragePoolVO storagePool = _storagePoolDao.findById(volume.getPoolId()); if (storagePool != null && storagePool.isManaged()) { Map<String, String> info = new HashMap<>(3); info.put(DiskTO.STORAGE_HOST, storagePool.getHostAddress()); info.put(DiskTO.STORAGE_PORT, String.valueOf(storagePool.getPort())); info.put(DiskTO.IQN, volume.get_iScsiName()); volumesToDisconnect.add(info); } } return volumesToDisconnect; } protected boolean sendStop(final VirtualMachineGuru guru, final VirtualMachineProfile profile, final boolean force, final boolean checkBeforeCleanup) { final VirtualMachine vm = profile.getVirtualMachine(); StopCommand stpCmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), checkBeforeCleanup); stpCmd.setControlIp(getControlNicIpForVM(vm)); stpCmd.setVolumesToDisconnect(getVolumesToDisconnect(vm)); final StopCommand stop = stpCmd; try { Answer answer = null; if (vm.getHostId() != null) { answer = _agentMgr.send(vm.getHostId(), stop); } if (answer != null && answer instanceof StopAnswer) { final StopAnswer stopAns = (StopAnswer) answer; if (vm.getType() == VirtualMachine.Type.User) { final String platform = stopAns.getPlatform(); if (platform != null) { final UserVmVO userVm = _userVmDao.findById(vm.getId()); _userVmDao.loadDetails(userVm); userVm.setDetail("platform", platform); _userVmDao.saveDetails(userVm); } } final GPUDeviceTO gpuDevice = stop.getGpuDevice(); if (gpuDevice != null) { _resourceMgr.updateGPUDetails(vm.getHostId(), gpuDevice.getGroupDetails()); } if (!answer.getResult()) { final String details = answer.getDetails(); s_logger.debug("Unable to stop VM due to " + details); return false; } guru.finalizeStop(profile, answer); } else { s_logger.error("Invalid answer received in response to a StopCommand for " + vm.getInstanceName()); return false; } } catch (final AgentUnavailableException e) { if (!force) { return false; } } catch (final OperationTimedoutException e) { if (!force) { return false; } } return true; } protected boolean cleanup(final VirtualMachineGuru guru, final VirtualMachineProfile profile, final ItWorkVO work, final Event event, final boolean cleanUpEvenIfUnableToStop) { final VirtualMachine vm = profile.getVirtualMachine(); final State state = vm.getState(); s_logger.debug("Cleaning up resources for the vm " + vm + " in " + state + " state"); try { if (state == State.Starting) { if (work != null) { final Step step = work.getStep(); if (step == Step.Starting && !cleanUpEvenIfUnableToStop) { s_logger.warn("Unable to cleanup vm " + vm + "; work state is incorrect: " + step); return false; } if (step == Step.Started || step == Step.Starting || step == Step.Release) { if (vm.getHostId() != null) { if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop, false)) { s_logger.warn("Failed to stop vm " + vm + " in " + State.Starting + " state as a part of cleanup process"); return false; } } } if (step != Step.Release && step != Step.Prepare && step != Step.Started && step != Step.Starting) { s_logger.debug("Cleanup is not needed for vm " + vm + "; work state is incorrect: " + step); return true; } } else { if (vm.getHostId() != null) { if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop, false)) { s_logger.warn("Failed to stop vm " + vm + " in " + State.Starting + " state as a part of cleanup process"); return false; } } } } else if (state == State.Stopping) { if (vm.getHostId() != null) { if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop, false)) { s_logger.warn("Failed to stop vm " + vm + " in " + State.Stopping + " state as a part of cleanup process"); return false; } } } else if (state == State.Migrating) { if (vm.getHostId() != null) { if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop, false)) { s_logger.warn("Failed to stop vm " + vm + " in " + State.Migrating + " state as a part of cleanup process"); return false; } } if (vm.getLastHostId() != null) { if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop, false)) { s_logger.warn("Failed to stop vm " + vm + " in " + State.Migrating + " state as a part of cleanup process"); return false; } } } else if (state == State.Running) { if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop, false)) { s_logger.warn("Failed to stop vm " + vm + " in " + State.Running + " state as a part of cleanup process"); return false; } } } finally { try { _networkMgr.release(profile, cleanUpEvenIfUnableToStop); s_logger.debug("Successfully released network resources for the vm " + vm); } catch (final Exception e) { s_logger.warn("Unable to release some network resources.", e); } volumeMgr.release(profile); s_logger.debug("Successfully cleanued up resources for the vm " + vm + " in " + state + " state"); } return true; } @Override public void advanceStop(final String vmUuid, final boolean cleanUpEvenIfUnableToStop) throws AgentUnavailableException, OperationTimedoutException, ConcurrentOperationException { final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { // avoid re-entrance VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); placeHolder = createPlaceHolderWork(vm.getId()); try { orchestrateStop(vmUuid, cleanUpEvenIfUnableToStop); } finally { if (placeHolder != null) { _workJobDao.expunge(placeHolder.getId()); } } } else { final Outcome<VirtualMachine> outcome = stopVmThroughJobQueue(vmUuid, cleanUpEvenIfUnableToStop); try { final VirtualMachine vm = outcome.get(); } catch (final InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (final java.util.concurrent.ExecutionException e) { throw new RuntimeException("Execution excetion", e); } final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); if (jobResult != null) { if (jobResult instanceof AgentUnavailableException) { throw (AgentUnavailableException) jobResult; } else if (jobResult instanceof ConcurrentOperationException) { throw (ConcurrentOperationException) jobResult; } else if (jobResult instanceof OperationTimedoutException) { throw (OperationTimedoutException) jobResult; } else if (jobResult instanceof RuntimeException) { throw (RuntimeException) jobResult; } else if (jobResult instanceof Throwable) { throw new RuntimeException("Unexpected exception", (Throwable) jobResult); } } } } private void orchestrateStop(final String vmUuid, final boolean cleanUpEvenIfUnableToStop) throws AgentUnavailableException, OperationTimedoutException, ConcurrentOperationException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); advanceStop(vm, cleanUpEvenIfUnableToStop); } private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnableToStop) throws AgentUnavailableException, OperationTimedoutException, ConcurrentOperationException { final State state = vm.getState(); if (state == State.Stopped) { if (s_logger.isDebugEnabled()) { s_logger.debug("VM is already stopped: " + vm); } return; } if (state == State.Destroyed || state == State.Expunging || state == State.Error) { if (s_logger.isDebugEnabled()) { s_logger.debug("Stopped called on " + vm + " but the state is " + state); } return; } // grab outstanding work item if any final ItWorkVO work = _workDao.findByOutstandingWork(vm.getId(), vm.getState()); if (work != null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Found an outstanding work item for this vm " + vm + " with state:" + vm.getState() + ", work id:" + work.getId()); } } final Long hostId = vm.getHostId(); if (hostId == null) { if (!cleanUpEvenIfUnableToStop) { if (s_logger.isDebugEnabled()) { s_logger.debug("HostId is null but this is not a forced stop, cannot stop vm " + vm + " with state:" + vm.getState()); } throw new CloudRuntimeException("Unable to stop " + vm); } try { stateTransitTo(vm, Event.AgentReportStopped, null, null); } catch (final NoTransitionException e) { s_logger.warn(e.getMessage()); } // mark outstanding work item if any as done if (work != null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Updating work item to Done, id:" + work.getId()); } work.setStep(Step.Done); _workDao.update(work.getId(), work); } return; } else { HostVO host = _hostDao.findById(hostId); if (!cleanUpEvenIfUnableToStop && vm.getState() == State.Running && host.getResourceState() == ResourceState.PrepareForMaintenance) { s_logger.debug("Host is in PrepareForMaintenance state - Stop VM operation on the VM id: " + vm.getId() + " is not allowed"); throw new CloudRuntimeException("Stop VM operation on the VM id: " + vm.getId() + " is not allowed as host is preparing for maintenance mode"); } } final VirtualMachineGuru vmGuru = getVmGuru(vm); final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); try { if (!stateTransitTo(vm, Event.StopRequested, vm.getHostId())) { throw new ConcurrentOperationException("VM is being operated on."); } } catch (final NoTransitionException e1) { if (!cleanUpEvenIfUnableToStop) { throw new CloudRuntimeException("We cannot stop " + vm + " when it is in state " + vm.getState()); } final boolean doCleanup = true; if (s_logger.isDebugEnabled()) { s_logger.debug("Unable to transition the state but we're moving on because it's forced stop"); } if (doCleanup) { if (cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.StopRequested, cleanUpEvenIfUnableToStop)) { try { if (s_logger.isDebugEnabled() && work != null) { s_logger.debug("Updating work item to Done, id:" + work.getId()); } if (!changeState(vm, Event.AgentReportStopped, null, work, Step.Done)) { throw new CloudRuntimeException("Unable to stop " + vm); } } catch (final NoTransitionException e) { s_logger.warn("Unable to cleanup " + vm); throw new CloudRuntimeException("Unable to stop " + vm, e); } } else { if (s_logger.isDebugEnabled()) { s_logger.debug("Failed to cleanup VM: " + vm); } throw new CloudRuntimeException( "Failed to cleanup " + vm + " , current state " + vm.getState()); } } } if (vm.getState() != State.Stopping) { throw new CloudRuntimeException("We cannot proceed with stop VM " + vm + " since it is not in 'Stopping' state, current state: " + vm.getState()); } vmGuru.prepareStop(profile); final StopCommand stop = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false, cleanUpEvenIfUnableToStop); stop.setControlIp(getControlNicIpForVM(vm)); boolean stopped = false; Answer answer = null; try { answer = _agentMgr.send(vm.getHostId(), stop); if (answer != null) { if (answer instanceof StopAnswer) { final StopAnswer stopAns = (StopAnswer) answer; if (vm.getType() == VirtualMachine.Type.User) { final String platform = stopAns.getPlatform(); if (platform != null) { final UserVmVO userVm = _userVmDao.findById(vm.getId()); _userVmDao.loadDetails(userVm); userVm.setDetail("platform", platform); _userVmDao.saveDetails(userVm); } } } stopped = answer.getResult(); if (!stopped) { throw new CloudRuntimeException( "Unable to stop the virtual machine due to " + answer.getDetails()); } vmGuru.finalizeStop(profile, answer); final GPUDeviceTO gpuDevice = stop.getGpuDevice(); if (gpuDevice != null) { _resourceMgr.updateGPUDetails(vm.getHostId(), gpuDevice.getGroupDetails()); } } else { throw new CloudRuntimeException( "Invalid answer received in response to a StopCommand on " + vm.instanceName); } } catch (final AgentUnavailableException e) { s_logger.warn("Unable to stop vm, agent unavailable: " + e.toString()); } catch (final OperationTimedoutException e) { s_logger.warn("Unable to stop vm, operation timed out: " + e.toString()); } finally { if (!stopped) { if (!cleanUpEvenIfUnableToStop) { s_logger.warn("Unable to stop vm " + vm); try { stateTransitTo(vm, Event.OperationFailed, vm.getHostId()); } catch (final NoTransitionException e) { s_logger.warn("Unable to transition the state " + vm); } throw new CloudRuntimeException("Unable to stop " + vm); } else { s_logger.warn("Unable to actually stop " + vm + " but continue with release because it's a force stop"); vmGuru.finalizeStop(profile, answer); } } } if (s_logger.isDebugEnabled()) { s_logger.debug(vm + " is stopped on the host. Proceeding to release resource held."); } try { _networkMgr.release(profile, cleanUpEvenIfUnableToStop); s_logger.debug("Successfully released network resources for the vm " + vm); } catch (final Exception e) { s_logger.warn("Unable to release some network resources.", e); } try { if (vm.getHypervisorType() != HypervisorType.BareMetal) { volumeMgr.release(profile); s_logger.debug("Successfully released storage resources for the vm " + vm); } } catch (final Exception e) { s_logger.warn("Unable to release storage resources.", e); } try { if (work != null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Updating the outstanding work item to Done, id:" + work.getId()); } work.setStep(Step.Done); _workDao.update(work.getId(), work); } if (!stateTransitTo(vm, Event.OperationSucceeded, null)) { throw new CloudRuntimeException("unable to stop " + vm); } } catch (final NoTransitionException e) { s_logger.warn(e.getMessage()); throw new CloudRuntimeException("Unable to stop " + vm); } } private void setStateMachine() { _stateMachine = VirtualMachine.State.getStateMachine(); } protected boolean stateTransitTo(final VMInstanceVO vm, final VirtualMachine.Event e, final Long hostId, final String reservationId) throws NoTransitionException { // if there are active vm snapshots task, state change is not allowed // Disable this hacking thing, VM snapshot task need to be managed by its orchestartion flow istelf instead of // hacking it here at general VM manager /* if (_vmSnapshotMgr.hasActiveVMSnapshotTasks(vm.getId())) { s_logger.error("State transit with event: " + e + " failed due to: " + vm.getInstanceName() + " has active VM snapshots tasks"); return false; } */ vm.setReservationId(reservationId); return _stateMachine.transitTo(vm, e, new Pair<Long, Long>(vm.getHostId(), hostId), _vmDao); } @Override public boolean stateTransitTo(final VirtualMachine vm1, final VirtualMachine.Event e, final Long hostId) throws NoTransitionException { final VMInstanceVO vm = (VMInstanceVO) vm1; /* * Remove the hacking logic here. // if there are active vm snapshots task, state change is not allowed if (_vmSnapshotMgr.hasActiveVMSnapshotTasks(vm.getId())) { s_logger.error("State transit with event: " + e + " failed due to: " + vm.getInstanceName() + " has active VM snapshots tasks"); return false; } */ final State oldState = vm.getState(); if (oldState == State.Starting) { if (e == Event.OperationSucceeded) { vm.setLastHostId(hostId); } } else if (oldState == State.Stopping) { if (e == Event.OperationSucceeded) { vm.setLastHostId(vm.getHostId()); } } return _stateMachine.transitTo(vm, e, new Pair<Long, Long>(vm.getHostId(), hostId), _vmDao); } @Override public void destroy(final String vmUuid, final boolean expunge) throws AgentUnavailableException, OperationTimedoutException, ConcurrentOperationException { VMInstanceVO vm = _vmDao.findByUuid(vmUuid); if (vm == null || vm.getState() == State.Destroyed || vm.getState() == State.Expunging || vm.getRemoved() != null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Unable to find vm or vm is destroyed: " + vm); } return; } if (s_logger.isDebugEnabled()) { s_logger.debug("Destroying vm " + vm + ", expunge flag " + (expunge ? "on" : "off")); } advanceStop(vmUuid, VmDestroyForcestop.value()); deleteVMSnapshots(vm, expunge); Transaction.execute(new TransactionCallbackWithExceptionNoReturn<CloudRuntimeException>() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) throws CloudRuntimeException { VMInstanceVO vm = _vmDao.findByUuid(vmUuid); try { if (!stateTransitTo(vm, VirtualMachine.Event.DestroyRequested, vm.getHostId())) { s_logger.debug("Unable to destroy the vm because it is not in the correct state: " + vm); throw new CloudRuntimeException("Unable to destroy " + vm); } } catch (final NoTransitionException e) { s_logger.debug(e.getMessage()); throw new CloudRuntimeException("Unable to destroy " + vm, e); } } }); } /** * Delete vm snapshots depending on vm's hypervisor type. For Vmware, vm snapshots removal is delegated to vm cleanup thread * to reduce tasks sent to hypervisor (one tasks to delete vm snapshots and vm itself * instead of one task for each vm snapshot plus another for the vm) * @param vm vm * @param expunge indicates if vm should be expunged */ private void deleteVMSnapshots(VMInstanceVO vm, boolean expunge) { if (!vm.getHypervisorType().equals(HypervisorType.VMware)) { if (!_vmSnapshotMgr.deleteAllVMSnapshots(vm.getId(), null)) { s_logger.debug("Unable to delete all snapshots for " + vm); throw new CloudRuntimeException("Unable to delete vm snapshots for " + vm); } } else { if (expunge) { _vmSnapshotMgr.deleteVMSnapshotsFromDB(vm.getId()); } } } protected boolean checkVmOnHost(final VirtualMachine vm, final long hostId) throws AgentUnavailableException, OperationTimedoutException { final Answer answer = _agentMgr.send(hostId, new CheckVirtualMachineCommand(vm.getInstanceName())); if (answer == null || !answer.getResult()) { return false; } if (answer instanceof CheckVirtualMachineAnswer) { final CheckVirtualMachineAnswer vmAnswer = (CheckVirtualMachineAnswer) answer; if (vmAnswer.getState() == PowerState.PowerOff) { return false; } } UserVmVO userVm = _userVmDao.findById(vm.getId()); if (userVm != null) { List<VMSnapshotVO> vmSnapshots = _vmSnapshotDao.findByVm(vm.getId()); RestoreVMSnapshotCommand command = _vmSnapshotMgr.createRestoreCommand(userVm, vmSnapshots); if (command != null) { RestoreVMSnapshotAnswer restoreVMSnapshotAnswer = (RestoreVMSnapshotAnswer) _agentMgr.send(hostId, command); if (restoreVMSnapshotAnswer == null || !restoreVMSnapshotAnswer.getResult()) { s_logger.warn( "Unable to restore the vm snapshot from image file after live migration of vm with vmsnapshots: " + restoreVMSnapshotAnswer.getDetails()); } } } return true; } @Override public void storageMigration(final String vmUuid, final StoragePool destPool) { final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { // avoid re-entrance VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); placeHolder = createPlaceHolderWork(vm.getId()); try { orchestrateStorageMigration(vmUuid, destPool); } finally { if (placeHolder != null) { _workJobDao.expunge(placeHolder.getId()); } } } else { final Outcome<VirtualMachine> outcome = migrateVmStorageThroughJobQueue(vmUuid, destPool); try { final VirtualMachine vm = outcome.get(); } catch (final InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (final java.util.concurrent.ExecutionException e) { throw new RuntimeException("Execution excetion", e); } final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); if (jobResult != null) { if (jobResult instanceof RuntimeException) { throw (RuntimeException) jobResult; } else if (jobResult instanceof Throwable) { throw new RuntimeException("Unexpected exception", (Throwable) jobResult); } } } } private void orchestrateStorageMigration(final String vmUuid, final StoragePool destPool) { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); if (destPool == null) { throw new CloudRuntimeException("Unable to migrate vm: missing destination storage pool"); } try { stateTransitTo(vm, VirtualMachine.Event.StorageMigrationRequested, null); } catch (final NoTransitionException e) { s_logger.debug("Unable to migrate vm: " + e.toString()); throw new CloudRuntimeException("Unable to migrate vm: " + e.toString()); } final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); boolean migrationResult = false; try { migrationResult = volumeMgr.storageMigration(profile, destPool); if (migrationResult) { //if the vm is migrated to different pod in basic mode, need to reallocate ip if (destPool.getPodId() != null && !destPool.getPodId().equals(vm.getPodIdToDeployIn())) { final DataCenterDeployment plan = new DataCenterDeployment(vm.getDataCenterId(), destPool.getPodId(), null, null, null, null); final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vm, null, null, null, null); _networkMgr.reallocate(vmProfile, plan); } //when start the vm next time, don;'t look at last_host_id, only choose the host based on volume/storage pool vm.setLastHostId(null); vm.setPodIdToDeployIn(destPool.getPodId()); // If VM was cold migrated between clusters belonging to two different VMware DCs, // unregister the VM from the source host and cleanup the associated VM files. if (vm.getHypervisorType().equals(HypervisorType.VMware)) { Long srcClusterId = null; Long srcHostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId(); if (srcHostId != null) { HostVO srcHost = _hostDao.findById(srcHostId); srcClusterId = srcHost.getClusterId(); } final Long destClusterId = destPool.getClusterId(); if (srcClusterId != null && destClusterId != null && !srcClusterId.equals(destClusterId)) { final String srcDcName = _clusterDetailsDao.getVmwareDcName(srcClusterId); final String destDcName = _clusterDetailsDao.getVmwareDcName(destClusterId); if (srcDcName != null && destDcName != null && !srcDcName.equals(destDcName)) { s_logger.debug( "Since VM's storage was successfully migrated across VMware Datacenters, unregistering VM: " + vm.getInstanceName() + " from source host: " + srcHostId); final UnregisterVMCommand uvc = new UnregisterVMCommand(vm.getInstanceName()); uvc.setCleanupVmFiles(true); try { _agentMgr.send(srcHostId, uvc); } catch (final AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException("Failed to unregister VM: " + vm.getInstanceName() + " from source host: " + srcHostId + " after successfully migrating VM's storage across VMware Datacenters"); } } } } } else { s_logger.debug("Storage migration failed"); } } catch (final ConcurrentOperationException e) { s_logger.debug("Failed to migration: " + e.toString()); throw new CloudRuntimeException("Failed to migration: " + e.toString()); } catch (final InsufficientVirtualNetworkCapacityException e) { s_logger.debug("Failed to migration: " + e.toString()); throw new CloudRuntimeException("Failed to migration: " + e.toString()); } catch (final InsufficientAddressCapacityException e) { s_logger.debug("Failed to migration: " + e.toString()); throw new CloudRuntimeException("Failed to migration: " + e.toString()); } catch (final InsufficientCapacityException e) { s_logger.debug("Failed to migration: " + e.toString()); throw new CloudRuntimeException("Failed to migration: " + e.toString()); } catch (final StorageUnavailableException e) { s_logger.debug("Failed to migration: " + e.toString()); throw new CloudRuntimeException("Failed to migration: " + e.toString()); } finally { try { stateTransitTo(vm, VirtualMachine.Event.AgentReportStopped, null); } catch (final NoTransitionException e) { s_logger.debug("Failed to change vm state: " + e.toString()); throw new CloudRuntimeException("Failed to change vm state: " + e.toString()); } } } @Override public void migrate(final String vmUuid, final long srcHostId, final DeployDestination dest) throws ResourceUnavailableException, ConcurrentOperationException { final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { // avoid re-entrance VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); placeHolder = createPlaceHolderWork(vm.getId()); try { orchestrateMigrate(vmUuid, srcHostId, dest); } finally { if (placeHolder != null) { _workJobDao.expunge(placeHolder.getId()); } } } else { final Outcome<VirtualMachine> outcome = migrateVmThroughJobQueue(vmUuid, srcHostId, dest); try { final VirtualMachine vm = outcome.get(); } catch (final InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (final java.util.concurrent.ExecutionException e) { throw new RuntimeException("Execution excetion", e); } final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); if (jobResult != null) { if (jobResult instanceof ResourceUnavailableException) { throw (ResourceUnavailableException) jobResult; } else if (jobResult instanceof ConcurrentOperationException) { throw (ConcurrentOperationException) jobResult; } else if (jobResult instanceof RuntimeException) { throw (RuntimeException) jobResult; } else if (jobResult instanceof Throwable) { throw new RuntimeException("Unexpected exception", (Throwable) jobResult); } } } } private void orchestrateMigrate(final String vmUuid, final long srcHostId, final DeployDestination dest) throws ResourceUnavailableException, ConcurrentOperationException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); if (vm == null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Unable to find the vm " + vmUuid); } throw new CloudRuntimeException("Unable to find a virtual machine with id " + vmUuid); } migrate(vm, srcHostId, dest); } protected void migrate(final VMInstanceVO vm, final long srcHostId, final DeployDestination dest) throws ResourceUnavailableException, ConcurrentOperationException { s_logger.info("Migrating " + vm + " to " + dest); final long dstHostId = dest.getHost().getId(); final Host fromHost = _hostDao.findById(srcHostId); if (fromHost == null) { s_logger.info("Unable to find the host to migrate from: " + srcHostId); throw new CloudRuntimeException("Unable to find the host to migrate from: " + srcHostId); } if (fromHost.getClusterId().longValue() != dest.getCluster().getId()) { final List<VolumeVO> volumes = _volsDao.findCreatedByInstance(vm.getId()); for (final VolumeVO volume : volumes) { if (!_storagePoolDao.findById(volume.getPoolId()).getScope().equals(ScopeType.ZONE)) { s_logger.info( "Source and destination host are not in same cluster and all volumes are not on zone wide primary store, unable to migrate to host: " + dest.getHost().getId()); throw new CloudRuntimeException( "Source and destination host are not in same cluster and all volumes are not on zone wide primary store, unable to migrate to host: " + dest.getHost().getId()); } } } final VirtualMachineGuru vmGuru = getVmGuru(vm); if (vm.getState() != State.Running) { if (s_logger.isDebugEnabled()) { s_logger.debug("VM is not Running, unable to migrate the vm " + vm); } throw new CloudRuntimeException("VM is not Running, unable to migrate the vm currently " + vm + " , current state: " + vm.getState().toString()); } AlertManager.AlertType alertType = AlertManager.AlertType.ALERT_TYPE_USERVM_MIGRATE; if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) { alertType = AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER_MIGRATE; } else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { alertType = AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY_MIGRATE; } final VirtualMachineProfile vmSrc = new VirtualMachineProfileImpl(vm); for (final NicProfile nic : _networkMgr.getNicProfiles(vm)) { vmSrc.addNic(nic); } final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm, null, _offeringDao.findById(vm.getId(), vm.getServiceOfferingId()), null, null); _networkMgr.prepareNicForMigration(profile, dest); volumeMgr.prepareForMigration(profile, dest); profile.setConfigDriveLabel(VmConfigDriveLabel.value()); final VirtualMachineTO to = toVmTO(profile); final PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(to); ItWorkVO work = new ItWorkVO(UUID.randomUUID().toString(), _nodeId, State.Migrating, vm.getType(), vm.getId()); work.setStep(Step.Prepare); work.setResourceType(ItWorkVO.ResourceType.Host); work.setResourceId(dstHostId); work = _workDao.persist(work); Answer pfma = null; try { pfma = _agentMgr.send(dstHostId, pfmc); if (pfma == null || !pfma.getResult()) { final String details = pfma != null ? pfma.getDetails() : "null answer returned"; final String msg = "Unable to prepare for migration due to " + details; pfma = null; throw new AgentUnavailableException(msg, dstHostId); } } catch (final OperationTimedoutException e1) { throw new AgentUnavailableException("Operation timed out", dstHostId); } finally { if (pfma == null) { _networkMgr.rollbackNicForMigration(vmSrc, profile); work.setStep(Step.Done); _workDao.update(work.getId(), work); } } vm.setLastHostId(srcHostId); try { if (vm == null || vm.getHostId() == null || vm.getHostId() != srcHostId || !changeState(vm, Event.MigrationRequested, dstHostId, work, Step.Migrating)) { _networkMgr.rollbackNicForMigration(vmSrc, profile); s_logger.info("Migration cancelled because state has changed: " + vm); throw new ConcurrentOperationException("Migration cancelled because state has changed: " + vm); } } catch (final NoTransitionException e1) { _networkMgr.rollbackNicForMigration(vmSrc, profile); s_logger.info("Migration cancelled because " + e1.getMessage()); throw new ConcurrentOperationException("Migration cancelled because " + e1.getMessage()); } boolean migrated = false; try { final boolean isWindows = _guestOsCategoryDao .findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName() .equalsIgnoreCase("Windows"); final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType())); String autoConvergence = _configDao.getValue(Config.KvmAutoConvergence.toString()); boolean kvmAutoConvergence = Boolean.parseBoolean(autoConvergence); mc.setAutoConvergence(kvmAutoConvergence); mc.setHostGuid(dest.getHost().getGuid()); try { final Answer ma = _agentMgr.send(vm.getLastHostId(), mc); if (ma == null || !ma.getResult()) { final String details = ma != null ? ma.getDetails() : "null answer returned"; throw new CloudRuntimeException(details); } } catch (final OperationTimedoutException e) { if (e.isActive()) { s_logger.warn("Active migration command so scheduling a restart for " + vm); _haMgr.scheduleRestart(vm, true); } throw new AgentUnavailableException("Operation timed out on migrating " + vm, dstHostId); } try { if (!changeState(vm, VirtualMachine.Event.OperationSucceeded, dstHostId, work, Step.Started)) { throw new ConcurrentOperationException("Unable to change the state for " + vm); } } catch (final NoTransitionException e1) { throw new ConcurrentOperationException("Unable to change state due to " + e1.getMessage()); } try { if (!checkVmOnHost(vm, dstHostId)) { s_logger.error("Unable to complete migration for " + vm); try { _agentMgr.send(srcHostId, new Commands(cleanup(vm)), null); } catch (final AgentUnavailableException e) { s_logger.error("AgentUnavailableException while cleanup on source host: " + srcHostId); } cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.AgentReportStopped, true); throw new CloudRuntimeException("Unable to complete migration for " + vm); } } catch (final OperationTimedoutException e) { s_logger.debug("Error while checking the vm " + vm + " on host " + dstHostId, e); } migrated = true; } finally { if (!migrated) { s_logger.info("Migration was unsuccessful. Cleaning up: " + vm); _networkMgr.rollbackNicForMigration(vmSrc, profile); _alertMgr.sendAlert(alertType, fromHost.getDataCenterId(), fromHost.getPodId(), "Unable to migrate vm " + vm.getInstanceName() + " from host " + fromHost.getName() + " in zone " + dest.getDataCenter().getName() + " and pod " + dest.getPod().getName(), "Migrate Command failed. Please check logs."); try { _agentMgr.send(dstHostId, new Commands(cleanup(vm)), null); } catch (final AgentUnavailableException ae) { s_logger.info("Looks like the destination Host is unavailable for cleanup"); } try { stateTransitTo(vm, Event.OperationFailed, srcHostId); } catch (final NoTransitionException e) { s_logger.warn(e.getMessage()); } } else { _networkMgr.commitNicForMigration(vmSrc, profile); } work.setStep(Step.Done); _workDao.update(work.getId(), work); } } /** * Create the mapping of volumes and storage pools. If the user did not enter a mapping on her/his own, we create one using {@link #getDefaultMappingOfVolumesAndStoragePoolForMigration(VirtualMachineProfile, Host)}. * If the user provided a mapping, we use whatever the user has provided (check the method {@link #createMappingVolumeAndStoragePoolEnteredByUser(VirtualMachineProfile, Host, Map)}). */ private Map<Volume, StoragePool> getPoolListForVolumesForMigration(VirtualMachineProfile profile, Host targetHost, Map<Long, Long> volumeToPool) { if (MapUtils.isEmpty(volumeToPool)) { return getDefaultMappingOfVolumesAndStoragePoolForMigration(profile, targetHost); } return createMappingVolumeAndStoragePoolEnteredByUser(profile, targetHost, volumeToPool); } /** * We create the mapping of volumes and storage pool to migrate the VMs according to the information sent by the user. */ private Map<Volume, StoragePool> createMappingVolumeAndStoragePoolEnteredByUser(VirtualMachineProfile profile, Host host, Map<Long, Long> volumeToPool) { Map<Volume, StoragePool> volumeToPoolObjectMap = new HashMap<Volume, StoragePool>(); for (Long volumeId : volumeToPool.keySet()) { VolumeVO volume = _volsDao.findById(volumeId); Long poolId = volumeToPool.get(volumeId); StoragePoolVO targetPool = _storagePoolDao.findById(poolId); StoragePoolVO currentPool = _storagePoolDao.findById(volume.getPoolId()); if (_poolHostDao.findByPoolHost(targetPool.getId(), host.getId()) == null) { throw new CloudRuntimeException(String.format( "Cannot migrate the volume [%s] to the storage pool [%s] while migrating VM [%s] to target host [%s]. The host does not have access to the storage pool entered.", volume.getUuid(), targetPool.getUuid(), profile.getUuid(), host.getUuid())); } if (currentPool.getId() == targetPool.getId()) { s_logger.info(String.format("The volume [%s] is already allocated in storage pool [%s].", volume.getUuid(), targetPool.getUuid())); } volumeToPoolObjectMap.put(volume, targetPool); } return volumeToPoolObjectMap; } /** * We create the default mapping of volumes and storage pools for the migration of the VM to the target host. * If the current storage pool of one of the volumes is using local storage in the host, it then needs to be migrated to a local storage in the target host. * Otherwise, we do not need to migrate, and the volume can be kept in its current storage pool. */ private Map<Volume, StoragePool> getDefaultMappingOfVolumesAndStoragePoolForMigration( VirtualMachineProfile profile, Host targetHost) { Map<Volume, StoragePool> volumeToPoolObjectMap = new HashMap<Volume, StoragePool>(); List<VolumeVO> allVolumes = _volsDao.findUsableVolumesForInstance(profile.getId()); for (VolumeVO volume : allVolumes) { StoragePoolVO currentPool = _storagePoolDao.findById(volume.getPoolId()); if (ScopeType.HOST.equals(currentPool.getScope())) { createVolumeToStoragePoolMappingIfNeeded(profile, targetHost, volumeToPoolObjectMap, volume, currentPool); } else { volumeToPoolObjectMap.put(volume, currentPool); } } return volumeToPoolObjectMap; } /** * We will add a mapping of volume to storage pool if needed. The conditions to add a mapping are the following: * <ul> * <li> The current storage pool where the volume is allocated can be accessed by the target host * <li> If not storage pool is found to allocate the volume we throw an exception. * </ul> * */ private void createVolumeToStoragePoolMappingIfNeeded(VirtualMachineProfile profile, Host targetHost, Map<Volume, StoragePool> volumeToPoolObjectMap, VolumeVO volume, StoragePoolVO currentPool) { List<StoragePool> poolList = getCandidateStoragePoolsToMigrateLocalVolume(profile, targetHost, volume); Collections.shuffle(poolList); boolean canTargetHostAccessVolumeStoragePool = false; for (StoragePool storagePool : poolList) { if (storagePool.getId() == currentPool.getId()) { canTargetHostAccessVolumeStoragePool = true; break; } } if (!canTargetHostAccessVolumeStoragePool && CollectionUtils.isEmpty(poolList)) { throw new CloudRuntimeException(String.format( "There is not storage pools avaliable at the target host [%s] to migrate volume [%s]", targetHost.getUuid(), volume.getUuid())); } if (!canTargetHostAccessVolumeStoragePool) { volumeToPoolObjectMap.put(volume, _storagePoolDao.findByUuid(poolList.get(0).getUuid())); } if (!canTargetHostAccessVolumeStoragePool && !volumeToPoolObjectMap.containsKey(volume)) { throw new CloudRuntimeException(String.format( "Cannot find a storage pool which is available for volume [%s] while migrating virtual machine [%s] to host [%s]", volume.getUuid(), profile.getUuid(), targetHost.getUuid())); } } /** * We use {@link StoragePoolAllocator} objects to find local storage pools connected to the targetHost where we would be able to allocate the given volume. */ private List<StoragePool> getCandidateStoragePoolsToMigrateLocalVolume(VirtualMachineProfile profile, Host targetHost, VolumeVO volume) { List<StoragePool> poolList = new ArrayList<>(); DiskOfferingVO diskOffering = _diskOfferingDao.findById(volume.getDiskOfferingId()); DiskProfile diskProfile = new DiskProfile(volume, diskOffering, profile.getHypervisorType()); DataCenterDeployment plan = new DataCenterDeployment(targetHost.getDataCenterId(), targetHost.getPodId(), targetHost.getClusterId(), targetHost.getId(), null, null); ExcludeList avoid = new ExcludeList(); StoragePoolVO volumeStoragePool = _storagePoolDao.findById(volume.getPoolId()); if (volumeStoragePool.isLocal()) { diskProfile.setUseLocalStorage(true); } for (StoragePoolAllocator allocator : _storagePoolAllocators) { List<StoragePool> poolListFromAllocator = allocator.allocateToPool(diskProfile, profile, plan, avoid, StoragePoolAllocator.RETURN_UPTO_ALL); if (CollectionUtils.isEmpty(poolListFromAllocator)) { continue; } for (StoragePool pool : poolListFromAllocator) { if (pool.isLocal()) { poolList.add(pool); } } } return poolList; } private <T extends VMInstanceVO> void moveVmToMigratingState(final T vm, final Long hostId, final ItWorkVO work) throws ConcurrentOperationException { // Put the vm in migrating state. try { if (!changeState(vm, Event.MigrationRequested, hostId, work, Step.Migrating)) { s_logger.info("Migration cancelled because state has changed: " + vm); throw new ConcurrentOperationException("Migration cancelled because state has changed: " + vm); } } catch (final NoTransitionException e) { s_logger.info("Migration cancelled because " + e.getMessage()); throw new ConcurrentOperationException("Migration cancelled because " + e.getMessage()); } } private <T extends VMInstanceVO> void moveVmOutofMigratingStateOnSuccess(final T vm, final Long hostId, final ItWorkVO work) throws ConcurrentOperationException { // Put the vm in running state. try { if (!changeState(vm, Event.OperationSucceeded, hostId, work, Step.Started)) { s_logger.error("Unable to change the state for " + vm); throw new ConcurrentOperationException("Unable to change the state for " + vm); } } catch (final NoTransitionException e) { s_logger.error("Unable to change state due to " + e.getMessage()); throw new ConcurrentOperationException("Unable to change state due to " + e.getMessage()); } } @Override public void migrateWithStorage(final String vmUuid, final long srcHostId, final long destHostId, final Map<Long, Long> volumeToPool) throws ResourceUnavailableException, ConcurrentOperationException { final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { // avoid re-entrance VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); placeHolder = createPlaceHolderWork(vm.getId()); try { orchestrateMigrateWithStorage(vmUuid, srcHostId, destHostId, volumeToPool); } finally { if (placeHolder != null) { _workJobDao.expunge(placeHolder.getId()); } } } else { final Outcome<VirtualMachine> outcome = migrateVmWithStorageThroughJobQueue(vmUuid, srcHostId, destHostId, volumeToPool); try { final VirtualMachine vm = outcome.get(); } catch (final InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (final java.util.concurrent.ExecutionException e) { throw new RuntimeException("Execution excetion", e); } final Object jobException = _jobMgr.unmarshallResultObject(outcome.getJob()); if (jobException != null) { if (jobException instanceof ResourceUnavailableException) { throw (ResourceUnavailableException) jobException; } else if (jobException instanceof ConcurrentOperationException) { throw (ConcurrentOperationException) jobException; } else if (jobException instanceof RuntimeException) { throw (RuntimeException) jobException; } else if (jobException instanceof Throwable) { throw new RuntimeException("Unexpected exception", (Throwable) jobException); } } } } private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHostId, final long destHostId, final Map<Long, Long> volumeToPool) throws ResourceUnavailableException, ConcurrentOperationException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); final HostVO srcHost = _hostDao.findById(srcHostId); final HostVO destHost = _hostDao.findById(destHostId); final VirtualMachineGuru vmGuru = getVmGuru(vm); final DataCenterVO dc = _dcDao.findById(destHost.getDataCenterId()); final HostPodVO pod = _podDao.findById(destHost.getPodId()); final Cluster cluster = _clusterDao.findById(destHost.getClusterId()); final DeployDestination destination = new DeployDestination(dc, pod, cluster, destHost); // Create a map of which volume should go in which storage pool. final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); final Map<Volume, StoragePool> volumeToPoolMap = getPoolListForVolumesForMigration(profile, destHost, volumeToPool); // If none of the volumes have to be migrated, fail the call. Administrator needs to make a call for migrating // a vm and not migrating a vm with storage. if (volumeToPoolMap == null || volumeToPoolMap.isEmpty()) { throw new InvalidParameterValueException("Migration of the vm " + vm + "from host " + srcHost + " to destination host " + destHost + " doesn't involve migrating the volumes."); } AlertManager.AlertType alertType = AlertManager.AlertType.ALERT_TYPE_USERVM_MIGRATE; if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) { alertType = AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER_MIGRATE; } else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { alertType = AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY_MIGRATE; } _networkMgr.prepareNicForMigration(profile, destination); volumeMgr.prepareForMigration(profile, destination); final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType()); final VirtualMachineTO to = hvGuru.implement(profile); ItWorkVO work = new ItWorkVO(UUID.randomUUID().toString(), _nodeId, State.Migrating, vm.getType(), vm.getId()); work.setStep(Step.Prepare); work.setResourceType(ItWorkVO.ResourceType.Host); work.setResourceId(destHostId); work = _workDao.persist(work); // Put the vm in migrating state. vm.setLastHostId(srcHostId); vm.setPodIdToDeployIn(destHost.getPodId()); moveVmToMigratingState(vm, destHostId, work); boolean migrated = false; try { // config drive: Detach the config drive at source host // After migration successful attach the config drive in destination host // On migration failure VM will be stopped, So configIso will be deleted Nic defaultNic = _networkModel.getDefaultNic(vm.getId()); List<String[]> vmData = null; if (defaultNic != null) { UserVmVO userVm = _userVmDao.findById(vm.getId()); Map<String, String> details = _vmDetailsDao.listDetailsKeyPairs(vm.getId()); vm.setDetails(details); Network network = _networkModel.getNetwork(defaultNic.getNetworkId()); if (_networkModel.isSharedNetworkWithoutServices(network.getId())) { final String serviceOffering = _serviceOfferingDao .findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText(); final String zoneName = _dcDao.findById(vm.getDataCenterId()).getName(); boolean isWindows = _guestOSCategoryDao .findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName() .equalsIgnoreCase("Windows"); vmData = _networkModel.generateVmData(userVm.getUserData(), serviceOffering, zoneName, vm.getInstanceName(), vm.getId(), (String) profile.getParameter(VirtualMachineProfile.Param.VmSshPubKey), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows); String vmName = vm.getInstanceName(); String configDriveIsoRootFolder = "/tmp"; String isoFile = configDriveIsoRootFolder + "/" + vmName + "/configDrive/" + vmName + ".iso"; profile.setVmData(vmData); profile.setConfigDriveLabel(VmConfigDriveLabel.value()); profile.setConfigDriveIsoRootFolder(configDriveIsoRootFolder); profile.setConfigDriveIsoFile(isoFile); // At source host detach the config drive iso. AttachOrDettachConfigDriveCommand dettachCommand = new AttachOrDettachConfigDriveCommand( vm.getInstanceName(), vmData, VmConfigDriveLabel.value(), false); try { _agentMgr.send(srcHost.getId(), dettachCommand); s_logger.debug( "Deleted config drive ISO for vm " + vm.getInstanceName() + " In host " + srcHost); } catch (OperationTimedoutException e) { s_logger.debug("TIme out occured while exeuting command AttachOrDettachConfigDrive " + e.getMessage()); } } } // Migrate the vm and its volume. volumeMgr.migrateVolumes(vm, to, srcHost, destHost, volumeToPoolMap); // Put the vm back to running state. moveVmOutofMigratingStateOnSuccess(vm, destHost.getId(), work); try { if (!checkVmOnHost(vm, destHostId)) { s_logger.error("Vm not found on destination host. Unable to complete migration for " + vm); try { _agentMgr.send(srcHostId, new Commands(cleanup(vm.getInstanceName())), null); } catch (final AgentUnavailableException e) { s_logger.error("AgentUnavailableException while cleanup on source host: " + srcHostId); } cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.AgentReportStopped, true); throw new CloudRuntimeException( "VM not found on desintation host. Unable to complete migration for " + vm); } } catch (final OperationTimedoutException e) { s_logger.warn("Error while checking the vm " + vm + " is on host " + destHost, e); } migrated = true; } finally { if (!migrated) { s_logger.info("Migration was unsuccessful. Cleaning up: " + vm); _alertMgr.sendAlert(alertType, srcHost.getDataCenterId(), srcHost.getPodId(), "Unable to migrate vm " + vm.getInstanceName() + " from host " + srcHost.getName() + " in zone " + dc.getName() + " and pod " + dc.getName(), "Migrate Command failed. Please check logs."); try { _agentMgr.send(destHostId, new Commands(cleanup(vm.getInstanceName())), null); vm.setPodIdToDeployIn(srcHost.getPodId()); stateTransitTo(vm, Event.OperationFailed, srcHostId); } catch (final AgentUnavailableException e) { s_logger.warn("Looks like the destination Host is unavailable for cleanup.", e); } catch (final NoTransitionException e) { s_logger.error("Error while transitioning vm from migrating to running state.", e); } } work.setStep(Step.Done); _workDao.update(work.getId(), work); } } @Override public VirtualMachineTO toVmTO(final VirtualMachineProfile profile) { final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(profile.getVirtualMachine().getHypervisorType()); final VirtualMachineTO to = hvGuru.implement(profile); return to; } protected void cancelWorkItems(final long nodeId) { final GlobalLock scanLock = GlobalLock.getInternLock("vmmgr.cancel.workitem"); try { if (scanLock.lock(3)) { try { final List<ItWorkVO> works = _workDao.listWorkInProgressFor(nodeId); for (final ItWorkVO work : works) { s_logger.info("Handling unfinished work item: " + work); try { final VMInstanceVO vm = _vmDao.findById(work.getInstanceId()); if (vm != null) { if (work.getType() == State.Starting) { _haMgr.scheduleRestart(vm, true); work.setManagementServerId(_nodeId); work.setStep(Step.Done); _workDao.update(work.getId(), work); } else if (work.getType() == State.Stopping) { _haMgr.scheduleStop(vm, vm.getHostId(), WorkType.CheckStop); work.setManagementServerId(_nodeId); work.setStep(Step.Done); _workDao.update(work.getId(), work); } else if (work.getType() == State.Migrating) { _haMgr.scheduleMigration(vm); work.setStep(Step.Done); _workDao.update(work.getId(), work); } } } catch (final Exception e) { s_logger.error("Error while handling " + work, e); } } } finally { scanLock.unlock(); } } } finally { scanLock.releaseRef(); } } @Override public void migrateAway(final String vmUuid, final long srcHostId) throws InsufficientServerCapacityException { final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { // avoid re-entrance VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); placeHolder = createPlaceHolderWork(vm.getId()); try { try { orchestrateMigrateAway(vmUuid, srcHostId, null); } catch (final InsufficientServerCapacityException e) { s_logger.warn("Failed to deploy vm " + vmUuid + " with original planner, sending HAPlanner"); orchestrateMigrateAway(vmUuid, srcHostId, _haMgr.getHAPlanner()); } } finally { _workJobDao.expunge(placeHolder.getId()); } } else { final Outcome<VirtualMachine> outcome = migrateVmAwayThroughJobQueue(vmUuid, srcHostId); try { final VirtualMachine vm = outcome.get(); } catch (final InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (final java.util.concurrent.ExecutionException e) { throw new RuntimeException("Execution excetion", e); } final Object jobException = _jobMgr.unmarshallResultObject(outcome.getJob()); if (jobException != null) { if (jobException instanceof InsufficientServerCapacityException) { throw (InsufficientServerCapacityException) jobException; } else if (jobException instanceof ConcurrentOperationException) { throw (ConcurrentOperationException) jobException; } else if (jobException instanceof RuntimeException) { throw (RuntimeException) jobException; } else if (jobException instanceof Throwable) { throw new RuntimeException("Unexpected exception", (Throwable) jobException); } } } } private void orchestrateMigrateAway(final String vmUuid, final long srcHostId, final DeploymentPlanner planner) throws InsufficientServerCapacityException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); if (vm == null) { s_logger.debug("Unable to find a VM for " + vmUuid); throw new CloudRuntimeException("Unable to find " + vmUuid); } ServiceOfferingVO offeringVO = _offeringDao.findById(vm.getId(), vm.getServiceOfferingId()); final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm, null, offeringVO, null, null); final Long hostId = vm.getHostId(); if (hostId == null) { s_logger.debug("Unable to migrate because the VM doesn't have a host id: " + vm); throw new CloudRuntimeException("Unable to migrate " + vmUuid); } final Host host = _hostDao.findById(hostId); Long poolId = null; final List<VolumeVO> vols = _volsDao.findReadyRootVolumesByInstance(vm.getId()); for (final VolumeVO rootVolumeOfVm : vols) { final StoragePoolVO rootDiskPool = _storagePoolDao.findById(rootVolumeOfVm.getPoolId()); if (rootDiskPool != null) { poolId = rootDiskPool.getId(); } } final DataCenterDeployment plan = new DataCenterDeployment(host.getDataCenterId(), host.getPodId(), host.getClusterId(), null, poolId, null); final ExcludeList excludes = new ExcludeList(); excludes.addHost(hostId); DeployDestination dest = null; while (true) { try { dest = _dpMgr.planDeployment(profile, plan, excludes, planner); } catch (final AffinityConflictException e2) { s_logger.warn("Unable to create deployment, affinity rules associted to the VM conflict", e2); throw new CloudRuntimeException( "Unable to create deployment, affinity rules associted to the VM conflict"); } if (dest != null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Found destination " + dest + " for migrating to."); } } else { if (s_logger.isDebugEnabled()) { s_logger.debug("Unable to find destination for migrating the vm " + profile); } throw new InsufficientServerCapacityException("Unable to find a server to migrate to.", host.getClusterId()); } excludes.addHost(dest.getHost().getId()); try { migrate(vm, srcHostId, dest); return; } catch (final ResourceUnavailableException e) { s_logger.debug("Unable to migrate to unavailable " + dest); } catch (final ConcurrentOperationException e) { s_logger.debug("Unable to migrate VM due to: " + e.getMessage()); } try { advanceStop(vmUuid, true); throw new CloudRuntimeException("Unable to migrate " + vm); } catch (final ResourceUnavailableException e) { s_logger.debug("Unable to stop VM due to " + e.getMessage()); throw new CloudRuntimeException("Unable to migrate " + vm); } catch (final ConcurrentOperationException e) { s_logger.debug("Unable to stop VM due to " + e.getMessage()); throw new CloudRuntimeException("Unable to migrate " + vm); } catch (final OperationTimedoutException e) { s_logger.debug("Unable to stop VM due to " + e.getMessage()); throw new CloudRuntimeException("Unable to migrate " + vm); } } } protected class CleanupTask extends ManagedContextRunnable { @Override protected void runInContext() { s_logger.trace("VM Operation Thread Running"); try { _workDao.cleanup(VmOpCleanupWait.value()); final Date cutDate = new Date( DateUtil.currentGMTTime().getTime() - VmOpCleanupInterval.value() * 1000); _workJobDao.expungeCompletedWorkJobs(cutDate); } catch (final Exception e) { s_logger.error("VM Operations failed due to ", e); } } } @Override public boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering) { boolean isMachineUpgradable = true; for (final HostAllocator allocator : hostAllocators) { isMachineUpgradable = allocator.isVirtualMachineUpgradable(vm, offering); if (isMachineUpgradable) { continue; } else { break; } } return isMachineUpgradable; } @Override public void reboot(final String vmUuid, final Map<VirtualMachineProfile.Param, Object> params) throws InsufficientCapacityException, ResourceUnavailableException { try { advanceReboot(vmUuid, params); } catch (final ConcurrentOperationException e) { throw new CloudRuntimeException("Unable to reboot a VM due to concurrent operation", e); } } @Override public void advanceReboot(final String vmUuid, final Map<VirtualMachineProfile.Param, Object> params) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { // avoid re-entrance VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); placeHolder = createPlaceHolderWork(vm.getId()); try { orchestrateReboot(vmUuid, params); } finally { if (placeHolder != null) { _workJobDao.expunge(placeHolder.getId()); } } } else { final Outcome<VirtualMachine> outcome = rebootVmThroughJobQueue(vmUuid, params); try { final VirtualMachine vm = outcome.get(); } catch (final InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (final java.util.concurrent.ExecutionException e) { throw new RuntimeException("Execution excetion", e); } final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); if (jobResult != null) { if (jobResult instanceof ResourceUnavailableException) { throw (ResourceUnavailableException) jobResult; } else if (jobResult instanceof ConcurrentOperationException) { throw (ConcurrentOperationException) jobResult; } else if (jobResult instanceof InsufficientCapacityException) { throw (InsufficientCapacityException) jobResult; } else if (jobResult instanceof RuntimeException) { throw (RuntimeException) jobResult; } else if (jobResult instanceof Throwable) { throw new RuntimeException("Unexpected exception", (Throwable) jobResult); } } } } private void orchestrateReboot(final String vmUuid, final Map<VirtualMachineProfile.Param, Object> params) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); // if there are active vm snapshots task, state change is not allowed if (_vmSnapshotMgr.hasActiveVMSnapshotTasks(vm.getId())) { s_logger.error("Unable to reboot VM " + vm + " due to: " + vm.getInstanceName() + " has active VM snapshots tasks"); throw new CloudRuntimeException("Unable to reboot VM " + vm + " due to: " + vm.getInstanceName() + " has active VM snapshots tasks"); } final DataCenter dc = _entityMgr.findById(DataCenter.class, vm.getDataCenterId()); final Host host = _hostDao.findById(vm.getHostId()); if (host == null) { // Should findById throw an Exception is the host is not found? throw new CloudRuntimeException("Unable to retrieve host with id " + vm.getHostId()); } final Cluster cluster = _entityMgr.findById(Cluster.class, host.getClusterId()); final Pod pod = _entityMgr.findById(Pod.class, host.getPodId()); final DeployDestination dest = new DeployDestination(dc, pod, cluster, host); try { final Commands cmds = new Commands(Command.OnError.Stop); cmds.addCommand(new RebootCommand(vm.getInstanceName(), getExecuteInSequence(vm.getHypervisorType()))); _agentMgr.send(host.getId(), cmds); final Answer rebootAnswer = cmds.getAnswer(RebootAnswer.class); if (rebootAnswer != null && rebootAnswer.getResult()) { return; } s_logger.info("Unable to reboot VM " + vm + " on " + dest.getHost() + " due to " + (rebootAnswer == null ? " no reboot answer" : rebootAnswer.getDetails())); } catch (final OperationTimedoutException e) { s_logger.warn("Unable to send the reboot command to host " + dest.getHost() + " for the vm " + vm + " due to operation timeout", e); throw new CloudRuntimeException("Failed to reboot the vm on host " + dest.getHost()); } } public Command cleanup(final VirtualMachine vm) { StopCommand cmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false); cmd.setControlIp(getControlNicIpForVM(vm)); return cmd; } private String getControlNicIpForVM(VirtualMachine vm) { if (vm.getType() == VirtualMachine.Type.ConsoleProxy || vm.getType() == VirtualMachine.Type.SecondaryStorageVm) { NicVO nic = _nicsDao.getControlNicForVM(vm.getId()); return nic.getIPv4Address(); } else if (vm.getType() == VirtualMachine.Type.DomainRouter) { return vm.getPrivateIpAddress(); } else { return null; } } public Command cleanup(final String vmName) { VirtualMachine vm = _vmDao.findVMByInstanceName(vmName); StopCommand cmd = new StopCommand(vmName, getExecuteInSequence(null), false); cmd.setControlIp(getControlNicIpForVM(vm)); return cmd; } // this is XenServer specific public void syncVMMetaData(final Map<String, String> vmMetadatum) { if (vmMetadatum == null || vmMetadatum.isEmpty()) { return; } List<Pair<Pair<String, VirtualMachine.Type>, Pair<Long, String>>> vmDetails = _userVmDao .getVmsDetailByNames(vmMetadatum.keySet(), "platform"); for (final Map.Entry<String, String> entry : vmMetadatum.entrySet()) { final String name = entry.getKey(); final String platform = entry.getValue(); if (platform == null || platform.isEmpty()) { continue; } boolean found = false; for (Pair<Pair<String, VirtualMachine.Type>, Pair<Long, String>> vmDetail : vmDetails) { Pair<String, VirtualMachine.Type> vmNameTypePair = vmDetail.first(); if (vmNameTypePair.first().equals(name)) { found = true; if (vmNameTypePair.second() == VirtualMachine.Type.User) { Pair<Long, String> detailPair = vmDetail.second(); String platformDetail = detailPair.second(); if (platformDetail != null && platformDetail.equals(platform)) { break; } updateVmMetaData(detailPair.first(), platform); } break; } } if (!found) { VMInstanceVO vm = _vmDao.findVMByInstanceName(name); if (vm != null && vm.getType() == VirtualMachine.Type.User) { updateVmMetaData(vm.getId(), platform); } } } } // this is XenServer specific private void updateVmMetaData(Long vmId, String platform) { UserVmVO userVm = _userVmDao.findById(vmId); _userVmDao.loadDetails(userVm); if (userVm.details.containsKey("timeoffset")) { userVm.details.remove("timeoffset"); } userVm.setDetail("platform", platform); String pvdriver = "xenserver56"; if (platform.contains("device_id")) { pvdriver = "xenserver61"; } if (!userVm.details.containsKey("hypervisortoolsversion") || !userVm.details.get("hypervisortoolsversion").equals(pvdriver)) { userVm.setDetail("hypervisortoolsversion", pvdriver); } _userVmDao.saveDetails(userVm); } @Override public boolean isRecurring() { return true; } @Override public boolean processAnswers(final long agentId, final long seq, final Answer[] answers) { for (final Answer answer : answers) { if (answer instanceof ClusterVMMetaDataSyncAnswer) { final ClusterVMMetaDataSyncAnswer cvms = (ClusterVMMetaDataSyncAnswer) answer; if (!cvms.isExecuted()) { syncVMMetaData(cvms.getVMMetaDatum()); cvms.setExecuted(); } } } return true; } @Override public boolean processTimeout(final long agentId, final long seq) { return true; } @Override public int getTimeout() { return -1; } @Override public boolean processCommands(final long agentId, final long seq, final Command[] cmds) { boolean processed = false; for (final Command cmd : cmds) { if (cmd instanceof PingRoutingCommand) { final PingRoutingCommand ping = (PingRoutingCommand) cmd; if (ping.getHostVmStateReport() != null) { _syncMgr.processHostVmStatePingReport(agentId, ping.getHostVmStateReport()); } // take the chance to scan VMs that are stuck in transitional states // and are missing from the report scanStalledVMInTransitionStateOnUpHost(agentId); processed = true; } } return processed; } @Override public AgentControlAnswer processControlCommand(final long agentId, final AgentControlCommand cmd) { return null; } @Override public boolean processDisconnect(final long agentId, final Status state) { return true; } @Override public void processHostAboutToBeRemoved(long hostId) { } @Override public void processHostRemoved(long hostId, long clusterId) { } @Override public void processHostAdded(long hostId) { } @Override public void processConnect(final Host agent, final StartupCommand cmd, final boolean forRebalance) throws ConnectionException { if (!(cmd instanceof StartupRoutingCommand)) { return; } if (s_logger.isDebugEnabled()) { s_logger.debug("Received startup command from hypervisor host. host id: " + agent.getId()); } _syncMgr.resetHostSyncState(agent.getId()); if (forRebalance) { s_logger.debug("Not processing listener " + this + " as connect happens on rebalance process"); return; } final Long clusterId = agent.getClusterId(); final long agentId = agent.getId(); if (agent.getHypervisorType() == HypervisorType.XenServer) { // only for Xen // initiate the cron job final ClusterVMMetaDataSyncCommand syncVMMetaDataCmd = new ClusterVMMetaDataSyncCommand( ClusterVMMetaDataSyncInterval.value(), clusterId); try { final long seq_no = _agentMgr.send(agentId, new Commands(syncVMMetaDataCmd), this); s_logger.debug("Cluster VM metadata sync started with jobid " + seq_no); } catch (final AgentUnavailableException e) { s_logger.fatal("The Cluster VM metadata sync process failed for cluster id " + clusterId + " with ", e); } } } protected class TransitionTask extends ManagedContextRunnable { @Override protected void runInContext() { final GlobalLock lock = GlobalLock.getInternLock("TransitionChecking"); if (lock == null) { s_logger.debug("Couldn't get the global lock"); return; } if (!lock.lock(30)) { s_logger.debug("Couldn't lock the db"); return; } try { scanStalledVMInTransitionStateOnDisconnectedHosts(); final List<VMInstanceVO> instances = _vmDao.findVMInTransition( new Date(DateUtil.currentGMTTime().getTime() - AgentManager.Wait.value() * 1000), State.Starting, State.Stopping); for (final VMInstanceVO instance : instances) { final State state = instance.getState(); if (state == State.Stopping) { _haMgr.scheduleStop(instance, instance.getHostId(), WorkType.CheckStop); } else if (state == State.Starting) { _haMgr.scheduleRestart(instance, true); } } } catch (final Exception e) { s_logger.warn("Caught the following exception on transition checking", e); } finally { lock.unlock(); } } } @Override public VMInstanceVO findById(final long vmId) { return _vmDao.findById(vmId); } @Override public void checkIfCanUpgrade(final VirtualMachine vmInstance, final ServiceOffering newServiceOffering) { if (newServiceOffering == null) { throw new InvalidParameterValueException("Invalid parameter, newServiceOffering can't be null"); } // Check that the VM is stopped / running if (!(vmInstance.getState().equals(State.Stopped) || vmInstance.getState().equals(State.Running))) { s_logger.warn("Unable to upgrade virtual machine " + vmInstance.toString() + " in state " + vmInstance.getState()); throw new InvalidParameterValueException( "Unable to upgrade virtual machine " + vmInstance.toString() + " " + " in state " + vmInstance.getState() + "; make sure the virtual machine is stopped/running"); } // Check if the service offering being upgraded to is what the VM is already running with if (!newServiceOffering.isDynamic() && vmInstance.getServiceOfferingId() == newServiceOffering.getId()) { if (s_logger.isInfoEnabled()) { s_logger.info("Not upgrading vm " + vmInstance.toString() + " since it already has the requested " + "service offering (" + newServiceOffering.getName() + ")"); } throw new InvalidParameterValueException( "Not upgrading vm " + vmInstance.toString() + " since it already " + "has the requested service offering (" + newServiceOffering.getName() + ")"); } final ServiceOfferingVO currentServiceOffering = _offeringDao.findByIdIncludingRemoved(vmInstance.getId(), vmInstance.getServiceOfferingId()); // Check that the service offering being upgraded to has the same Guest IP type as the VM's current service offering // NOTE: With the new network refactoring in 2.2, we shouldn't need the check for same guest IP type anymore. /* * if (!currentServiceOffering.getGuestIpType().equals(newServiceOffering.getGuestIpType())) { String errorMsg = * "The service offering being upgraded to has a guest IP type: " + newServiceOffering.getGuestIpType(); errorMsg += * ". Please select a service offering with the same guest IP type as the VM's current service offering (" + * currentServiceOffering.getGuestIpType() + ")."; throw new InvalidParameterValueException(errorMsg); } */ // Check that the service offering being upgraded to has the same storage pool preference as the VM's current service // offering if (currentServiceOffering.getUseLocalStorage() != newServiceOffering.getUseLocalStorage()) { throw new InvalidParameterValueException("Unable to upgrade virtual machine " + vmInstance.toString() + ", cannot switch between local storage and shared storage service offerings. Current offering " + "useLocalStorage=" + currentServiceOffering.getUseLocalStorage() + ", target offering useLocalStorage=" + newServiceOffering.getUseLocalStorage()); } // if vm is a system vm, check if it is a system service offering, if yes return with error as it cannot be used for user vms if (currentServiceOffering.getSystemUse() != newServiceOffering.getSystemUse()) { throw new InvalidParameterValueException( "isSystem property is different for current service offering and new service offering"); } // Check that there are enough resources to upgrade the service offering if (!isVirtualMachineUpgradable(vmInstance, newServiceOffering)) { throw new InvalidParameterValueException( "Unable to upgrade virtual machine, not enough resources available " + "for an offering of " + newServiceOffering.getCpu() + " cpu(s) at " + newServiceOffering.getSpeed() + " Mhz, and " + newServiceOffering.getRamSize() + " MB of memory"); } // Check that the service offering being upgraded to has all the tags of the current service offering. final List<String> currentTags = StringUtils.csvTagsToList(currentServiceOffering.getTags()); final List<String> newTags = StringUtils.csvTagsToList(newServiceOffering.getTags()); if (!newTags.containsAll(currentTags)) { throw new InvalidParameterValueException( "Unable to upgrade virtual machine; the current service offering " + " should have tags as subset of " + "the new service offering tags. Current service offering tags: " + currentTags + "; " + "new service " + "offering tags: " + newTags); } } @Override public boolean upgradeVmDb(final long vmId, final long serviceOfferingId) { final VMInstanceVO vmForUpdate = _vmDao.createForUpdate(); vmForUpdate.setServiceOfferingId(serviceOfferingId); final ServiceOffering newSvcOff = _entityMgr.findById(ServiceOffering.class, serviceOfferingId); vmForUpdate.setHaEnabled(newSvcOff.getOfferHA()); vmForUpdate.setLimitCpuUse(newSvcOff.getLimitCpuUse()); vmForUpdate.setServiceOfferingId(newSvcOff.getId()); return _vmDao.update(vmId, vmForUpdate); } @Override public NicProfile addVmToNetwork(final VirtualMachine vm, final Network network, final NicProfile requested) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { // avoid re-entrance VmWorkJobVO placeHolder = null; placeHolder = createPlaceHolderWork(vm.getId()); try { return orchestrateAddVmToNetwork(vm, network, requested); } finally { if (placeHolder != null) { _workJobDao.expunge(placeHolder.getId()); } } } else { final Outcome<VirtualMachine> outcome = addVmToNetworkThroughJobQueue(vm, network, requested); try { outcome.get(); } catch (final InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (final java.util.concurrent.ExecutionException e) { throw new RuntimeException("Execution exception", e); } final Object jobException = _jobMgr.unmarshallResultObject(outcome.getJob()); if (jobException != null) { if (jobException instanceof ResourceUnavailableException) { throw (ResourceUnavailableException) jobException; } else if (jobException instanceof ConcurrentOperationException) { throw (ConcurrentOperationException) jobException; } else if (jobException instanceof InsufficientCapacityException) { throw (InsufficientCapacityException) jobException; } else if (jobException instanceof RuntimeException) { throw (RuntimeException) jobException; } else if (jobException instanceof Throwable) { throw new RuntimeException("Unexpected exception", (Throwable) jobException); } else if (jobException instanceof NicProfile) { return (NicProfile) jobException; } } throw new RuntimeException("Unexpected job execution result"); } } private NicProfile orchestrateAddVmToNetwork(final VirtualMachine vm, final Network network, final NicProfile requested) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { final CallContext cctx = CallContext.current(); s_logger.debug("Adding vm " + vm + " to network " + network + "; requested nic profile " + requested); final VMInstanceVO vmVO = _vmDao.findById(vm.getId()); final ReservationContext context = new ReservationContextImpl(null, null, cctx.getCallingUser(), cctx.getCallingAccount()); final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vmVO, null, null, null, null); final DataCenter dc = _entityMgr.findById(DataCenter.class, network.getDataCenterId()); final Host host = _hostDao.findById(vm.getHostId()); final DeployDestination dest = new DeployDestination(dc, null, null, host); //check vm state if (vm.getState() == State.Running) { //1) allocate and prepare nic final NicProfile nic = _networkMgr.createNicForVm(network, requested, context, vmProfile, true); //2) Convert vmProfile to vmTO final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vmProfile.getVirtualMachine().getHypervisorType()); final VirtualMachineTO vmTO = hvGuru.implement(vmProfile); //3) Convert nicProfile to NicTO final NicTO nicTO = toNicTO(nic, vmProfile.getVirtualMachine().getHypervisorType()); //4) plug the nic to the vm s_logger.debug("Plugging nic for vm " + vm + " in network " + network); boolean result = false; try { result = plugNic(network, nicTO, vmTO, context, dest); if (result) { s_logger.debug("Nic is plugged successfully for vm " + vm + " in network " + network + ". Vm is a part of network now"); final long isDefault = nic.isDefaultNic() ? 1 : 0; // insert nic's Id into DB as resource_name if (VirtualMachine.Type.User.equals(vmVO.getType())) { //Log usage event for user Vms only UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vmVO.getAccountId(), vmVO.getDataCenterId(), vmVO.getId(), Long.toString(nic.getId()), network.getNetworkOfferingId(), null, isDefault, VirtualMachine.class.getName(), vmVO.getUuid(), vm.isDisplay()); } return nic; } else { s_logger.warn("Failed to plug nic to the vm " + vm + " in network " + network); return null; } } finally { if (!result) { s_logger.debug("Removing nic " + nic + " from vm " + vmProfile.getVirtualMachine() + " as nic plug failed on the backend"); _networkMgr.removeNic(vmProfile, _nicsDao.findById(nic.getId())); } } } else if (vm.getState() == State.Stopped) { //1) allocate nic return _networkMgr.createNicForVm(network, requested, context, vmProfile, false); } else { s_logger.warn("Unable to add vm " + vm + " to network " + network); throw new ResourceUnavailableException( "Unable to add vm " + vm + " to network, is not in the right state", DataCenter.class, vm.getDataCenterId()); } } @Override public NicTO toNicTO(final NicProfile nic, final HypervisorType hypervisorType) { final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(hypervisorType); final NicTO nicTO = hvGuru.toNicTO(nic); return nicTO; } @Override public boolean removeNicFromVm(final VirtualMachine vm, final Nic nic) throws ConcurrentOperationException, ResourceUnavailableException { final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { // avoid re-entrance VmWorkJobVO placeHolder = null; placeHolder = createPlaceHolderWork(vm.getId()); try { return orchestrateRemoveNicFromVm(vm, nic); } finally { if (placeHolder != null) { _workJobDao.expunge(placeHolder.getId()); } } } else { final Outcome<VirtualMachine> outcome = removeNicFromVmThroughJobQueue(vm, nic); try { outcome.get(); } catch (final InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (final java.util.concurrent.ExecutionException e) { throw new RuntimeException("Execution excetion", e); } final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); if (jobResult != null) { if (jobResult instanceof ResourceUnavailableException) { throw (ResourceUnavailableException) jobResult; } else if (jobResult instanceof ConcurrentOperationException) { throw (ConcurrentOperationException) jobResult; } else if (jobResult instanceof RuntimeException) { throw (RuntimeException) jobResult; } else if (jobResult instanceof Throwable) { throw new RuntimeException("Unexpected exception", (Throwable) jobResult); } else if (jobResult instanceof Boolean) { return (Boolean) jobResult; } } throw new RuntimeException("Job failed with un-handled exception"); } } private boolean orchestrateRemoveNicFromVm(final VirtualMachine vm, final Nic nic) throws ConcurrentOperationException, ResourceUnavailableException { final CallContext cctx = CallContext.current(); final VMInstanceVO vmVO = _vmDao.findById(vm.getId()); final NetworkVO network = _networkDao.findById(nic.getNetworkId()); final ReservationContext context = new ReservationContextImpl(null, null, cctx.getCallingUser(), cctx.getCallingAccount()); final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vmVO, null, null, null, null); final DataCenter dc = _entityMgr.findById(DataCenter.class, network.getDataCenterId()); final Host host = _hostDao.findById(vm.getHostId()); final DeployDestination dest = new DeployDestination(dc, null, null, host); final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vmProfile.getVirtualMachine().getHypervisorType()); final VirtualMachineTO vmTO = hvGuru.implement(vmProfile); final NicProfile nicProfile = new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), _networkModel.getNetworkRate(network.getId(), vm.getId()), _networkModel.isSecurityGroupSupportedInNetwork(network), _networkModel.getNetworkTag(vmProfile.getVirtualMachine().getHypervisorType(), network)); //1) Unplug the nic if (vm.getState() == State.Running) { final NicTO nicTO = toNicTO(nicProfile, vmProfile.getVirtualMachine().getHypervisorType()); s_logger.debug("Un-plugging nic " + nic + " for vm " + vm + " from network " + network); final boolean result = unplugNic(network, nicTO, vmTO, context, dest); if (result) { s_logger.debug("Nic is unplugged successfully for vm " + vm + " in network " + network); final long isDefault = nic.isDefaultNic() ? 1 : 0; UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), Long.toString(nic.getId()), network.getNetworkOfferingId(), null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplay()); } else { s_logger.warn("Failed to unplug nic for the vm " + vm + " from network " + network); return false; } } else if (vm.getState() != State.Stopped) { s_logger.warn("Unable to remove vm " + vm + " from network " + network); throw new ResourceUnavailableException( "Unable to remove vm " + vm + " from network, is not in the right state", DataCenter.class, vm.getDataCenterId()); } //2) Release the nic _networkMgr.releaseNic(vmProfile, nic); s_logger.debug("Successfully released nic " + nic + "for vm " + vm); //3) Remove the nic _networkMgr.removeNic(vmProfile, nic); _nicsDao.expunge(nic.getId()); return true; } @Override @DB public boolean removeVmFromNetwork(final VirtualMachine vm, final Network network, final URI broadcastUri) throws ConcurrentOperationException, ResourceUnavailableException { // TODO will serialize on the VM object later to resolve operation conflicts return orchestrateRemoveVmFromNetwork(vm, network, broadcastUri); } @DB private boolean orchestrateRemoveVmFromNetwork(final VirtualMachine vm, final Network network, final URI broadcastUri) throws ConcurrentOperationException, ResourceUnavailableException { final CallContext cctx = CallContext.current(); final VMInstanceVO vmVO = _vmDao.findById(vm.getId()); final ReservationContext context = new ReservationContextImpl(null, null, cctx.getCallingUser(), cctx.getCallingAccount()); final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vmVO, null, null, null, null); final DataCenter dc = _entityMgr.findById(DataCenter.class, network.getDataCenterId()); final Host host = _hostDao.findById(vm.getHostId()); final DeployDestination dest = new DeployDestination(dc, null, null, host); final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vmProfile.getVirtualMachine().getHypervisorType()); final VirtualMachineTO vmTO = hvGuru.implement(vmProfile); Nic nic = null; if (broadcastUri != null) { nic = _nicsDao.findByNetworkIdInstanceIdAndBroadcastUri(network.getId(), vm.getId(), broadcastUri.toString()); } else { nic = _networkModel.getNicInNetwork(vm.getId(), network.getId()); } if (nic == null) { s_logger.warn("Could not get a nic with " + network); return false; } // don't delete default NIC on a user VM if (nic.isDefaultNic() && vm.getType() == VirtualMachine.Type.User) { s_logger.warn("Failed to remove nic from " + vm + " in " + network + ", nic is default."); throw new CloudRuntimeException( "Failed to remove nic from " + vm + " in " + network + ", nic is default."); } //Lock on nic is needed here final Nic lock = _nicsDao.acquireInLockTable(nic.getId()); if (lock == null) { //check if nic is still there. Return if it was released already if (_nicsDao.findById(nic.getId()) == null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Not need to remove the vm " + vm + " from network " + network + " as the vm doesn't have nic in this network"); } return true; } throw new ConcurrentOperationException("Unable to lock nic " + nic.getId()); } if (s_logger.isDebugEnabled()) { s_logger.debug("Lock is acquired for nic id " + lock.getId() + " as a part of remove vm " + vm + " from network " + network); } try { final NicProfile nicProfile = new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), _networkModel.getNetworkRate(network.getId(), vm.getId()), _networkModel.isSecurityGroupSupportedInNetwork(network), _networkModel.getNetworkTag(vmProfile.getVirtualMachine().getHypervisorType(), network)); //1) Unplug the nic if (vm.getState() == State.Running) { final NicTO nicTO = toNicTO(nicProfile, vmProfile.getVirtualMachine().getHypervisorType()); s_logger.debug("Un-plugging nic for vm " + vm + " from network " + network); final boolean result = unplugNic(network, nicTO, vmTO, context, dest); if (result) { s_logger.debug("Nic is unplugged successfully for vm " + vm + " in network " + network); } else { s_logger.warn("Failed to unplug nic for the vm " + vm + " from network " + network); return false; } } else if (vm.getState() != State.Stopped) { s_logger.warn("Unable to remove vm " + vm + " from network " + network); throw new ResourceUnavailableException( "Unable to remove vm " + vm + " from network, is not in the right state", DataCenter.class, vm.getDataCenterId()); } //2) Release the nic _networkMgr.releaseNic(vmProfile, nic); s_logger.debug("Successfully released nic " + nic + "for vm " + vm); //3) Remove the nic _networkMgr.removeNic(vmProfile, nic); return true; } finally { if (lock != null) { _nicsDao.releaseFromLockTable(lock.getId()); if (s_logger.isDebugEnabled()) { s_logger.debug("Lock is released for nic id " + lock.getId() + " as a part of remove vm " + vm + " from network " + network); } } } } @Override public void findHostAndMigrate(final String vmUuid, final Long newSvcOfferingId, final ExcludeList excludes) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); if (vm == null) { throw new CloudRuntimeException("Unable to find " + vmUuid); } final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); final Long srcHostId = vm.getHostId(); final Long oldSvcOfferingId = vm.getServiceOfferingId(); if (srcHostId == null) { throw new CloudRuntimeException("Unable to scale the vm because it doesn't have a host id"); } final Host host = _hostDao.findById(srcHostId); final DataCenterDeployment plan = new DataCenterDeployment(host.getDataCenterId(), host.getPodId(), host.getClusterId(), null, null, null); excludes.addHost(vm.getHostId()); vm.setServiceOfferingId(newSvcOfferingId); // Need to find the destination host based on new svc offering DeployDestination dest = null; try { dest = _dpMgr.planDeployment(profile, plan, excludes, null); } catch (final AffinityConflictException e2) { s_logger.warn("Unable to create deployment, affinity rules associted to the VM conflict", e2); throw new CloudRuntimeException( "Unable to create deployment, affinity rules associted to the VM conflict"); } if (dest != null) { if (s_logger.isDebugEnabled()) { s_logger.debug(" Found " + dest + " for scaling the vm to."); } } if (dest == null) { throw new InsufficientServerCapacityException("Unable to find a server to scale the vm to.", host.getClusterId()); } excludes.addHost(dest.getHost().getId()); try { migrateForScale(vm.getUuid(), srcHostId, dest, oldSvcOfferingId); } catch (final ResourceUnavailableException e) { s_logger.debug("Unable to migrate to unavailable " + dest); throw e; } catch (final ConcurrentOperationException e) { s_logger.debug("Unable to migrate VM due to: " + e.getMessage()); throw e; } } @Override public void migrateForScale(final String vmUuid, final long srcHostId, final DeployDestination dest, final Long oldSvcOfferingId) throws ResourceUnavailableException, ConcurrentOperationException { final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { // avoid re-entrance VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); placeHolder = createPlaceHolderWork(vm.getId()); try { orchestrateMigrateForScale(vmUuid, srcHostId, dest, oldSvcOfferingId); } finally { if (placeHolder != null) { _workJobDao.expunge(placeHolder.getId()); } } } else { final Outcome<VirtualMachine> outcome = migrateVmForScaleThroughJobQueue(vmUuid, srcHostId, dest, oldSvcOfferingId); try { final VirtualMachine vm = outcome.get(); } catch (final InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (final java.util.concurrent.ExecutionException e) { throw new RuntimeException("Execution excetion", e); } final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); if (jobResult != null) { if (jobResult instanceof ResourceUnavailableException) { throw (ResourceUnavailableException) jobResult; } else if (jobResult instanceof ConcurrentOperationException) { throw (ConcurrentOperationException) jobResult; } else if (jobResult instanceof RuntimeException) { throw (RuntimeException) jobResult; } else if (jobResult instanceof Throwable) { throw new RuntimeException("Unexpected exception", (Throwable) jobResult); } } } } private void orchestrateMigrateForScale(final String vmUuid, final long srcHostId, final DeployDestination dest, final Long oldSvcOfferingId) throws ResourceUnavailableException, ConcurrentOperationException { VMInstanceVO vm = _vmDao.findByUuid(vmUuid); s_logger.info("Migrating " + vm + " to " + dest); vm.getServiceOfferingId(); final long dstHostId = dest.getHost().getId(); final Host fromHost = _hostDao.findById(srcHostId); if (fromHost == null) { s_logger.info("Unable to find the host to migrate from: " + srcHostId); throw new CloudRuntimeException("Unable to find the host to migrate from: " + srcHostId); } if (fromHost.getClusterId().longValue() != dest.getCluster().getId()) { s_logger.info( "Source and destination host are not in same cluster, unable to migrate to host: " + dstHostId); throw new CloudRuntimeException( "Source and destination host are not in same cluster, unable to migrate to host: " + dest.getHost().getId()); } final VirtualMachineGuru vmGuru = getVmGuru(vm); final long vmId = vm.getId(); vm = _vmDao.findByUuid(vmUuid); if (vm == null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Unable to find the vm " + vm); } throw new CloudRuntimeException("Unable to find a virtual machine with id " + vmId); } if (vm.getState() != State.Running) { if (s_logger.isDebugEnabled()) { s_logger.debug("VM is not Running, unable to migrate the vm " + vm); } throw new CloudRuntimeException("VM is not Running, unable to migrate the vm currently " + vm + " , current state: " + vm.getState().toString()); } AlertManager.AlertType alertType = AlertManager.AlertType.ALERT_TYPE_USERVM_MIGRATE; if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) { alertType = AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER_MIGRATE; } else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { alertType = AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY_MIGRATE; } final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); _networkMgr.prepareNicForMigration(profile, dest); volumeMgr.prepareForMigration(profile, dest); final VirtualMachineTO to = toVmTO(profile); final PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(to); ItWorkVO work = new ItWorkVO(UUID.randomUUID().toString(), _nodeId, State.Migrating, vm.getType(), vm.getId()); work.setStep(Step.Prepare); work.setResourceType(ItWorkVO.ResourceType.Host); work.setResourceId(dstHostId); work = _workDao.persist(work); Answer pfma = null; try { pfma = _agentMgr.send(dstHostId, pfmc); if (pfma == null || !pfma.getResult()) { final String details = pfma != null ? pfma.getDetails() : "null answer returned"; final String msg = "Unable to prepare for migration due to " + details; pfma = null; throw new AgentUnavailableException(msg, dstHostId); } } catch (final OperationTimedoutException e1) { throw new AgentUnavailableException("Operation timed out", dstHostId); } finally { if (pfma == null) { work.setStep(Step.Done); _workDao.update(work.getId(), work); } } vm.setLastHostId(srcHostId); try { if (vm == null || vm.getHostId() == null || vm.getHostId() != srcHostId || !changeState(vm, Event.MigrationRequested, dstHostId, work, Step.Migrating)) { s_logger.info("Migration cancelled because state has changed: " + vm); throw new ConcurrentOperationException("Migration cancelled because state has changed: " + vm); } } catch (final NoTransitionException e1) { s_logger.info("Migration cancelled because " + e1.getMessage()); throw new ConcurrentOperationException("Migration cancelled because " + e1.getMessage()); } boolean migrated = false; try { final boolean isWindows = _guestOsCategoryDao .findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName() .equalsIgnoreCase("Windows"); final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType())); String autoConvergence = _configDao.getValue(Config.KvmAutoConvergence.toString()); boolean kvmAutoConvergence = Boolean.parseBoolean(autoConvergence); mc.setAutoConvergence(kvmAutoConvergence); mc.setHostGuid(dest.getHost().getGuid()); try { final Answer ma = _agentMgr.send(vm.getLastHostId(), mc); if (ma == null || !ma.getResult()) { final String details = ma != null ? ma.getDetails() : "null answer returned"; final String msg = "Unable to migrate due to " + details; s_logger.error(msg); throw new CloudRuntimeException(msg); } } catch (final OperationTimedoutException e) { if (e.isActive()) { s_logger.warn("Active migration command so scheduling a restart for " + vm); _haMgr.scheduleRestart(vm, true); } throw new AgentUnavailableException("Operation timed out on migrating " + vm, dstHostId); } try { final long newServiceOfferingId = vm.getServiceOfferingId(); vm.setServiceOfferingId(oldSvcOfferingId); // release capacity for the old service offering only if (!changeState(vm, VirtualMachine.Event.OperationSucceeded, dstHostId, work, Step.Started)) { throw new ConcurrentOperationException("Unable to change the state for " + vm); } vm.setServiceOfferingId(newServiceOfferingId); } catch (final NoTransitionException e1) { throw new ConcurrentOperationException("Unable to change state due to " + e1.getMessage()); } try { if (!checkVmOnHost(vm, dstHostId)) { s_logger.error("Unable to complete migration for " + vm); try { _agentMgr.send(srcHostId, new Commands(cleanup(vm.getInstanceName())), null); } catch (final AgentUnavailableException e) { s_logger.error("AgentUnavailableException while cleanup on source host: " + srcHostId); } cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.AgentReportStopped, true); throw new CloudRuntimeException("Unable to complete migration for " + vm); } } catch (final OperationTimedoutException e) { s_logger.debug("Error while checking the vm " + vm + " on host " + dstHostId, e); } migrated = true; } finally { if (!migrated) { s_logger.info("Migration was unsuccessful. Cleaning up: " + vm); _alertMgr.sendAlert(alertType, fromHost.getDataCenterId(), fromHost.getPodId(), "Unable to migrate vm " + vm.getInstanceName() + " from host " + fromHost.getName() + " in zone " + dest.getDataCenter().getName() + " and pod " + dest.getPod().getName(), "Migrate Command failed. Please check logs."); try { _agentMgr.send(dstHostId, new Commands(cleanup(vm.getInstanceName())), null); } catch (final AgentUnavailableException ae) { s_logger.info("Looks like the destination Host is unavailable for cleanup"); } try { stateTransitTo(vm, Event.OperationFailed, srcHostId); } catch (final NoTransitionException e) { s_logger.warn(e.getMessage()); } } work.setStep(Step.Done); _workDao.update(work.getId(), work); } } @Override public boolean replugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final ReservationContext context, final DeployDestination dest) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { boolean result = true; final VMInstanceVO router = _vmDao.findById(vm.getId()); if (router.getState() == State.Running) { try { final ReplugNicCommand replugNicCmd = new ReplugNicCommand(nic, vm.getName(), vm.getType(), vm.getDetails()); final Commands cmds = new Commands(Command.OnError.Stop); cmds.addCommand("replugnic", replugNicCmd); _agentMgr.send(dest.getHost().getId(), cmds); final ReplugNicAnswer replugNicAnswer = cmds.getAnswer(ReplugNicAnswer.class); if (replugNicAnswer == null || !replugNicAnswer.getResult()) { s_logger.warn("Unable to replug nic for vm " + vm.getName()); result = false; } } catch (final OperationTimedoutException e) { throw new AgentUnavailableException( "Unable to plug nic for router " + vm.getName() + " in network " + network, dest.getHost().getId(), e); } } else { s_logger.warn( "Unable to apply ReplugNic, vm " + router + " is not in the right state " + router.getState()); throw new ResourceUnavailableException( "Unable to apply ReplugNic on the backend," + " vm " + vm + " is not in the right state", DataCenter.class, router.getDataCenterId()); } return result; } public boolean plugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final ReservationContext context, final DeployDestination dest) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { boolean result = true; final VMInstanceVO router = _vmDao.findById(vm.getId()); if (router.getState() == State.Running) { try { final PlugNicCommand plugNicCmd = new PlugNicCommand(nic, vm.getName(), vm.getType(), vm.getDetails()); final Commands cmds = new Commands(Command.OnError.Stop); cmds.addCommand("plugnic", plugNicCmd); _agentMgr.send(dest.getHost().getId(), cmds); final PlugNicAnswer plugNicAnswer = cmds.getAnswer(PlugNicAnswer.class); if (plugNicAnswer == null || !plugNicAnswer.getResult()) { s_logger.warn("Unable to plug nic for vm " + vm.getName()); result = false; } } catch (final OperationTimedoutException e) { throw new AgentUnavailableException( "Unable to plug nic for router " + vm.getName() + " in network " + network, dest.getHost().getId(), e); } } else { s_logger.warn( "Unable to apply PlugNic, vm " + router + " is not in the right state " + router.getState()); throw new ResourceUnavailableException( "Unable to apply PlugNic on the backend," + " vm " + vm + " is not in the right state", DataCenter.class, router.getDataCenterId()); } return result; } public boolean unplugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final ReservationContext context, final DeployDestination dest) throws ConcurrentOperationException, ResourceUnavailableException { boolean result = true; final VMInstanceVO router = _vmDao.findById(vm.getId()); if (router.getState() == State.Running) { // collect vm network statistics before unplug a nic UserVmVO userVm = _userVmDao.findById(vm.getId()); if (userVm != null && userVm.getType() == VirtualMachine.Type.User) { _userVmService.collectVmNetworkStatistics(userVm); } try { final Commands cmds = new Commands(Command.OnError.Stop); final UnPlugNicCommand unplugNicCmd = new UnPlugNicCommand(nic, vm.getName()); cmds.addCommand("unplugnic", unplugNicCmd); _agentMgr.send(dest.getHost().getId(), cmds); final UnPlugNicAnswer unplugNicAnswer = cmds.getAnswer(UnPlugNicAnswer.class); if (unplugNicAnswer == null || !unplugNicAnswer.getResult()) { s_logger.warn("Unable to unplug nic from router " + router); result = false; } } catch (final OperationTimedoutException e) { throw new AgentUnavailableException( "Unable to unplug nic from rotuer " + router + " from network " + network, dest.getHost().getId(), e); } } else if (router.getState() == State.Stopped || router.getState() == State.Stopping) { s_logger.debug("Vm " + router.getInstanceName() + " is in " + router.getState() + ", so not sending unplug nic command to the backend"); } else { s_logger.warn( "Unable to apply unplug nic, Vm " + router + " is not in the right state " + router.getState()); throw new ResourceUnavailableException( "Unable to apply unplug nic on the backend," + " vm " + router + " is not in the right state", DataCenter.class, router.getDataCenterId()); } return result; } @Override public VMInstanceVO reConfigureVm(final String vmUuid, final ServiceOffering oldServiceOffering, final boolean reconfiguringOnExistingHost) throws ResourceUnavailableException, InsufficientServerCapacityException, ConcurrentOperationException { final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext(); if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) { // avoid re-entrance VmWorkJobVO placeHolder = null; final VirtualMachine vm = _vmDao.findByUuid(vmUuid); placeHolder = createPlaceHolderWork(vm.getId()); try { return orchestrateReConfigureVm(vmUuid, oldServiceOffering, reconfiguringOnExistingHost); } finally { if (placeHolder != null) { _workJobDao.expunge(placeHolder.getId()); } } } else { final Outcome<VirtualMachine> outcome = reconfigureVmThroughJobQueue(vmUuid, oldServiceOffering, reconfiguringOnExistingHost); VirtualMachine vm = null; try { vm = outcome.get(); } catch (final InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (final java.util.concurrent.ExecutionException e) { throw new RuntimeException("Execution excetion", e); } final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); if (jobResult != null) { if (jobResult instanceof ResourceUnavailableException) { throw (ResourceUnavailableException) jobResult; } else if (jobResult instanceof ConcurrentOperationException) { throw (ConcurrentOperationException) jobResult; } else if (jobResult instanceof InsufficientServerCapacityException) { throw (InsufficientServerCapacityException) jobResult; } else if (jobResult instanceof Throwable) { s_logger.error("Unhandled exception", (Throwable) jobResult); throw new RuntimeException("Unhandled exception", (Throwable) jobResult); } } return (VMInstanceVO) vm; } } private VMInstanceVO orchestrateReConfigureVm(final String vmUuid, final ServiceOffering oldServiceOffering, final boolean reconfiguringOnExistingHost) throws ResourceUnavailableException, ConcurrentOperationException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); final long newServiceofferingId = vm.getServiceOfferingId(); final ServiceOffering newServiceOffering = _offeringDao.findById(vm.getId(), newServiceofferingId); final HostVO hostVo = _hostDao.findById(vm.getHostId()); final Float memoryOvercommitRatio = CapacityManager.MemOverprovisioningFactor .valueIn(hostVo.getClusterId()); final Float cpuOvercommitRatio = CapacityManager.CpuOverprovisioningFactor.valueIn(hostVo.getClusterId()); final long minMemory = (long) (newServiceOffering.getRamSize() / memoryOvercommitRatio); final ScaleVmCommand reconfigureCmd = new ScaleVmCommand(vm.getInstanceName(), newServiceOffering.getCpu(), (int) (newServiceOffering.getSpeed() / cpuOvercommitRatio), newServiceOffering.getSpeed(), minMemory * 1024L * 1024L, newServiceOffering.getRamSize() * 1024L * 1024L, newServiceOffering.getLimitCpuUse()); final Long dstHostId = vm.getHostId(); if (vm.getHypervisorType().equals(HypervisorType.VMware)) { final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType()); Map<String, String> details = null; details = hvGuru.getClusterSettings(vm.getId()); reconfigureCmd.getVirtualMachine().setDetails(details); } final ItWorkVO work = new ItWorkVO(UUID.randomUUID().toString(), _nodeId, State.Running, vm.getType(), vm.getId()); work.setStep(Step.Prepare); work.setResourceType(ItWorkVO.ResourceType.Host); work.setResourceId(vm.getHostId()); _workDao.persist(work); boolean success = false; try { if (reconfiguringOnExistingHost) { vm.setServiceOfferingId(oldServiceOffering.getId()); _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId()); //release the old capacity vm.setServiceOfferingId(newServiceofferingId); _capacityMgr.allocateVmCapacity(vm, false); // lock the new capacity } final Answer reconfigureAnswer = _agentMgr.send(vm.getHostId(), reconfigureCmd); if (reconfigureAnswer == null || !reconfigureAnswer.getResult()) { s_logger.error("Unable to scale vm due to " + (reconfigureAnswer == null ? "" : reconfigureAnswer.getDetails())); throw new CloudRuntimeException("Unable to scale vm due to " + (reconfigureAnswer == null ? "" : reconfigureAnswer.getDetails())); } success = true; } catch (final OperationTimedoutException e) { throw new AgentUnavailableException("Operation timed out on reconfiguring " + vm, dstHostId); } catch (final AgentUnavailableException e) { throw e; } finally { if (!success) { _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId()); // release the new capacity vm.setServiceOfferingId(oldServiceOffering.getId()); _capacityMgr.allocateVmCapacity(vm, false); // allocate the old capacity } } return vm; } @Override public String getConfigComponentName() { return VirtualMachineManager.class.getSimpleName(); } @Override public ConfigKey<?>[] getConfigKeys() { return new ConfigKey<?>[] { ClusterDeltaSyncInterval, StartRetry, VmDestroyForcestop, VmOpCancelInterval, VmOpCleanupInterval, VmOpCleanupWait, VmOpLockStateRetry, VmOpWaitInterval, ExecuteInSequence, VmJobCheckInterval, VmJobTimeout, VmJobStateReportInterval, VmConfigDriveLabel }; } public List<StoragePoolAllocator> getStoragePoolAllocators() { return _storagePoolAllocators; } @Inject public void setStoragePoolAllocators(final List<StoragePoolAllocator> storagePoolAllocators) { _storagePoolAllocators = storagePoolAllocators; } // // PowerState report handling for out-of-band changes and handling of left-over transitional VM states // @MessageHandler(topic = Topics.VM_POWER_STATE) protected void HandlePowerStateReport(final String subject, final String senderAddress, final Object args) { assert args != null; final Long vmId = (Long) args; final List<VmWorkJobVO> pendingWorkJobs = _workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vmId); if (pendingWorkJobs.size() == 0 && !_haMgr.hasPendingHaWork(vmId)) { // there is no pending operation job final VMInstanceVO vm = _vmDao.findById(vmId); if (vm != null) { switch (vm.getPowerState()) { case PowerOn: handlePowerOnReportWithNoPendingJobsOnVM(vm); break; case PowerOff: case PowerReportMissing: handlePowerOffReportWithNoPendingJobsOnVM(vm); break; // PowerUnknown shouldn't be reported, it is a derived // VM power state from host state (host un-reachable) case PowerUnknown: default: assert false; break; } } else { s_logger.warn("VM " + vmId + " no longer exists when processing VM state report"); } } else { s_logger.info("There is pending job or HA tasks working on the VM. vm id: " + vmId + ", postpone power-change report by resetting power-change counters"); // reset VM power state tracking so that we won't lost signal when VM has // been translated to _vmDao.resetVmPowerStateTracking(vmId); } } private void handlePowerOnReportWithNoPendingJobsOnVM(final VMInstanceVO vm) { // // 1) handle left-over transitional VM states // 2) handle out of band VM live migration // 3) handle out of sync stationary states, marking VM from Stopped to Running with // alert messages // switch (vm.getState()) { case Starting: s_logger.info("VM " + vm.getInstanceName() + " is at " + vm.getState() + " and we received a power-on report while there is no pending jobs on it"); try { stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOnReport, vm.getPowerHostId()); } catch (final NoTransitionException e) { s_logger.warn("Unexpected VM state transition exception, race-condition?", e); } s_logger.info("VM " + vm.getInstanceName() + " is sync-ed to at Running state according to power-on report from hypervisor"); // we need to alert admin or user about this risky state transition _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(), VM_SYNC_ALERT_SUBJECT, "VM " + vm.getHostName() + "(" + vm.getInstanceName() + ") state is sync-ed (Starting -> Running) from out-of-context transition. VM network environment may need to be reset"); break; case Running: try { if (vm.getHostId() != null && vm.getHostId().longValue() != vm.getPowerHostId().longValue()) { s_logger.info("Detected out of band VM migration from host " + vm.getHostId() + " to host " + vm.getPowerHostId()); } stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOnReport, vm.getPowerHostId()); } catch (final NoTransitionException e) { s_logger.warn("Unexpected VM state transition exception, race-condition?", e); } break; case Stopping: case Stopped: s_logger.info("VM " + vm.getInstanceName() + " is at " + vm.getState() + " and we received a power-on report while there is no pending jobs on it"); try { stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOnReport, vm.getPowerHostId()); } catch (final NoTransitionException e) { s_logger.warn("Unexpected VM state transition exception, race-condition?", e); } _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(), VM_SYNC_ALERT_SUBJECT, "VM " + vm.getHostName() + "(" + vm.getInstanceName() + ") state is sync-ed (" + vm.getState() + " -> Running) from out-of-context transition. VM network environment may need to be reset"); s_logger.info("VM " + vm.getInstanceName() + " is sync-ed to at Running state according to power-on report from hypervisor"); break; case Destroyed: case Expunging: s_logger.info("Receive power on report when VM is in destroyed or expunging state. vm: " + vm.getId() + ", state: " + vm.getState()); break; case Migrating: s_logger.info("VM " + vm.getInstanceName() + " is at " + vm.getState() + " and we received a power-on report while there is no pending jobs on it"); try { stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOnReport, vm.getPowerHostId()); } catch (final NoTransitionException e) { s_logger.warn("Unexpected VM state transition exception, race-condition?", e); } s_logger.info("VM " + vm.getInstanceName() + " is sync-ed to at Running state according to power-on report from hypervisor"); break; case Error: default: s_logger.info("Receive power on report when VM is in error or unexpected state. vm: " + vm.getId() + ", state: " + vm.getState()); break; } } private void handlePowerOffReportWithNoPendingJobsOnVM(final VMInstanceVO vm) { // 1) handle left-over transitional VM states // 2) handle out of sync stationary states, schedule force-stop to release resources // switch (vm.getState()) { case Starting: case Stopping: case Running: case Stopped: case Migrating: s_logger.info("VM " + vm.getInstanceName() + " is at " + vm.getState() + " and we received a power-off report while there is no pending jobs on it"); if (vm.isHaEnabled() && vm.getState() == State.Running && vm.getHypervisorType() != HypervisorType.VMware && vm.getHypervisorType() != HypervisorType.Hyperv) { s_logger.info("Detected out-of-band stop of a HA enabled VM " + vm.getInstanceName() + ", will schedule restart"); if (!_haMgr.hasPendingHaWork(vm.getId())) { _haMgr.scheduleRestart(vm, true); } else { s_logger.info("VM " + vm.getInstanceName() + " already has an pending HA task working on it"); } return; } final VirtualMachineGuru vmGuru = getVmGuru(vm); final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); if (!sendStop(vmGuru, profile, true, true)) { // In case StopCommand fails, don't proceed further return; } try { stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOffReport, null); } catch (final NoTransitionException e) { s_logger.warn("Unexpected VM state transition exception, race-condition?", e); } _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(), VM_SYNC_ALERT_SUBJECT, "VM " + vm.getHostName() + "(" + vm.getInstanceName() + ") state is sync-ed (" + vm.getState() + " -> Stopped) from out-of-context transition."); s_logger.info("VM " + vm.getInstanceName() + " is sync-ed to at Stopped state according to power-off report from hypervisor"); break; case Destroyed: case Expunging: break; case Error: default: break; } } private void scanStalledVMInTransitionStateOnUpHost(final long hostId) { // // Check VM that is stuck in Starting, Stopping, Migrating states, we won't check // VMs in expunging state (this need to be handled specially) // // checking condition // 1) no pending VmWork job // 2) on hostId host and host is UP // // When host is UP, soon or later we will get a report from the host about the VM, // however, if VM is missing from the host report (it may happen in out of band changes // or from designed behave of XS/KVM), the VM may not get a chance to run the state-sync logic // // Therefore, we will scan thoses VMs on UP host based on last update timestamp, if the host is UP // and a VM stalls for status update, we will consider them to be powered off // (which is relatively safe to do so) final long stallThresholdInMs = VmJobStateReportInterval.value() + (VmJobStateReportInterval.value() >> 1); final Date cutTime = new Date(DateUtil.currentGMTTime().getTime() - stallThresholdInMs); final List<Long> mostlikelyStoppedVMs = listStalledVMInTransitionStateOnUpHost(hostId, cutTime); for (final Long vmId : mostlikelyStoppedVMs) { final VMInstanceVO vm = _vmDao.findById(vmId); assert vm != null; handlePowerOffReportWithNoPendingJobsOnVM(vm); } final List<Long> vmsWithRecentReport = listVMInTransitionStateWithRecentReportOnUpHost(hostId, cutTime); for (final Long vmId : vmsWithRecentReport) { final VMInstanceVO vm = _vmDao.findById(vmId); assert vm != null; if (vm.getPowerState() == PowerState.PowerOn) { handlePowerOnReportWithNoPendingJobsOnVM(vm); } else { handlePowerOffReportWithNoPendingJobsOnVM(vm); } } } private void scanStalledVMInTransitionStateOnDisconnectedHosts() { final Date cutTime = new Date(DateUtil.currentGMTTime().getTime() - VmOpWaitInterval.value() * 1000); final List<Long> stuckAndUncontrollableVMs = listStalledVMInTransitionStateOnDisconnectedHosts(cutTime); for (final Long vmId : stuckAndUncontrollableVMs) { final VMInstanceVO vm = _vmDao.findById(vmId); // We now only alert administrator about this situation _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(), VM_SYNC_ALERT_SUBJECT, "VM " + vm.getHostName() + "(" + vm.getInstanceName() + ") is stuck in " + vm.getState() + " state and its host is unreachable for too long"); } } // VMs that in transitional state without recent power state report private List<Long> listStalledVMInTransitionStateOnUpHost(final long hostId, final Date cutTime) { final String sql = "SELECT i.* FROM vm_instance as i, host as h WHERE h.status = 'UP' " + "AND h.id = ? AND i.power_state_update_time < ? AND i.host_id = h.id " + "AND (i.state ='Starting' OR i.state='Stopping' OR i.state='Migrating') " + "AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)" + "AND i.removed IS NULL"; final List<Long> l = new ArrayList<Long>(); TransactionLegacy txn = null; try { txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB); PreparedStatement pstmt = null; try { pstmt = txn.prepareAutoCloseStatement(sql); pstmt.setLong(1, hostId); pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime)); pstmt.setInt(3, JobInfo.Status.IN_PROGRESS.ordinal()); final ResultSet rs = pstmt.executeQuery(); while (rs.next()) { l.add(rs.getLong(1)); } } catch (final SQLException e) { } catch (final Throwable e) { } } finally { if (txn != null) { txn.close(); } } return l; } // VMs that in transitional state and recently have power state update private List<Long> listVMInTransitionStateWithRecentReportOnUpHost(final long hostId, final Date cutTime) { final String sql = "SELECT i.* FROM vm_instance as i, host as h WHERE h.status = 'UP' " + "AND h.id = ? AND i.power_state_update_time > ? AND i.host_id = h.id " + "AND (i.state ='Starting' OR i.state='Stopping' OR i.state='Migrating') " + "AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)" + "AND i.removed IS NULL"; final List<Long> l = new ArrayList<Long>(); TransactionLegacy txn = null; try { txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB); PreparedStatement pstmt = null; try { pstmt = txn.prepareAutoCloseStatement(sql); pstmt.setLong(1, hostId); pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime)); pstmt.setInt(3, JobInfo.Status.IN_PROGRESS.ordinal()); final ResultSet rs = pstmt.executeQuery(); while (rs.next()) { l.add(rs.getLong(1)); } } catch (final SQLException e) { } catch (final Throwable e) { } return l; } finally { if (txn != null) { txn.close(); } } } private List<Long> listStalledVMInTransitionStateOnDisconnectedHosts(final Date cutTime) { final String sql = "SELECT i.* FROM vm_instance as i, host as h WHERE h.status != 'UP' " + "AND i.power_state_update_time < ? AND i.host_id = h.id " + "AND (i.state ='Starting' OR i.state='Stopping' OR i.state='Migrating') " + "AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)" + "AND i.removed IS NULL"; final List<Long> l = new ArrayList<Long>(); TransactionLegacy txn = null; try { txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB); PreparedStatement pstmt = null; try { pstmt = txn.prepareAutoCloseStatement(sql); pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime)); pstmt.setInt(2, JobInfo.Status.IN_PROGRESS.ordinal()); final ResultSet rs = pstmt.executeQuery(); while (rs.next()) { l.add(rs.getLong(1)); } } catch (final SQLException e) { } catch (final Throwable e) { } return l; } finally { if (txn != null) { txn.close(); } } } // // VM operation based on new sync model // public class VmStateSyncOutcome extends OutcomeImpl<VirtualMachine> { private long _vmId; public VmStateSyncOutcome(final AsyncJob job, final PowerState desiredPowerState, final long vmId, final Long srcHostIdForMigration) { super(VirtualMachine.class, job, VmJobCheckInterval.value(), new Predicate() { @Override public boolean checkCondition() { final AsyncJobVO jobVo = _entityMgr.findById(AsyncJobVO.class, job.getId()); assert jobVo != null; if (jobVo == null || jobVo.getStatus() != JobInfo.Status.IN_PROGRESS) { return true; } return false; } }, Topics.VM_POWER_STATE, AsyncJob.Topics.JOB_STATE); _vmId = vmId; } @Override protected VirtualMachine retrieve() { return _vmDao.findById(_vmId); } } public class VmJobVirtualMachineOutcome extends OutcomeImpl<VirtualMachine> { private long _vmId; public VmJobVirtualMachineOutcome(final AsyncJob job, final long vmId) { super(VirtualMachine.class, job, VmJobCheckInterval.value(), new Predicate() { @Override public boolean checkCondition() { final AsyncJobVO jobVo = _entityMgr.findById(AsyncJobVO.class, job.getId()); assert jobVo != null; if (jobVo == null || jobVo.getStatus() != JobInfo.Status.IN_PROGRESS) { return true; } return false; } }, AsyncJob.Topics.JOB_STATE); _vmId = vmId; } @Override protected VirtualMachine retrieve() { return _vmDao.findById(_vmId); } } // // TODO build a common pattern to reduce code duplication in following methods // no time for this at current iteration // public Outcome<VirtualMachine> startVmThroughJobQueue(final String vmUuid, final Map<VirtualMachineProfile.Param, Object> params, final DeploymentPlan planToDeploy, final DeploymentPlanner planner) { final CallContext context = CallContext.current(); final User callingUser = context.getCallingUser(); final Account callingAccount = context.getCallingAccount(); final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); VmWorkJobVO workJob = null; final List<VmWorkJobVO> pendingWorkJobs = _workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vm.getId(), VmWorkStart.class.getName()); if (pendingWorkJobs.size() > 0) { assert pendingWorkJobs.size() == 1; workJob = pendingWorkJobs.get(0); } else { workJob = new VmWorkJobVO(context.getContextId()); workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); workJob.setCmd(VmWorkStart.class.getName()); workJob.setAccountId(callingAccount.getId()); workJob.setUserId(callingUser.getId()); workJob.setStep(VmWorkJobVO.Step.Starting); workJob.setVmType(VirtualMachine.Type.Instance); workJob.setVmInstanceId(vm.getId()); workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); // save work context info (there are some duplications) final VmWorkStart workInfo = new VmWorkStart(callingUser.getId(), callingAccount.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER); workInfo.setPlan(planToDeploy); workInfo.setParams(params); if (planner != null) { workInfo.setDeploymentPlanner(planner.getName()); } workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmStateSyncOutcome(workJob, VirtualMachine.PowerState.PowerOn, vm.getId(), null); } public Outcome<VirtualMachine> stopVmThroughJobQueue(final String vmUuid, final boolean cleanup) { final CallContext context = CallContext.current(); final Account account = context.getCallingAccount(); final User user = context.getCallingUser(); final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); final List<VmWorkJobVO> pendingWorkJobs = _workJobDao.listPendingWorkJobs(vm.getType(), vm.getId(), VmWorkStop.class.getName()); VmWorkJobVO workJob = null; if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { assert pendingWorkJobs.size() == 1; workJob = pendingWorkJobs.get(0); } else { workJob = new VmWorkJobVO(context.getContextId()); workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); workJob.setCmd(VmWorkStop.class.getName()); workJob.setAccountId(account.getId()); workJob.setUserId(user.getId()); workJob.setStep(VmWorkJobVO.Step.Prepare); workJob.setVmType(VirtualMachine.Type.Instance); workJob.setVmInstanceId(vm.getId()); workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); // save work context info (there are some duplications) final VmWorkStop workInfo = new VmWorkStop(user.getId(), account.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, cleanup); workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmStateSyncOutcome(workJob, VirtualMachine.PowerState.PowerOff, vm.getId(), null); } public Outcome<VirtualMachine> rebootVmThroughJobQueue(final String vmUuid, final Map<VirtualMachineProfile.Param, Object> params) { final CallContext context = CallContext.current(); final Account account = context.getCallingAccount(); final User user = context.getCallingUser(); final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); final List<VmWorkJobVO> pendingWorkJobs = _workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vm.getId(), VmWorkReboot.class.getName()); VmWorkJobVO workJob = null; if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { assert pendingWorkJobs.size() == 1; workJob = pendingWorkJobs.get(0); } else { workJob = new VmWorkJobVO(context.getContextId()); workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); workJob.setCmd(VmWorkReboot.class.getName()); workJob.setAccountId(account.getId()); workJob.setUserId(user.getId()); workJob.setStep(VmWorkJobVO.Step.Prepare); workJob.setVmType(VirtualMachine.Type.Instance); workJob.setVmInstanceId(vm.getId()); workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); // save work context info (there are some duplications) final VmWorkReboot workInfo = new VmWorkReboot(user.getId(), account.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, params); workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmJobVirtualMachineOutcome(workJob, vm.getId()); } public Outcome<VirtualMachine> migrateVmThroughJobQueue(final String vmUuid, final long srcHostId, final DeployDestination dest) { final CallContext context = CallContext.current(); final User user = context.getCallingUser(); final Account account = context.getCallingAccount(); final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); final List<VmWorkJobVO> pendingWorkJobs = _workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vm.getId(), VmWorkMigrate.class.getName()); VmWorkJobVO workJob = null; if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { assert pendingWorkJobs.size() == 1; workJob = pendingWorkJobs.get(0); } else { workJob = new VmWorkJobVO(context.getContextId()); workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); workJob.setCmd(VmWorkMigrate.class.getName()); workJob.setAccountId(account.getId()); workJob.setUserId(user.getId()); workJob.setVmType(VirtualMachine.Type.Instance); workJob.setVmInstanceId(vm.getId()); workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); // save work context info (there are some duplications) final VmWorkMigrate workInfo = new VmWorkMigrate(user.getId(), account.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, srcHostId, dest); workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmStateSyncOutcome(workJob, VirtualMachine.PowerState.PowerOn, vm.getId(), vm.getPowerHostId()); } public Outcome<VirtualMachine> migrateVmAwayThroughJobQueue(final String vmUuid, final long srcHostId) { final CallContext context = CallContext.current(); final User user = context.getCallingUser(); final Account account = context.getCallingAccount(); final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); final List<VmWorkJobVO> pendingWorkJobs = _workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vm.getId(), VmWorkMigrateAway.class.getName()); VmWorkJobVO workJob = null; if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { assert pendingWorkJobs.size() == 1; workJob = pendingWorkJobs.get(0); } else { workJob = new VmWorkJobVO(context.getContextId()); workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); workJob.setCmd(VmWorkMigrateAway.class.getName()); workJob.setAccountId(account.getId()); workJob.setUserId(user.getId()); workJob.setVmType(VirtualMachine.Type.Instance); workJob.setVmInstanceId(vm.getId()); workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); // save work context info (there are some duplications) final VmWorkMigrateAway workInfo = new VmWorkMigrateAway(user.getId(), account.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, srcHostId); workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); } _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmStateSyncOutcome(workJob, VirtualMachine.PowerState.PowerOn, vm.getId(), vm.getPowerHostId()); } public Outcome<VirtualMachine> migrateVmWithStorageThroughJobQueue(final String vmUuid, final long srcHostId, final long destHostId, final Map<Long, Long> volumeToPool) { final CallContext context = CallContext.current(); final User user = context.getCallingUser(); final Account account = context.getCallingAccount(); final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); final List<VmWorkJobVO> pendingWorkJobs = _workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vm.getId(), VmWorkMigrateWithStorage.class.getName()); VmWorkJobVO workJob = null; if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { assert pendingWorkJobs.size() == 1; workJob = pendingWorkJobs.get(0); } else { workJob = new VmWorkJobVO(context.getContextId()); workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); workJob.setCmd(VmWorkMigrateWithStorage.class.getName()); workJob.setAccountId(account.getId()); workJob.setUserId(user.getId()); workJob.setVmType(VirtualMachine.Type.Instance); workJob.setVmInstanceId(vm.getId()); workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); // save work context info (there are some duplications) final VmWorkMigrateWithStorage workInfo = new VmWorkMigrateWithStorage(user.getId(), account.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, srcHostId, destHostId, volumeToPool); workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmStateSyncOutcome(workJob, VirtualMachine.PowerState.PowerOn, vm.getId(), destHostId); } public Outcome<VirtualMachine> migrateVmForScaleThroughJobQueue(final String vmUuid, final long srcHostId, final DeployDestination dest, final Long newSvcOfferingId) { final CallContext context = CallContext.current(); final User user = context.getCallingUser(); final Account account = context.getCallingAccount(); final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); final List<VmWorkJobVO> pendingWorkJobs = _workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vm.getId(), VmWorkMigrateForScale.class.getName()); VmWorkJobVO workJob = null; if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { assert pendingWorkJobs.size() == 1; workJob = pendingWorkJobs.get(0); } else { workJob = new VmWorkJobVO(context.getContextId()); workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); workJob.setCmd(VmWorkMigrateForScale.class.getName()); workJob.setAccountId(account.getId()); workJob.setUserId(user.getId()); workJob.setVmType(VirtualMachine.Type.Instance); workJob.setVmInstanceId(vm.getId()); workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); // save work context info (there are some duplications) final VmWorkMigrateForScale workInfo = new VmWorkMigrateForScale(user.getId(), account.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, srcHostId, dest, newSvcOfferingId); workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmJobVirtualMachineOutcome(workJob, vm.getId()); } public Outcome<VirtualMachine> migrateVmStorageThroughJobQueue(final String vmUuid, final StoragePool destPool) { final CallContext context = CallContext.current(); final User user = context.getCallingUser(); final Account account = context.getCallingAccount(); final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); final List<VmWorkJobVO> pendingWorkJobs = _workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vm.getId(), VmWorkStorageMigration.class.getName()); VmWorkJobVO workJob = null; if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { assert pendingWorkJobs.size() == 1; workJob = pendingWorkJobs.get(0); } else { workJob = new VmWorkJobVO(context.getContextId()); workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); workJob.setCmd(VmWorkStorageMigration.class.getName()); workJob.setAccountId(account.getId()); workJob.setUserId(user.getId()); workJob.setVmType(VirtualMachine.Type.Instance); workJob.setVmInstanceId(vm.getId()); workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); // save work context info (there are some duplications) final VmWorkStorageMigration workInfo = new VmWorkStorageMigration(user.getId(), account.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, destPool.getId()); workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmJobVirtualMachineOutcome(workJob, vm.getId()); } public Outcome<VirtualMachine> addVmToNetworkThroughJobQueue(final VirtualMachine vm, final Network network, final NicProfile requested) { final CallContext context = CallContext.current(); final User user = context.getCallingUser(); final Account account = context.getCallingAccount(); final List<VmWorkJobVO> pendingWorkJobs = _workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vm.getId(), VmWorkAddVmToNetwork.class.getName()); VmWorkJobVO workJob = null; if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { assert pendingWorkJobs.size() == 1; workJob = pendingWorkJobs.get(0); } else { workJob = new VmWorkJobVO(context.getContextId()); workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); workJob.setCmd(VmWorkAddVmToNetwork.class.getName()); workJob.setAccountId(account.getId()); workJob.setUserId(user.getId()); workJob.setVmType(VirtualMachine.Type.Instance); workJob.setVmInstanceId(vm.getId()); workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); // save work context info (there are some duplications) final VmWorkAddVmToNetwork workInfo = new VmWorkAddVmToNetwork(user.getId(), account.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, network.getId(), requested); workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmJobVirtualMachineOutcome(workJob, vm.getId()); } public Outcome<VirtualMachine> removeNicFromVmThroughJobQueue(final VirtualMachine vm, final Nic nic) { final CallContext context = CallContext.current(); final User user = context.getCallingUser(); final Account account = context.getCallingAccount(); final List<VmWorkJobVO> pendingWorkJobs = _workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vm.getId(), VmWorkRemoveNicFromVm.class.getName()); VmWorkJobVO workJob = null; if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { assert pendingWorkJobs.size() == 1; workJob = pendingWorkJobs.get(0); } else { workJob = new VmWorkJobVO(context.getContextId()); workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); workJob.setCmd(VmWorkRemoveNicFromVm.class.getName()); workJob.setAccountId(account.getId()); workJob.setUserId(user.getId()); workJob.setVmType(VirtualMachine.Type.Instance); workJob.setVmInstanceId(vm.getId()); workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); // save work context info (there are some duplications) final VmWorkRemoveNicFromVm workInfo = new VmWorkRemoveNicFromVm(user.getId(), account.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, nic.getId()); workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmJobVirtualMachineOutcome(workJob, vm.getId()); } public Outcome<VirtualMachine> removeVmFromNetworkThroughJobQueue(final VirtualMachine vm, final Network network, final URI broadcastUri) { final CallContext context = CallContext.current(); final User user = context.getCallingUser(); final Account account = context.getCallingAccount(); final List<VmWorkJobVO> pendingWorkJobs = _workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vm.getId(), VmWorkRemoveVmFromNetwork.class.getName()); VmWorkJobVO workJob = null; if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { assert pendingWorkJobs.size() == 1; workJob = pendingWorkJobs.get(0); } else { workJob = new VmWorkJobVO(context.getContextId()); workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); workJob.setCmd(VmWorkRemoveVmFromNetwork.class.getName()); workJob.setAccountId(account.getId()); workJob.setUserId(user.getId()); workJob.setVmType(VirtualMachine.Type.Instance); workJob.setVmInstanceId(vm.getId()); workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); // save work context info (there are some duplications) final VmWorkRemoveVmFromNetwork workInfo = new VmWorkRemoveVmFromNetwork(user.getId(), account.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, network, broadcastUri); workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmJobVirtualMachineOutcome(workJob, vm.getId()); } public Outcome<VirtualMachine> reconfigureVmThroughJobQueue(final String vmUuid, final ServiceOffering newServiceOffering, final boolean reconfiguringOnExistingHost) { final CallContext context = CallContext.current(); final User user = context.getCallingUser(); final Account account = context.getCallingAccount(); final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); final List<VmWorkJobVO> pendingWorkJobs = _workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vm.getId(), VmWorkReconfigure.class.getName()); VmWorkJobVO workJob = null; if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { assert pendingWorkJobs.size() == 1; workJob = pendingWorkJobs.get(0); } else { workJob = new VmWorkJobVO(context.getContextId()); workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); workJob.setCmd(VmWorkReconfigure.class.getName()); workJob.setAccountId(account.getId()); workJob.setUserId(user.getId()); workJob.setVmType(VirtualMachine.Type.Instance); workJob.setVmInstanceId(vm.getId()); workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); // save work context info (there are some duplications) final VmWorkReconfigure workInfo = new VmWorkReconfigure(user.getId(), account.getId(), vm.getId(), VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, newServiceOffering.getId(), reconfiguringOnExistingHost); workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); } AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); return new VmJobVirtualMachineOutcome(workJob, vm.getId()); } @ReflectionUse private Pair<JobInfo.Status, String> orchestrateStart(final VmWorkStart work) throws Exception { final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); if (vm == null) { s_logger.info("Unable to find vm " + work.getVmId()); } assert vm != null; try { orchestrateStart(vm.getUuid(), work.getParams(), work.getPlan(), _dpMgr.getDeploymentPlannerByName(work.getDeploymentPlanner())); } catch (CloudRuntimeException e) { e.printStackTrace(); s_logger.info("Caught CloudRuntimeException, returning job failed " + e); CloudRuntimeException ex = new CloudRuntimeException("Unable to start VM instance"); return new Pair<JobInfo.Status, String>(JobInfo.Status.FAILED, JobSerializerHelper.toObjectSerializedString(ex)); } return new Pair<JobInfo.Status, String>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair<JobInfo.Status, String> orchestrateStop(final VmWorkStop work) throws Exception { final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); if (vm == null) { s_logger.info("Unable to find vm " + work.getVmId()); throw new CloudRuntimeException("Unable to find VM id=" + work.getVmId()); } orchestrateStop(vm.getUuid(), work.isCleanup()); return new Pair<JobInfo.Status, String>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair<JobInfo.Status, String> orchestrateMigrate(final VmWorkMigrate work) throws Exception { final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); if (vm == null) { s_logger.info("Unable to find vm " + work.getVmId()); } assert vm != null; orchestrateMigrate(vm.getUuid(), work.getSrcHostId(), work.getDeployDestination()); return new Pair<JobInfo.Status, String>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair<JobInfo.Status, String> orchestrateMigrateAway(final VmWorkMigrateAway work) throws Exception { final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); if (vm == null) { s_logger.info("Unable to find vm " + work.getVmId()); } assert vm != null; try { orchestrateMigrateAway(vm.getUuid(), work.getSrcHostId(), null); } catch (final InsufficientServerCapacityException e) { s_logger.warn("Failed to deploy vm " + vm.getId() + " with original planner, sending HAPlanner"); orchestrateMigrateAway(vm.getUuid(), work.getSrcHostId(), _haMgr.getHAPlanner()); } return new Pair<JobInfo.Status, String>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair<JobInfo.Status, String> orchestrateMigrateWithStorage(final VmWorkMigrateWithStorage work) throws Exception { final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); if (vm == null) { s_logger.info("Unable to find vm " + work.getVmId()); } assert vm != null; orchestrateMigrateWithStorage(vm.getUuid(), work.getSrcHostId(), work.getDestHostId(), work.getVolumeToPool()); return new Pair<JobInfo.Status, String>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair<JobInfo.Status, String> orchestrateMigrateForScale(final VmWorkMigrateForScale work) throws Exception { final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); if (vm == null) { s_logger.info("Unable to find vm " + work.getVmId()); } assert vm != null; orchestrateMigrateForScale(vm.getUuid(), work.getSrcHostId(), work.getDeployDestination(), work.getNewServiceOfferringId()); return new Pair<JobInfo.Status, String>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair<JobInfo.Status, String> orchestrateReboot(final VmWorkReboot work) throws Exception { final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); if (vm == null) { s_logger.info("Unable to find vm " + work.getVmId()); } assert vm != null; orchestrateReboot(vm.getUuid(), work.getParams()); return new Pair<JobInfo.Status, String>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair<JobInfo.Status, String> orchestrateAddVmToNetwork(final VmWorkAddVmToNetwork work) throws Exception { final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); if (vm == null) { s_logger.info("Unable to find vm " + work.getVmId()); } assert vm != null; final Network network = _networkDao.findById(work.getNetworkId()); final NicProfile nic = orchestrateAddVmToNetwork(vm, network, work.getRequestedNicProfile()); return new Pair<JobInfo.Status, String>(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(nic)); } @ReflectionUse private Pair<JobInfo.Status, String> orchestrateRemoveNicFromVm(final VmWorkRemoveNicFromVm work) throws Exception { final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); if (vm == null) { s_logger.info("Unable to find vm " + work.getVmId()); } assert vm != null; final NicVO nic = _entityMgr.findById(NicVO.class, work.getNicId()); final boolean result = orchestrateRemoveNicFromVm(vm, nic); return new Pair<JobInfo.Status, String>(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(result)); } @ReflectionUse private Pair<JobInfo.Status, String> orchestrateRemoveVmFromNetwork(final VmWorkRemoveVmFromNetwork work) throws Exception { final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); if (vm == null) { s_logger.info("Unable to find vm " + work.getVmId()); } assert vm != null; final boolean result = orchestrateRemoveVmFromNetwork(vm, work.getNetwork(), work.getBroadcastUri()); return new Pair<JobInfo.Status, String>(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(result)); } @ReflectionUse private Pair<JobInfo.Status, String> orchestrateReconfigure(final VmWorkReconfigure work) throws Exception { final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); if (vm == null) { s_logger.info("Unable to find vm " + work.getVmId()); } assert vm != null; final ServiceOffering newServiceOffering = _offeringDao.findById(vm.getId(), work.getNewServiceOfferingId()); reConfigureVm(vm.getUuid(), newServiceOffering, work.isSameHost()); return new Pair<JobInfo.Status, String>(JobInfo.Status.SUCCEEDED, null); } @ReflectionUse private Pair<JobInfo.Status, String> orchestrateStorageMigration(final VmWorkStorageMigration work) throws Exception { final VMInstanceVO vm = _entityMgr.findById(VMInstanceVO.class, work.getVmId()); if (vm == null) { s_logger.info("Unable to find vm " + work.getVmId()); } assert vm != null; final StoragePool pool = (PrimaryDataStoreInfo) dataStoreMgr .getPrimaryDataStore(work.getDestStoragePoolId()); orchestrateStorageMigration(vm.getUuid(), pool); return new Pair<JobInfo.Status, String>(JobInfo.Status.SUCCEEDED, null); } @Override public Pair<JobInfo.Status, String> handleVmWorkJob(final VmWork work) throws Exception { return _jobHandlerProxy.handleVmWorkJob(work); } private VmWorkJobVO createPlaceHolderWork(final long instanceId) { final VmWorkJobVO workJob = new VmWorkJobVO(""); workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_PLACEHOLDER); workJob.setCmd(""); workJob.setCmdInfo(""); workJob.setAccountId(0); workJob.setUserId(0); workJob.setStep(VmWorkJobVO.Step.Starting); workJob.setVmType(VirtualMachine.Type.Instance); workJob.setVmInstanceId(instanceId); workJob.setInitMsid(ManagementServerNode.getManagementServerId()); _workJobDao.persist(workJob); return workJob; } }