Java tutorial
/************************************************************************* * Copyright 2009-2016 Eucalyptus Systems, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; version 3 of the License. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see http://www.gnu.org/licenses/. * * Please contact Eucalyptus Systems, Inc., 6755 Hollister Ave., Goleta * CA 93117, USA or visit http://www.eucalyptus.com/licenses/ if you need * additional information or have any questions. ************************************************************************/ package com.eucalyptus.loadbalancing.workflow; import static com.eucalyptus.loadbalancing.LoadBalancerZone.LoadBalancerZoneCoreView.name; import static com.eucalyptus.loadbalancing.LoadBalancerZone.LoadBalancerZoneCoreView.subnetId; import static com.eucalyptus.loadbalancing.service.LoadBalancingService.MAX_HEALTHCHECK_INTERVAL_SEC; import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.eucalyptus.auth.AuthException; import com.eucalyptus.auth.principal.Role; import com.eucalyptus.auth.tokens.SecurityTokenAWSCredentialsProvider; import com.eucalyptus.compute.common.*; import com.eucalyptus.loadbalancing.*; import com.eucalyptus.loadbalancing.common.LoadBalancing; import com.eucalyptus.loadbalancing.LoadBalancingSystemVpcs; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import javax.annotation.Nullable; import com.eucalyptus.loadbalancing.common.msgs.HealthCheck; import com.eucalyptus.loadbalancing.common.msgs.PolicyDescription; import com.eucalyptus.objectstorage.client.EucaS3Client; import com.eucalyptus.objectstorage.client.EucaS3ClientFactory; import com.eucalyptus.ws.StackConfiguration; import org.apache.log4j.Logger; import com.eucalyptus.auth.Accounts; import com.eucalyptus.auth.euare.GetRolePolicyResult; import com.eucalyptus.auth.euare.InstanceProfileType; import com.eucalyptus.auth.euare.RoleType; import com.eucalyptus.auth.euare.ServerCertificateType; import com.eucalyptus.auth.principal.AccountFullName; import com.eucalyptus.auth.principal.AccountIdentifiers; import com.eucalyptus.autoscaling.common.msgs.AutoScalingGroupType; import com.eucalyptus.autoscaling.common.msgs.AutoScalingGroupsType; import com.eucalyptus.autoscaling.common.msgs.DescribeAutoScalingGroupsResponseType; import com.eucalyptus.autoscaling.common.msgs.DescribeAutoScalingGroupsResult; import com.eucalyptus.autoscaling.common.msgs.Instance; import com.eucalyptus.autoscaling.common.msgs.Instances; import com.eucalyptus.autoscaling.common.msgs.LaunchConfigurationType; import com.eucalyptus.cloudwatch.common.msgs.MetricData; import com.eucalyptus.component.annotation.ComponentPart; import com.eucalyptus.crypto.util.B64; import com.eucalyptus.empyrean.ServiceStatusType; import com.eucalyptus.entities.Entities; import com.eucalyptus.entities.TransactionException; import com.eucalyptus.entities.TransactionResource; import com.eucalyptus.loadbalancing.LoadBalancer.LoadBalancerCoreView; import com.eucalyptus.loadbalancing.LoadBalancer.LoadBalancerEntityTransform; import com.eucalyptus.loadbalancing.LoadBalancerBackendInstance.LoadBalancerBackendInstanceCoreView; import com.eucalyptus.loadbalancing.LoadBalancerBackendInstance.LoadBalancerBackendInstanceEntityTransform; import com.eucalyptus.loadbalancing.LoadBalancerListener.LoadBalancerListenerCoreView; import com.eucalyptus.loadbalancing.LoadBalancerListener.LoadBalancerListenerEntityTransform; import com.eucalyptus.loadbalancing.LoadBalancerListener.PROTOCOL; import com.eucalyptus.loadbalancing.LoadBalancerPolicyDescription.LoadBalancerPolicyDescriptionCoreView; import com.eucalyptus.loadbalancing.LoadBalancerSecurityGroup.LoadBalancerSecurityGroupCoreView; import com.eucalyptus.loadbalancing.LoadBalancerSecurityGroup.LoadBalancerSecurityGroupEntityTransform; import com.eucalyptus.loadbalancing.LoadBalancerZone.LoadBalancerZoneCoreView; import com.eucalyptus.loadbalancing.LoadBalancerZone.LoadBalancerZoneEntityTransform; import com.eucalyptus.loadbalancing.activities.EucalyptusActivityException; import com.eucalyptus.loadbalancing.activities.EucalyptusActivityTasks; import com.eucalyptus.loadbalancing.activities.LoadBalancerAutoScalingGroup; import com.eucalyptus.loadbalancing.activities.LoadBalancerAutoScalingGroup.LoadBalancerAutoScalingGroupCoreView; import com.eucalyptus.loadbalancing.activities.LoadBalancerAutoScalingGroup.LoadBalancerAutoScalingGroupEntityTransform; import com.eucalyptus.loadbalancing.activities.LoadBalancerServoInstance; import com.eucalyptus.loadbalancing.activities.LoadBalancerServoInstance.LoadBalancerServoInstanceCoreView; import com.eucalyptus.loadbalancing.activities.LoadBalancerServoInstance.LoadBalancerServoInstanceEntityTransform; import com.eucalyptus.loadbalancing.common.msgs.Listener; import com.eucalyptus.loadbalancing.common.msgs.LoadBalancerServoDescription; import com.eucalyptus.loadbalancing.common.msgs.PolicyAttribute; import com.eucalyptus.resources.client.Ec2Client; import com.eucalyptus.util.CollectionUtils; import com.eucalyptus.util.DNSProperties; import com.eucalyptus.util.Exceptions; import com.google.common.base.Function; import com.google.common.base.Functions; import com.google.common.base.Joiner; import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.base.Strings; import com.google.common.collect.Collections2; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; /** * @author Sang-Min Park (sangmin.park@hpe.com) * */ @ComponentPart(LoadBalancing.class) public class LoadBalancingActivitiesImpl implements LoadBalancingActivities { private static Logger LOG = Logger.getLogger(LoadBalancingActivitiesImpl.class); private static int findAvailableResources(final List<ClusterInfoType> clusters, final String zoneName, final String instanceType) { // parse euca-describe-availability-zones verbose response // WARNING: this is not a standard API! for (int i = 0; i < clusters.size(); i++) { final ClusterInfoType cc = clusters.get(i); if (zoneName.equals(cc.getZoneName())) { for (int j = i + 1; j < clusters.size(); j++) { final ClusterInfoType candidate = clusters.get(j); if (candidate.getZoneName() != null && candidate.getZoneName().toLowerCase().contains(instanceType.toLowerCase())) { //<zoneState>0002 / 0002 2 512 10</zoneState> final String state = candidate.getZoneState(); final String[] tokens = state.split("/"); if (tokens.length > 0) { try { String strNum = tokens[0].trim().replaceFirst("0+", ""); if (strNum.length() <= 0) strNum = "0"; return Integer.parseInt(strNum); } catch (final NumberFormatException ex) { break; } catch (final Exception ex) { break; } } } } break; } } return Integer.MAX_VALUE; // when check fails, let's assume its abundant } @Override public boolean createLbAdmissionControl(final String accountNumber, final String lbName, final String[] zones) throws LoadBalancingActivityException { final String emi = LoadBalancingWorkerProperties.IMAGE; List<ImageDetails> images; try { images = EucalyptusActivityTasks.getInstance().describeImagesWithVerbose(Lists.newArrayList(emi)); if (images == null || images.size() <= 0 || !images.get(0).getImageId().toLowerCase().equals(emi.toLowerCase())) throw new Exception("No loadbalancer EMI is found"); } catch (final Exception ex) { throw new LoadBalancingActivityException("failed to validate the loadbalancer EMI", ex); } // zones: is the CC found? final List<String> requestedZones = Lists.newArrayList(zones); List<ClusterInfoType> clusters; try { clusters = EucalyptusActivityTasks.getInstance().describeAvailabilityZonesWithVerbose(); for (final ClusterInfoType cc : clusters) { requestedZones.remove(cc.getZoneName()); } } catch (final Exception ex) { throw new InvalidConfigurationRequestException("failed to validate the requested zones", ex); } if (requestedZones.size() > 0) { throw new InvalidConfigurationRequestException("unknown zone is requested"); } // are there enough resources? final String instanceType = LoadBalancingWorkerProperties.INSTANCE_TYPE; int numVm = 1; try { //// TODO: fix numVm = Integer.parseInt(LoadBalancingServiceProperties.VM_PER_ZONE); } catch (final NumberFormatException ex) { LOG.warn("unable to parse loadbalancer_num_vm"); } for (final String zone : zones) { final int capacity = findAvailableResources(clusters, zone, instanceType); if (numVm > capacity) { throw new NotEnoughResourcesException(); } } // check if the keyname is configured and exists, the key name for new ELB's should be from // loadbalancing account final String keyName = LoadBalancingWorkerProperties.KEYNAME; if (keyName != null && !keyName.isEmpty()) { try { Ec2Client.getInstance().describeKeyPairs( Accounts.lookupSystemAccountByAlias(AccountIdentifiers.ELB_SYSTEM_ACCOUNT).getUserId(), Lists.newArrayList(keyName)); } catch (Exception ex) { throw new LoadBalancingActivityException( "The configured keyname is not found." + " Do you have keypair " + keyName + " that belongs to " + AccountIdentifiers.ELB_SYSTEM_ACCOUNT + " account?"); } } return true; } private static final String DEFAULT_ROLE_PATH_PREFIX = "/internal/loadbalancer"; public static final String ROLE_NAME_PREFIX = "loadbalancer-vm"; private static final String DEFAULT_ASSUME_ROLE_POLICY = "{\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":[\"ec2.amazonaws.com\"]},\"Action\":[\"sts:AssumeRole\"]}]}"; // FIXME: use lambda expression public static String getRoleName(final String accountNumber, final String loadbalancer) { return String.format("%s-%s-%s", ROLE_NAME_PREFIX, accountNumber, loadbalancer); } @Override public String iamRoleSetup(final String accountNumber, final String lbName) throws LoadBalancingActivityException { RoleType role = null; final String roleName = getRoleName(accountNumber, lbName); // list-roles. try { List<RoleType> result = EucalyptusActivityTasks.getInstance().listRoles(DEFAULT_ROLE_PATH_PREFIX); if (result != null) { for (RoleType r : result) { if (roleName.equals(r.getRoleName())) { role = r; break; } } } } catch (Exception ex) { throw new LoadBalancingActivityException("Failed to list IAM roles", ex); } // if no role found, create a new role with assume-role policy for elb if (role == null) { /// create a new role try { role = EucalyptusActivityTasks.getInstance().createRole(roleName, DEFAULT_ROLE_PATH_PREFIX, DEFAULT_ASSUME_ROLE_POLICY); } catch (Exception ex) { throw new LoadBalancingActivityException("Failed to create the role for ELB Vms"); } } if (role == null) throw new LoadBalancingActivityException("No role is found for LoadBalancer Vms"); return role.getRoleName(); } public static final String DEFAULT_INSTANCE_PROFILE_PATH_PREFIX = "/internal/loadbalancer"; public static final String INSTANCE_PROFILE_NAME_PREFIX = "loadbalancer-vm"; private static String getInstanceProfileName(final String accountNumber, final String loadbalancer) { return String.format("%s-%s-%s", INSTANCE_PROFILE_NAME_PREFIX, accountNumber, loadbalancer); } @Override public String instanceProfileSetup(final String accountNumber, final String lbName, String roleName) throws LoadBalancingActivityException { InstanceProfileType instanceProfile = null; final String instanceProfileName = getInstanceProfileName(accountNumber, lbName); // list instance profiles try { // check if the instance profile for ELB VM is found List<InstanceProfileType> instanceProfiles = EucalyptusActivityTasks.getInstance() .listInstanceProfiles(DEFAULT_INSTANCE_PROFILE_PATH_PREFIX); for (InstanceProfileType ip : instanceProfiles) { if (instanceProfileName.equals(ip.getInstanceProfileName())) { instanceProfile = ip; break; } } } catch (Exception ex) { throw new LoadBalancingActivityException("Failed to list instance profiles", ex); } if (instanceProfile == null) { // if not create one try { instanceProfile = EucalyptusActivityTasks.getInstance().createInstanceProfile(instanceProfileName, DEFAULT_INSTANCE_PROFILE_PATH_PREFIX); } catch (Exception ex) { throw new LoadBalancingActivityException("Failed to create instance profile", ex); } } if (instanceProfile == null) throw new LoadBalancingActivityException("No instance profile for loadbalancer VM is found"); try { List<RoleType> roles = instanceProfile.getRoles().getMember(); boolean roleFound = false; for (RoleType role : roles) { if (role.getRoleName().equals(roleName)) { roleFound = true; break; } } if (!roleFound) throw new NoSuchElementException(); } catch (Exception ex) { if (roleName == null) throw new LoadBalancingActivityException("No role name is found for loadbalancer VMs"); try { EucalyptusActivityTasks.getInstance() .addRoleToInstanceProfile(instanceProfile.getInstanceProfileName(), roleName); } catch (Exception ex2) { throw new LoadBalancingActivityException("Failed to add role to the instance profile", ex2); } } return instanceProfile.getInstanceProfileName(); } static final String SERVO_ROLE_POLICY_NAME = "euca-internal-loadbalancer-vm-policy"; private static final String SERVO_ROLE_POLICY_DOCUMENT = "{\"Statement\":[{\"Action\": [\"swf:PollForActivityTask\", \"swf:RegisterActivityType\", \"swf:RespondActivityTaskCanceled\", \"swf:RespondActivityTaskCompleted\", \"swf:RespondActivityTaskFailed\", \"swf:RecordActivityTaskHeartbeat\"],\"Effect\": \"Allow\",\"Resource\": \"*\"}]}"; @Override public String iamPolicySetup(final String accountNumber, final String lbName, String roleName) throws LoadBalancingActivityException { GetRolePolicyResult policy = null; try { final List<String> policies = EucalyptusActivityTasks.getInstance().listRolePolicies(roleName); if (policies.contains(SERVO_ROLE_POLICY_NAME)) { policy = EucalyptusActivityTasks.getInstance().getRolePolicy(roleName, SERVO_ROLE_POLICY_NAME); } } catch (final Exception ex) { } boolean putPolicy; if (policy == null || policy.getPolicyName() == null || !policy.getPolicyName().equals(SERVO_ROLE_POLICY_NAME)) { putPolicy = true; } else if (!SERVO_ROLE_POLICY_DOCUMENT.toLowerCase().equals(policy.getPolicyDocument().toLowerCase())) { try { EucalyptusActivityTasks.getInstance().deleteRolePolicy(roleName, SERVO_ROLE_POLICY_NAME); } catch (final Exception ex) { LOG.warn("failed to delete role policy", ex); } putPolicy = true; } else { putPolicy = false; } if (putPolicy) { try { EucalyptusActivityTasks.getInstance().putRolePolicy(roleName, SERVO_ROLE_POLICY_NAME, SERVO_ROLE_POLICY_DOCUMENT); } catch (final Exception ex) { throw new LoadBalancingActivityException("failed to put role policy for loadbalancer vm"); } } return SERVO_ROLE_POLICY_NAME; } public static String getSecurityGroupName(final String ownerAccountNumber, final String lbName) { return String.format("euca-internal-%s-%s", ownerAccountNumber, lbName); } private static String generateDefaultVPCSecurityGroupName(final String vpcId) { return String.format("default_elb_%s", UUID.nameUUIDFromBytes(vpcId.getBytes(StandardCharsets.UTF_8)).toString()); } @Override public SecurityGroupSetupActivityResult securityGroupSetup(final String accountNumber, final String lbName) throws LoadBalancingActivityException { final SecurityGroupSetupActivityResult result = new SecurityGroupSetupActivityResult(); LoadBalancer lbEntity; LoadBalancerCoreView lb; try { lbEntity = LoadBalancers.getLoadbalancer(accountNumber, lbName); lb = lbEntity.getCoreView(); } catch (NoSuchElementException ex) { throw new LoadBalancingActivityException("Could not find the loadbalancer with name=" + lbName, ex); } catch (Exception ex) { throw new LoadBalancingActivityException("Error while looking for loadbalancer with name=" + lbName, ex); } if (lb.getVpcId() == null) { final String groupName = getSecurityGroupName(lb.getOwnerAccountNumber(), lb.getDisplayName()); final String groupDesc = String.format("group for loadbalancer %s", lbName); // check if there's an existing group with the same name boolean groupFound = false; try { List<SecurityGroupItemType> groups = EucalyptusActivityTasks.getInstance() .describeSystemSecurityGroups(Lists.newArrayList(groupName)); if (groups != null) for (final SecurityGroupItemType group : groups) { if (groupName.equals(group.getGroupName()) && group.getVpcId() == null) { groupFound = true; result.setGroupName(groupName); result.setGroupId(group.getGroupId()); result.setGroupOwnerAccountId(group.getAccountId()); break; } } } catch (Exception ex) { groupFound = false; } // create a new security group if (!groupFound) { try { EucalyptusActivityTasks.getInstance().createSystemSecurityGroup(groupName, groupDesc); result.setCreatedGroupName(groupName); result.setGroupName(groupName); List<SecurityGroupItemType> groups = EucalyptusActivityTasks.getInstance() .describeSystemSecurityGroups(Lists.newArrayList(groupName)); if (groups != null) for (final SecurityGroupItemType group : groups) { if (groupName.equals(group.getGroupName()) && group.getVpcId() == null) { result.setCreatedGroupId(group.getGroupId()); result.setGroupId(group.getGroupId()); result.setGroupOwnerAccountId(group.getAccountId()); break; } } } catch (Exception ex) { throw new LoadBalancingActivityException("Failed to create the security group for loadbalancer", ex); } } if (result.getGroupName() == null || result.getGroupOwnerAccountId() == null) throw new LoadBalancingActivityException("Failed to create the security group for loadbalancer"); try (final TransactionResource db = Entities.transactionFor(LoadBalancerSecurityGroup.class)) { try { Entities.uniqueResult(LoadBalancerSecurityGroup.named(lbEntity, result.getGroupOwnerAccountId(), result.getGroupName())); } catch (NoSuchElementException ex) { Entities.persist(LoadBalancerSecurityGroup.create(lbEntity, result.getGroupOwnerAccountId(), result.getGroupName())); } db.commit(); } catch (Exception ex) { throw new LoadBalancingActivityException("Error while persisting security group", ex); } /// END OF NON-VPC CASE } else if (lb.getSecurityGroupIdsToNames().isEmpty()) { final String groupName = generateDefaultVPCSecurityGroupName(lb.getVpcId()); final String groupDesc = String.format( "ELB created security group used when no security group is specified during ELB creation - modifications could impact traffic to future ELBs"); final AccountFullName accountFullName = AccountFullName.getInstance(accountNumber); final List<SecurityGroupItemType> groups = EucalyptusActivityTasks.getInstance() .describeUserSecurityGroupsByName(accountFullName, lb.getVpcId(), groupName); final SecurityGroupItemType elbVpcGroup; if (groups.isEmpty()) { EucalyptusActivityTasks.getInstance().createUserSecurityGroup(accountFullName, groupName, groupDesc); final List<SecurityGroupItemType> createdGroupList = EucalyptusActivityTasks.getInstance() .describeUserSecurityGroupsByName(accountFullName, lb.getVpcId(), groupName); elbVpcGroup = Iterables.getOnlyElement(createdGroupList); result.setCreatedGroupId(elbVpcGroup.getGroupId()); result.setCreatedGroupName(elbVpcGroup.getGroupName()); } else { elbVpcGroup = Iterables.get(groups, 0); } Entities.asDistinctTransaction(LoadBalancer.class, new Predicate<String>() { @Override public boolean apply(@Nullable final String loadBalancerName) { try { final LoadBalancer lb = Entities.uniqueResult(LoadBalancer .namedByAccountId(accountFullName.getAccountNumber(), loadBalancerName)); lb.setSecurityGroupRefs(Lists.newArrayList(new LoadBalancerSecurityGroupRef( elbVpcGroup.getGroupId(), elbVpcGroup.getGroupName()))); } catch (TransactionException e) { throw Exceptions.toUndeclared(e); } return true; } }).apply(lb.getDisplayName()); result.setShouldRollback(false); /// In VPC, security groups are user-owned. So ELB shouldn't delete them during rollback result.setGroupId(elbVpcGroup.getGroupId()); result.setGroupName(elbVpcGroup.getGroupName()); result.setGroupOwnerAccountId(elbVpcGroup.getAccountId()); } return result; } @Override public CreateTagActivityResult createLbTagCreator(final String accountNumber, final String lbName, String sgroupId) throws LoadBalancingActivityException { final String TAG_KEY = "service-type"; final String TAG_VALUE = "loadbalancing"; CreateTagActivityResult result = new CreateTagActivityResult(); // security group if (sgroupId != null) { final boolean tagGroup; try { final LoadBalancer lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); tagGroup = lb.getVpcId() == null; } catch (NoSuchElementException ex) { throw new LoadBalancingActivityException("Failed to find the loadbalancer " + lbName, ex); } catch (Exception ex) { throw new LoadBalancingActivityException("Failed due to query exception", ex); } if (tagGroup) try { EucalyptusActivityTasks.getInstance().createTags(TAG_KEY, TAG_VALUE, Lists.newArrayList(sgroupId)); result.setSecurityGroup(sgroupId); } catch (final Exception ex) { LOG.warn("could not tag the security group", ex); } } result.setTagKey(TAG_KEY); result.setTagValue(TAG_VALUE); return result; } private static String getLaunchConfigName(final String ownerAccountNumber, final String loadBalancerName, final String availabilityZone) { String newLaunchConfigName = String.format("lc-euca-internal-elb-%s-%s-%s-%s", ownerAccountNumber, loadBalancerName, availabilityZone, UUID.randomUUID().toString().substring(0, 8)); if (newLaunchConfigName.length() > 255) newLaunchConfigName = newLaunchConfigName.substring(0, 255); return newLaunchConfigName; } private static String getAutoScalingGroupName(final String ownerAccountNumber, final String loadBalancerName, final String availabilityZone) { String groupName = String.format("euca-internal-elb-%s-%s-%s", ownerAccountNumber, loadBalancerName, availabilityZone); if (groupName.length() > 255) groupName = groupName.substring(0, 255); return groupName; } private static String getCredentialsString() { final String credStr = String.format("euca-%s:%s", B64.standard.encString("setup-credential"), LoadBalancingWorkerProperties.EXPIRATION_DAYS); return credStr; } private static String getLoadBalancerUserData(String initScript, final String ownerAccountNumber) { Map<String, String> kvMap = new HashMap<String, String>(); if (LoadBalancingWorkerProperties.NTP_SERVER != null) { kvMap.put("ntp_server", LoadBalancingWorkerProperties.NTP_SERVER); } if (LoadBalancingWorkerProperties.APP_COOKIE_DURATION != null) { kvMap.put("app-cookie-duration", LoadBalancingWorkerProperties.APP_COOKIE_DURATION); } kvMap.put("elb_service_url", String.format("loadbalancing.%s", DNSProperties.getDomain())); kvMap.put("euare_service_url", String.format("euare.%s", DNSProperties.getDomain())); kvMap.put("objectstorage_service_url", String.format("objectstorage.%s", DNSProperties.getDomain())); kvMap.put("simpleworkflow_service_url", String.format("simpleworkflow.%s", DNSProperties.getDomain())); kvMap.put("webservice_port", String.format("%d", StackConfiguration.PORT)); if (ownerAccountNumber != null) kvMap.put("loadbalancer_owner_account", ownerAccountNumber); try { List<ServiceStatusType> services = EucalyptusActivityTasks.getInstance().describeServices("eucalyptus"); if (services == null || services.size() <= 0) throw new EucalyptusActivityException("failed to describe eucalyptus services"); ServiceStatusType service = services.get(0); String serviceUrl = service.getServiceId().getUri(); // parse the service Url: e.g., http://192.168.0.1:8773/services/Eucalyptus String tmp = serviceUrl.replace("http://", "").replace("https://", ""); String host = tmp.substring(0, tmp.indexOf(":")); tmp = tmp.replace(host + ":", ""); String port = tmp.substring(0, tmp.indexOf("/")); String path = tmp.replace(port + "/", ""); kvMap.put("eucalyptus_host", host); kvMap.put("eucalyptus_port", port); kvMap.put("eucalyptus_path", path); } catch (Exception ex) { throw Exceptions.toUndeclared(ex); } final StringBuilder sb = new StringBuilder("#!/bin/bash").append("\n"); if (initScript != null && initScript.length() > 0) sb.append(initScript); sb.append("\n#System generated Load Balancer Servo config\n"); sb.append("mkdir -p /etc/load-balancer-servo/\n"); sb.append("yum -y --disablerepo \\* --enablerepo eucalyptus-service-image install load-balancer-servo\n"); sb.append("echo \""); for (String key : kvMap.keySet()) { String value = kvMap.get(key); sb.append(String.format("\n%s=%s", key, value)); } sb.append("\" > /etc/load-balancer-servo/servo.conf"); sb.append("\nchown -R servo:servo /etc/load-balancer-servo"); sb.append("\nservice load-balancer-servo start"); return sb.toString(); } private static final String TAG_KEY = "service-type"; private static final String TAG_VALUE = "loadbalancing"; @Override public AutoscalingGroupSetupActivityResult autoscalingGroupSetup(final String accountNumber, final String lbName, String instanceProfileName, String securityGroupName, List<String> zones, Map<String, String> zoneToSubnetIdMap) throws LoadBalancingActivityException { if (LoadBalancingWorkerProperties.IMAGE == null) throw new LoadBalancingActivityException("Loadbalancer's EMI is not configured"); final AutoscalingGroupSetupActivityResult activityResult = new AutoscalingGroupSetupActivityResult(); final LoadBalancer lbEntity; final LoadBalancer.LoadBalancerCoreView lb; try { lbEntity = LoadBalancers.getLoadbalancer(accountNumber, lbName); lb = lbEntity.getCoreView(); if (zoneToSubnetIdMap == null) { zoneToSubnetIdMap = CollectionUtils.putAll( Iterables.filter(lbEntity.getZones(), Predicates.compose(Predicates.notNull(), subnetId())), Maps.<String, String>newHashMap(), name(), subnetId()); } } catch (NoSuchElementException ex) { throw new LoadBalancingActivityException("Failed to find the loadbalancer " + lbName, ex); } catch (Exception ex) { throw new LoadBalancingActivityException("Failed due to query exception", ex); } if (zones == null) return null; // do nothing when zone/groups are not specified for (final String availabilityZone : zones) { final String groupName = getAutoScalingGroupName(accountNumber, lbName, availabilityZone); String launchConfigName = null; boolean asgFound = false; try { final DescribeAutoScalingGroupsResponseType response = EucalyptusActivityTasks.getInstance() .describeAutoScalingGroups(Lists.newArrayList(groupName), lb.useSystemAccount()); final List<AutoScalingGroupType> groups = response.getDescribeAutoScalingGroupsResult() .getAutoScalingGroups().getMember(); if (groups.size() > 0 && groups.get(0).getAutoScalingGroupName().equals(groupName)) { asgFound = true; launchConfigName = groups.get(0).getLaunchConfigurationName(); } } catch (final Exception ex) { asgFound = false; } activityResult.setGroupNames(Sets.<String>newHashSet()); activityResult.setLaunchConfigNames(Sets.<String>newHashSet()); activityResult.setCreatedGroupNames(Sets.<String>newHashSet()); activityResult.setCreatedLaunchConfigNames(Sets.<String>newHashSet()); final List<String> availabilityZones = Lists.newArrayList(availabilityZone); String vpcZoneIdentifier = null; String systemVpcZoneIdentifier = null; if (!asgFound) { try { vpcZoneIdentifier = zoneToSubnetIdMap.isEmpty() ? null : Strings.emptyToNull(Joiner.on(',').skipNulls().join( Iterables.transform(availabilityZones, Functions.forMap(zoneToSubnetIdMap)))); if (vpcZoneIdentifier != null) systemVpcZoneIdentifier = LoadBalancingSystemVpcs.getSystemVpcSubnetId(vpcZoneIdentifier); else systemVpcZoneIdentifier = null; } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to look up subnet ID", ex); } try { Set<String> securityGroupNamesOrIds = null; if (systemVpcZoneIdentifier == null) { securityGroupNamesOrIds = Sets.newHashSet(); if (!lb.getSecurityGroupIdsToNames().isEmpty()) { securityGroupNamesOrIds.addAll(lb.getSecurityGroupIdsToNames().keySet()); } else { if (securityGroupName != null) { securityGroupNamesOrIds.add(securityGroupName); } } } else { // if system VPC is used, use it's security group securityGroupNamesOrIds = Sets.newHashSet(); securityGroupNamesOrIds .add(LoadBalancingSystemVpcs.getSecurityGroupId(systemVpcZoneIdentifier)); } final String KEYNAME = LoadBalancingWorkerProperties.KEYNAME; final String keyName = KEYNAME != null && KEYNAME.length() > 0 ? KEYNAME : null; final String userData = B64.standard .encString(String.format("%s\n%s", getCredentialsString(), getLoadBalancerUserData( LoadBalancingWorkerProperties.INIT_SCRIPT, lb.getOwnerAccountNumber()))); launchConfigName = getLaunchConfigName(lb.getOwnerAccountNumber(), lb.getDisplayName(), availabilityZone); EucalyptusActivityTasks.getInstance().createLaunchConfiguration( LoadBalancingWorkerProperties.IMAGE, LoadBalancingWorkerProperties.INSTANCE_TYPE, instanceProfileName, launchConfigName, securityGroupNamesOrIds, keyName, userData, zoneToSubnetIdMap.isEmpty() ? null : false, lb.useSystemAccount()); activityResult.getLaunchConfigNames().add(launchConfigName); activityResult.getCreatedLaunchConfigNames().add(launchConfigName); } catch (Exception ex) { throw new LoadBalancingActivityException("Failed to create launch configuration", ex); } } activityResult.getLaunchConfigNames().add(launchConfigName); /// FIXME Integer capacity = LoadBalancingServiceProperties.getCapacityPerZone(); if (!asgFound) { // create autoscaling group with the zone and desired capacity try { EucalyptusActivityTasks.getInstance().createAutoScalingGroup(groupName, availabilityZones, systemVpcZoneIdentifier, capacity, launchConfigName, TAG_KEY, TAG_VALUE, lb.useSystemAccount()); activityResult.getGroupNames().add(groupName); activityResult.getCreatedGroupNames().add(groupName); if (activityResult.getNumVMsPerZone() == null || activityResult.getNumVMsPerZone() == 0) { activityResult.setNumVMsPerZone(capacity); } else { activityResult.setNumVMsPerZone(activityResult.getNumVMsPerZone() + capacity); } } catch (Exception ex) { throw new LoadBalancingActivityException("Failed to create autoscaling group", ex); } } else { try { EucalyptusActivityTasks.getInstance().updateAutoScalingGroup(groupName, availabilityZones, capacity, launchConfigName, lb.useSystemAccount()); } catch (Exception ex) { throw new LoadBalancingActivityException("Failed to update the autoscaling group", ex); } } activityResult.getGroupNames().add(groupName); if (activityResult.getNumVMsPerZone() == null || activityResult.getNumVMsPerZone() == 0) { activityResult.setNumVMsPerZone(capacity); } else { activityResult.setNumVMsPerZone(activityResult.getNumVMsPerZone() + capacity); } // commit ASG record to the database try (final TransactionResource db = Entities.transactionFor(LoadBalancerAutoScalingGroup.class)) { try { final LoadBalancerAutoScalingGroup group = Entities .uniqueResult(LoadBalancerAutoScalingGroup.named(lbEntity, availabilityZone)); if (capacity != null) group.setCapacity(capacity); } catch (NoSuchElementException ex) { final LoadBalancerAutoScalingGroup group = LoadBalancerAutoScalingGroup.newInstance(lbEntity, availabilityZone, vpcZoneIdentifier, systemVpcZoneIdentifier, groupName, launchConfigName); if (capacity != null) group.setCapacity(capacity); Entities.persist(group); } db.commit(); } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to commit the database", ex); } } // end of for all zones return activityResult; } @Override public void securityGroupSetupRollback(String accountNumber, String lbName, SecurityGroupSetupActivityResult result) { if (result.getCreatedGroupName() == null || !result.getShouldRollback()) return; // set security group with the loadbalancer; update db LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (Exception ex) { return; } try { EucalyptusActivityTasks.getInstance().deleteSystemSecurityGroup(result.getCreatedGroupName()); } catch (Exception ex) { // when there's any servo instance referencing the security group // SecurityGroupCleanup will clean up records } try (final TransactionResource db = Entities.transactionFor(LoadBalancerSecurityGroup.class)) { final LoadBalancerSecurityGroup group = Entities.uniqueResult(LoadBalancerSecurityGroup.named(lb, result.getGroupOwnerAccountId(), result.getCreatedGroupName())); group.setState(LoadBalancerSecurityGroup.STATE.OutOfService); group.setLoadBalancer(null); Entities.persist(group); db.commit(); } catch (NoSuchElementException ex) { } catch (Exception ex) { LOG.error("failed to mark the security group OutOfService", ex); } } @Override public void createLbTagCreatorRollback(CreateTagActivityResult result) { if (result.getSecurityGroup() != null) { try { EucalyptusActivityTasks.getInstance().deleteTags(result.getTagKey(), result.getTagValue(), Lists.newArrayList(result.getSecurityGroup())); } catch (final Exception ex) { ; } } } @Override public void autoscalingGroupSetupRollback(String accountNumber, String lbName, AutoscalingGroupSetupActivityResult result) { LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (Exception ex) { LOG.error("failed to find the loadbalancer: " + lbName); return; } if (result.getCreatedGroupNames() != null) { for (final String asgName : result.getCreatedGroupNames()) { // delete autoscaling group try { // terminate all instances EucalyptusActivityTasks.getInstance().deleteAutoScalingGroup(asgName, true, lb.useSystemAccount()); } catch (Exception ex) { LOG.error("failed to delete autoscaling group - " + asgName); } } } if (result.getCreatedLaunchConfigNames() != null) { for (final String launchConfigName : result.getCreatedLaunchConfigNames()) { // delete launch config try { EucalyptusActivityTasks.getInstance().deleteLaunchConfiguration(launchConfigName, lb.useSystemAccount()); } catch (Exception ex) { LOG.error("failed to delete launch configuration - " + launchConfigName); } } } } @Override public List<String> enableAvailabilityZonesPersistUpdatedZones(String accountNumber, String lbName, List<String> zonesToEnable, Map<String, String> zoneToSubnetIdMap) throws LoadBalancingActivityException { LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (NoSuchElementException ex) { throw new LoadBalancingActivityException("Could not find the loadbalancer with name=" + lbName, ex); } catch (Exception ex) { throw new LoadBalancingActivityException("Error while looking for loadbalancer with name=" + lbName, ex); } final List<String> persistedZones = Lists.newArrayList(); if (zonesToEnable != null) { for (final String zone : zonesToEnable) { try (final TransactionResource db = Entities.transactionFor(LoadBalancerZone.class)) { try { final LoadBalancerZone exist = Entities.uniqueResult(LoadBalancerZone.named(lb, zone)); exist.setState(LoadBalancerZone.STATE.InService); } catch (NoSuchElementException ex) { final String subnetId = zoneToSubnetIdMap == null ? null : zoneToSubnetIdMap.get(zone); final LoadBalancerZone newZone = LoadBalancerZone.create(lb, zone, subnetId); newZone.setState(LoadBalancerZone.STATE.InService); Entities.persist(newZone); } persistedZones.add(zone); db.commit(); } catch (Exception ex) { throw new LoadBalancingActivityException("Error adding load balancer zone", ex); } } } return persistedZones; } @Override public void enableAvailabilityZonesPersistUpdatedZonesRollback(String accountNumber, String lbName, List<String> zonesToRollback) { if (zonesToRollback == null || zonesToRollback.size() <= 0) return; LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (NoSuchElementException ex) { LOG.warn("Could not find the loadbalancer with name=" + lbName, ex); return; } catch (Exception ex) { LOG.warn("Error while looking for loadbalancer with name=" + lbName, ex); return; } for (final LoadBalancerZoneCoreView zoneView : lb.getZones()) { if (zonesToRollback.contains(zoneView.getName())) { try (final TransactionResource db = Entities.transactionFor(LoadBalancerZone.class)) { final LoadBalancerZone sample = LoadBalancerZone.named(lb, zoneView.getName()); final LoadBalancerZone update = Entities.uniqueResult(sample); update.setState(LoadBalancerZone.STATE.OutOfService); db.commit(); } catch (final Exception ex) { LOG.error("could not mark out of state for the zone", ex); } } } } @Override public void enableAvailabilityZonesPersistBackendInstanceState(String accountNumber, String lbName, List<String> enabledZones) throws LoadBalancingActivityException { LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (NoSuchElementException ex) { LOG.warn("Could not find the loadbalancer with name=" + lbName, ex); return; } catch (Exception ex) { LOG.warn("Error while looking for loadbalancer with name=" + lbName, ex); return; } try { for (final String enabledZone : enabledZones) { final LoadBalancerZone zone = LoadBalancers.findZone(lb, enabledZone); for (final LoadBalancerBackendInstanceCoreView instance : zone.getBackendInstances()) { try (final TransactionResource db = Entities .transactionFor(LoadBalancerBackendInstance.class)) { final LoadBalancerBackendInstance update = Entities .uniqueResult(LoadBalancerBackendInstance.named(lb, instance.getInstanceId())); update.setReasonCode(""); update.setDescription(""); db.commit(); } catch (final NoSuchElementException ex) { LOG.warn("failed to find the backend instance"); } catch (final Exception ex) { LOG.warn("failed to query the backend instance", ex); } } } } catch (final Exception ex) { LOG.warn("unable to update backend instances after enabling zone", ex); } } @Override public void createListenerCheckSSLCertificateId(String accountNumber, String lbName, Listener[] listeners) throws LoadBalancingActivityException { LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (Exception ex) { throw new LoadBalancingActivityException("could not find the loadbalancer", ex); } for (Listener listener : listeners) { final PROTOCOL protocol = PROTOCOL.valueOf(listener.getProtocol().toUpperCase()); if (protocol.equals(PROTOCOL.HTTPS) || protocol.equals(PROTOCOL.SSL)) { final String certArn = listener.getSSLCertificateId(); if (certArn == null || certArn.length() <= 0) throw new LoadBalancingActivityException("No SSLCertificateId is specified"); final String prefix = String.format("arn:aws:iam::%s:server-certificate", accountNumber); if (!certArn.startsWith(prefix)) throw new LoadBalancingActivityException("SSLCertificateId is not ARN format"); try { final String pathAndName = certArn.replace(prefix, ""); final String certName = pathAndName.substring(pathAndName.lastIndexOf("/") + 1); final ServerCertificateType cert = EucalyptusActivityTasks.getInstance() .getServerCertificate(accountNumber, certName); if (cert == null) throw new LoadBalancingActivityException("No SSL certificate is found with the ARN"); if (!certArn.equals(cert.getServerCertificateMetadata().getArn())) throw new LoadBalancingActivityException( "Returned certificate's ARN doesn't match the request"); } catch (final LoadBalancingActivityException ex) { throw ex; } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to get SSL server certificate", ex); } } } } public static final String SERVER_CERT_ROLE_POLICY_NAME_PREFIX = "loadbalancer-iam-policy"; public static final String ROLE_SERVER_CERT_POLICY_DOCUMENT = "{\"Statement\":[{\"Action\": [\"iam:DownloadServerCertificate\"],\"Effect\": \"Allow\",\"Resource\": \"CERT_ARN_PLACEHOLDER\"}]}"; @Override public AuthorizeSSLCertificateActivityResult createListenerAuthorizeSSLCertificate(String accountNumber, String lbName, Listener[] listeners) throws LoadBalancingActivityException { final AuthorizeSSLCertificateActivityResult result = new AuthorizeSSLCertificateActivityResult(); final Set<String> certArns = Sets.newHashSet(); final List<String> policyNames = Lists.newArrayList(); for (final Listener listener : listeners) { final PROTOCOL protocol = PROTOCOL.valueOf(listener.getProtocol().toUpperCase()); if (protocol.equals(PROTOCOL.HTTPS) || protocol.equals(PROTOCOL.SSL)) { certArns.add(listener.getSSLCertificateId()); } } if (certArns.size() <= 0) return result; LoadBalancer lb = null; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (Exception ex) { throw new LoadBalancingActivityException("could not find the loadbalancer", ex); } final String roleName = String.format("%s-%s-%s", ROLE_NAME_PREFIX, accountNumber, lbName); final String prefix = String.format("arn:aws:iam::%s:server-certificate", accountNumber); for (final String arn : certArns) { if (!arn.startsWith(prefix)) continue; String pathAndName = arn.replace(prefix, ""); String certName = pathAndName.substring(pathAndName.lastIndexOf("/") + 1); String policyName = String.format("%s-%s-%s-%s", SERVER_CERT_ROLE_POLICY_NAME_PREFIX, accountNumber, lbName, certName); final String rolePolicyDoc = ROLE_SERVER_CERT_POLICY_DOCUMENT.replace("CERT_ARN_PLACEHOLDER", arn); try { EucalyptusActivityTasks.getInstance().putRolePolicy(roleName, policyName, rolePolicyDoc, lb.useSystemAccount()); policyNames.add(policyName); } catch (final Exception ex) { throw new LoadBalancingActivityException("failed to authorize server certificate for SSL listener", ex); } } result.setPolicyNames(policyNames); result.setRoleName(roleName); return result; } @Override public void createListenerAuthorizeSSLCertificateRollback(String accountNumber, String lbName, AuthorizeSSLCertificateActivityResult result) { final String roleName = result.getRoleName(); final List<String> policyNames = result.getPolicyNames(); if (roleName != null && policyNames != null && policyNames.size() > 0) { LoadBalancer lb = null; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (Exception ex) { return; } for (final String policyName : policyNames) { try { EucalyptusActivityTasks.getInstance().deleteRolePolicy(roleName, policyName, lb.useSystemAccount()); } catch (final Exception ex) { LOG.warn("Failed to delete role policy during listener creation rollback", ex); } } } } @Override public AuthorizeIngressRuleActivityResult createListenerAuthorizeIngressRule(String accountNumber, String lbName, Listener[] listeners) throws LoadBalancingActivityException { final AuthorizeIngressRuleActivityResult result = new AuthorizeIngressRuleActivityResult(); result.setListeners(Lists.<Listener>newArrayList()); LoadBalancer lb; String groupName = null; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); final LoadBalancerSecurityGroupCoreView group = lb.getGroup(); if (group != null) groupName = group.getName(); } catch (Exception ex) { throw new LoadBalancingActivityException("could not find the loadbalancer", ex); } final Map<String, String> securityGroupIdsToNames = lb.getCoreView().getSecurityGroupIdsToNames(); String protocol = "tcp"; /// Loadbalancer listeners protocols: HTTP, HTTPS, TCP, SSL -> all tcp if (lb.getVpcId() == null) { if (groupName == null) throw new LoadBalancingActivityException("Group name is not found"); for (Listener listener : listeners) { int port = listener.getLoadBalancerPort(); try { EucalyptusActivityTasks.getInstance().authorizeSystemSecurityGroup(groupName, protocol, port, lb.useSystemAccount()); result.getListeners().add(listener); } catch (Exception ex) { throw new LoadBalancingActivityException( String.format("failed to authorize %s, %s, %d", groupName, protocol, port), ex); } } } else if (securityGroupIdsToNames.size() == 1) { if (securityGroupIdsToNames.values().contains(generateDefaultVPCSecurityGroupName(lb.getVpcId()))) { boolean isRuleEmpty = false; try { final SecurityGroupItemType defaultGroup = EucalyptusActivityTasks.getInstance() .describeUserSecurityGroupsByName(AccountFullName.getInstance(accountNumber), lb.getVpcId(), securityGroupIdsToNames.values().stream().findAny().get()) .stream().findAny().get(); if (defaultGroup.getIpPermissions() == null || defaultGroup.getIpPermissions().isEmpty()) { isRuleEmpty = true; } } catch (final Exception ex) { isRuleEmpty = false; } if (isRuleEmpty) { // the rule is created only for the first time the group is created final String groupId = Iterables.getOnlyElement(securityGroupIdsToNames.keySet()); for (Listener listener : listeners) { int port = listener.getLoadBalancerPort(); try { EucalyptusActivityTasks.getInstance().authorizeSystemSecurityGroup(groupId, protocol, port, false); } catch (Exception ex) { throw new LoadBalancingActivityException( String.format("failed to authorize %s, %s, %d", groupId, protocol, port), ex); } } } } } return result; } @Override public void createListenerAuthorizeIngressRuleRollback(String accountNumber, String lbName, AuthorizeIngressRuleActivityResult result) { final List<Listener> listeners = result.getListeners(); if (listeners == null || listeners.size() <= 0) return; LoadBalancer lb = null; String groupName = null; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); final LoadBalancerSecurityGroupCoreView group = lb.getGroup(); if (group != null) groupName = group.getName(); } catch (Exception ex) { ; } if (groupName == null) return; for (Listener listener : listeners) { int port = listener.getLoadBalancerPort(); String protocol = listener.getProtocol(); protocol = protocol.toLowerCase(); try { EucalyptusActivityTasks.getInstance().revokeSystemSecurityGroup(groupName, protocol, port, lb.useSystemAccount()); } catch (Exception ex) { ; } } } @Override public void createListenerUpdateHealthCheckConfig(String accountNumber, String lbName, Listener[] listeners) throws LoadBalancingActivityException { LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (final NoSuchElementException ex) { throw new LoadBalancingActivityException("Could not find the loadbalancer with name=" + lbName, ex); } catch (final Exception ex) { throw new LoadBalancingActivityException("Error while looking for loadbalancer with name=" + lbName, ex); } final int DEFAULT_HEALTHY_THRESHOLD = 3; final int DEFAULT_INTERVAL = 30; final int DEFAULT_TIMEOUT = 5; final int DEFAULT_UNHEALTHY_THRESHOLD = 3; /* default setting in AWS "HealthyThreshold": 10, "Interval": 30, "Target": "TCP:8000", "Timeout": 5, "UnhealthyThreshold": 2 */ try { lb.getHealthCheckTarget(); lb.getHealthCheckInterval(); lb.getHealthCheckTimeout(); lb.getHealthCheckUnhealthyThreshold(); lb.getHealthyThreshold(); } catch (final IllegalStateException ex) { /// only when the health check is not previously configured if (listeners == null || listeners.length <= 0) throw new LoadBalancingActivityException("No listener requested"); final Listener firstListener = listeners[0]; final String target = String.format("TCP:%d", firstListener.getInstancePort()); try (final TransactionResource db = Entities.transactionFor(LoadBalancer.class)) { final LoadBalancer update = Entities.uniqueResult(lb); update.setHealthCheck(DEFAULT_HEALTHY_THRESHOLD, DEFAULT_INTERVAL, target, DEFAULT_TIMEOUT, DEFAULT_UNHEALTHY_THRESHOLD); db.commit(); } catch (final NoSuchElementException exx) { LOG.warn("Loadbalancer not found in the database"); } catch (final Exception exx) { LOG.warn("Unable to query the loadbalancer", ex); } } } @Override public void createListenerAddDefaultSSLPolicy(String accountNumber, String lbName, Listener[] listeners) throws LoadBalancingActivityException { LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (NoSuchElementException ex) { throw new LoadBalancingActivityException("Could not find the loadbalancer with name=" + lbName, ex); } catch (Exception ex) { throw new LoadBalancingActivityException("Error while looking for loadbalancer with name=" + lbName, ex); } boolean sslListener = false; for (final Listener l : listeners) { final String protocol = l.getProtocol().toLowerCase(); if ("https".equals(protocol) || "ssl".equals(protocol)) { sslListener = true; break; } } if (!sslListener) return; try { /// this will load the sample policies into memory if (LoadBalancerPolicies.LATEST_SECURITY_POLICY_NAME == null) { LoadBalancerPolicies.getSamplePolicyDescription(); if (LoadBalancerPolicies.LATEST_SECURITY_POLICY_NAME == null) throw new LoadBalancingActivityException("Latest security policy is not found"); } boolean policyCreated = false; final Collection<LoadBalancerPolicyDescriptionCoreView> policies = lb.getPolicies(); if (policies != null) { for (final LoadBalancerPolicyDescriptionCoreView view : policies) { if ("SSLNegotiationPolicyType".equals(view.getPolicyTypeName()) && LoadBalancerPolicies.LATEST_SECURITY_POLICY_NAME.equals(view.getPolicyName())) { policyCreated = true; break; } } } if (!policyCreated) { final PolicyAttribute attr = new PolicyAttribute(); attr.setAttributeName("Reference-Security-Policy"); attr.setAttributeValue(LoadBalancerPolicies.LATEST_SECURITY_POLICY_NAME); LoadBalancerPolicies.addLoadBalancerPolicy(lb, LoadBalancerPolicies.LATEST_SECURITY_POLICY_NAME, "SSLNegotiationPolicyType", Lists.newArrayList(attr)); try { // reload with the newly created policy lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (NoSuchElementException ex) { throw new LoadBalancingActivityException("Could not find the loadbalancer with name=" + lbName, ex); } catch (Exception ex) { throw new LoadBalancingActivityException( "Error while looking for loadbalancer with name=" + lbName, ex); } } } catch (final Exception ex) { LOG.warn("Failed to create default security policy for https/ssl listeners", ex); return; } try { final LoadBalancerPolicyDescription policy = LoadBalancerPolicies.getLoadBalancerPolicyDescription(lb, LoadBalancerPolicies.LATEST_SECURITY_POLICY_NAME); if (policy == null) throw new LoadBalancingActivityException( "No such policy is found: " + LoadBalancerPolicies.LATEST_SECURITY_POLICY_NAME); final Collection<LoadBalancerListenerCoreView> lbListeners = lb.getListeners(); for (final Listener l : listeners) { final String protocol = l.getProtocol().toLowerCase(); if ("https".equals(protocol) || "ssl".equals(protocol)) { LoadBalancerListener listener = null; for (final LoadBalancerListenerCoreView view : lbListeners) { if (view.getLoadbalancerPort() == l.getLoadBalancerPort()) { listener = LoadBalancerListenerEntityTransform.INSTANCE.apply(view); break; } } if (listener == null) throw new LoadBalancingActivityException("No such listener is found"); boolean policyAttached = false; final List<LoadBalancerPolicyDescriptionCoreView> listenerPolicies = listener.getPolicies(); if (listenerPolicies != null) { for (final LoadBalancerPolicyDescriptionCoreView listenerPolicy : listenerPolicies) { if ("SSLNegotiationPolicyType".equals(listenerPolicy.getPolicyTypeName()) && LoadBalancerPolicies.LATEST_SECURITY_POLICY_NAME .equals(listenerPolicy.getPolicyName())) { policyAttached = true; break; } } } if (!policyAttached && listener != null && policy != null) { LoadBalancerPolicies.addPoliciesToListener(listener, Lists.newArrayList(policy)); } } } } catch (final Exception ex) { LOG.warn("Failed to set default security policy to https/ssl listeners", ex); } } @Override public HealthCheck lookupLoadBalancerHealthCheck(final String accountNumber, final String lbName) throws LoadBalancingActivityException { try { final LoadBalancer lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); final HealthCheck hc = new HealthCheck(); hc.setHealthyThreshold(lb.getHealthyThreshold()); hc.setInterval(lb.getHealthCheckInterval()); hc.setTarget(lb.getHealthCheckTarget()); hc.setTimeout(lb.getHealthCheckTimeout()); hc.setUnhealthyThreshold(lb.getHealthCheckUnhealthyThreshold()); return hc; } catch (final Exception ex) { throw new LoadBalancingActivityException( String.format("Failed to lookup loadbalancer (%s:%s", accountNumber, lbName)); } } @Override public Map<String, String> filterInstanceStatus(final String accountNumber, final String lbName, final String servoInstanceId, final String encodedStatus) throws LoadBalancingActivityException { if (encodedStatus == null) { return Maps.newHashMap(); } String monitoringZone = null; try { final LoadBalancerServoInstance servo = LoadBalancers.lookupServoInstance(servoInstanceId); monitoringZone = servo.getAvailabilityZone().getName(); } catch (final Exception ex) { throw new LoadBalancingActivityException( "Failed to lookup the servo instance with id=" + servoInstanceId); } final Map<String, String> instanceToZone = Maps.newHashMap(); final Set<String> stoppedInstances = Sets.newHashSet(); try { final LoadBalancer lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); lb.getBackendInstances().stream() .forEach(instance -> instanceToZone.put(instance.getInstanceId(), instance.getPartition())); stoppedInstances.addAll(lb.getBackendInstances().stream() .filter(v -> LoadBalancerBackendInstanceStates.InstanceStopped.isInstanceState(v)) .map(v -> v.getInstanceId()).collect(Collectors.toList())); } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to lookup the loadbalancer with name=" + lbName); } final Set<String> validStates = Sets.newHashSet(LoadBalancerBackendInstance.STATE.InService.name(), LoadBalancerBackendInstance.STATE.OutOfService.name()); final Map<String, String> instanceToStatus = Maps.newHashMap(); try { final Map<String, String> statusMap = VmWorkflowMarshaller.unmarshalInstances(encodedStatus); for (final String instanceId : statusMap.keySet()) { final String instanceStatus = statusMap.get(instanceId); if (!validStates.contains(instanceStatus)) continue; if (!(instanceToZone.containsKey(instanceId) && instanceToZone.get(instanceId).equals(monitoringZone))) continue; if (stoppedInstances.contains(instanceId)) continue; // EUCA-11859: do not update health check result if instance stopped instanceToStatus.put(instanceId, instanceStatus); } } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed unmarshalling instance status message", ex); } return instanceToStatus; } //TODO: SCALE @Override public void updateInstanceStatus(final String accountNumber, final String lbName, final Map<String, String> statusMap) throws LoadBalancingActivityException { // for each status, deserialize to Instance type // merge the results for each instance // update database final Set<String> validStatus = Sets.newHashSet(LoadBalancerBackendInstance.STATE.InService.name(), LoadBalancerBackendInstance.STATE.OutOfService.name()); final Map<String, String> verifiedStatusMap = statusMap.entrySet().stream() .filter(entry -> validStatus.contains(entry.getValue())) .collect(Collectors.toMap(p -> p.getKey(), p -> p.getValue())); if (verifiedStatusMap.isEmpty()) return; boolean updated = false; boolean committed = false; final int TRANSACTION_RETRY = 5; final LoadBalancer lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); for (int i = 1; i <= TRANSACTION_RETRY; i++) { try (final TransactionResource db = Entities.transactionFor(LoadBalancerBackendInstance.class)) { for (final String instanceId : verifiedStatusMap.keySet()) { final LoadBalancerBackendInstance sample = LoadBalancerBackendInstance.named(lb, instanceId); final LoadBalancerBackendInstance update = Entities.uniqueResult(sample); final String newStatus = verifiedStatusMap.get(instanceId); final LoadBalancerBackendInstance.STATE oldState = update.getBackendState(); final LoadBalancerBackendInstance.STATE newState = LoadBalancerBackendInstance.STATE .valueOf(newStatus); if (!oldState.equals(newState)) updated = true; update.setBackendState(newState); final LoadBalancerBackendInstanceStates failure = LoadBalancerBackendInstanceStates.HealthCheckFailure; final LoadBalancerBackendInstanceStates success = LoadBalancerBackendInstanceStates.HealthCheckSuccess; if (success.getState().equals(newState)) { update.setReasonCode(success.getReasonCode()); update.setDescription(success.getDescription()); } else if (failure.getState().equals(newState)) { update.setReasonCode(failure.getReasonCode()); update.setDescription(failure.getDescription()); } update.updateInstanceStateTimestamp(); Entities.persist(update); } db.commit(); } catch (final Exception ex) { try { Thread.sleep((long) ((Math.random() * 100) * Math.pow(2, i))); } catch (final Exception ex2) { ; } continue; } committed = true; break; } if (!committed) { throw new LoadBalancingActivityException("Failed to persist instance status"); } // if changed, updating loadbalancer will cause registering instances in the servo VMs if (updated) { LoadBalancingWorkflows.updateLoadBalancer(accountNumber, lbName); } } @Override public void putCloudWatchInstanceHealth(String accountNumber, String lbName) throws LoadBalancingActivityException { List<LoadBalancerBackendInstance> backendInstances = Lists.newArrayList(); LoadBalancer lb = null; //TODO: SCALE try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); backendInstances = Lists.transform(Lists.newArrayList(lb.getBackendInstances()), LoadBalancerBackendInstanceEntityTransform.INSTANCE); } catch (final Exception ex) { LOG.error("failed to retrieve loadbalancer's backend instances", ex); return; } /// Update Cloudwatch for (final LoadBalancerBackendInstance backend : backendInstances) { final String zoneName = backend.getAvailabilityZone().getName(); if (backend.getState().equals(LoadBalancerBackendInstance.STATE.InService)) { LoadBalancerCwatchMetrics.getInstance().updateHealthy(lb.getCoreView(), zoneName, backend.getInstanceId()); } else if (backend.getState().equals(LoadBalancerBackendInstance.STATE.OutOfService)) { LoadBalancerCwatchMetrics.getInstance().updateUnHealthy(lb.getCoreView(), zoneName, backend.getInstanceId()); } } } @Override public void putCloudWatchMetrics(String accountNumber, String lbName, Map<String, String> metrics) throws LoadBalancingActivityException { if (metrics != null) { for (final String instanceId : metrics.keySet()) { final String metric = metrics.get(instanceId); if (metric == null) continue; /// metric data from the servo VM final MetricData data = VmWorkflowMarshaller.unmarshalMetrics(metric); if (data.getMember() == null || data.getMember().size() <= 0) continue; LoadBalancerZone zone = null; /// TODO: SCALE try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { final LoadBalancerServoInstance sample = LoadBalancerServoInstance.named(instanceId); final LoadBalancerServoInstance entity = Entities.uniqueResult(sample); zone = LoadBalancerZoneEntityTransform.INSTANCE.apply(entity.getAvailabilityZone()); } catch (final Exception ex) { LOG.error("Failed to lookup servo instance named: " + instanceId); ; } if (zone != null) { try { LoadBalancerCwatchMetrics.getInstance().addMetric(zone, data); } catch (Exception ex) { LOG.error("Failed to add ELB cloudwatch metric", ex); } } } } } @Override public List<String> listLoadBalancerPolicies(final String accountNumber, final String lbName) throws LoadBalancingActivityException { try { final LoadBalancer lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); final List<LoadBalancerListener> listeners = lb.getListeners().stream() .map(view -> LoadBalancerListenerEntityTransform.INSTANCE.apply(view)) .collect(Collectors.toList()); final List<LoadBalancerBackendServerDescription> backendServers = LoadBalancerBackendServers .getLoadBalancerBackendServerDescription(lb); final List<String> listenerPolicies = listeners.stream().map(l -> l.getPolicies()) .flatMap(p -> p.stream()).map(p -> p.getPolicyName()).distinct().collect(Collectors.toList()); final List<String> backendPolicies = backendServers.stream().map(s -> s.getPolicyDescriptions()) .flatMap(p -> p.stream()).map(p -> p.getPolicyName()).distinct().collect(Collectors.toList()); final List<String> publicKeyPolicies = lb.getPolicies().stream() .filter(p -> "PublicKeyPolicyType".equals(p.getPolicyTypeName())).map(p -> p.getPolicyName()) .distinct().collect(Collectors.toList()); final List<String> policies = Lists.newArrayList(listenerPolicies); policies.addAll(backendPolicies); policies.addAll(publicKeyPolicies); return policies.stream().distinct().collect(Collectors.toList()); } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to lookup loadbalancer policies", ex); } } @Override public PolicyDescription getLoadBalancerPolicy(final String accountNumber, final String lbName, final String policyName) throws LoadBalancingActivityException { try { final LoadBalancer lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); final LoadBalancerPolicyDescription policy = LoadBalancerPolicies.getLoadBalancerPolicyDescription(lb, policyName); return LoadBalancerPolicies.AsPolicyDescription.INSTANCE.apply(policy); } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to lookup loadbalancer policies", ex); } } // TODO: SCALE @Override public Map<String, LoadBalancerServoDescription> lookupLoadBalancerDescription(final String accountNumber, final String lbName) throws LoadBalancingActivityException { final Map<String, LoadBalancerServoDescription> result = Maps.newHashMap(); try { final LoadBalancer lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); for (final LoadBalancerZoneCoreView zoneView : lb.getZones()) { if (!LoadBalancerZone.STATE.InService.equals(zoneView.getState())) continue; final LoadBalancerServoDescription desc = LoadBalancers.getServoDescription(accountNumber, lbName, zoneView.getName()); final LoadBalancerZone zone = LoadBalancerZoneEntityTransform.INSTANCE.apply(zoneView); for (final LoadBalancerServoInstanceCoreView servoView : zone.getServoInstances()) { result.put(servoView.getInstanceId(), desc); } } } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to lookup loadbalancer descriptions", ex); } return result; } @Override public void deleteListenerRevokeSSLCertificatePolicy(String accountNumber, String lbName, List<Integer> portsToDelete) throws LoadBalancingActivityException { LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (Exception ex) { throw new LoadBalancingActivityException("could not find the loadbalancer", ex); } final Set<String> allArns = Sets.newHashSet(); final Set<String> arnsToKeep = Sets.newHashSet(); for (final LoadBalancerListenerCoreView listener : lb.getListeners()) { final PROTOCOL protocol = listener.getProtocol(); if (protocol.equals(PROTOCOL.HTTPS) || protocol.equals(PROTOCOL.SSL)) { allArns.add(listener.getCertificateId()); if (!portsToDelete.contains(listener.getLoadbalancerPort())) { arnsToKeep.add(listener.getCertificateId()); } } } final Set<String> arnToDelete = Sets.difference(allArns, arnsToKeep); if (arnToDelete.size() <= 0) return; final String roleName = getRoleName(accountNumber, lbName); final String prefix = String.format("arn:aws:iam::%s:server-certificate", accountNumber); for (final String arn : arnToDelete) { if (!arn.startsWith(prefix)) continue; String pathAndName = arn.replace(prefix, ""); String certName = pathAndName.substring(pathAndName.lastIndexOf("/") + 1); String policyName = String.format("%s-%s-%s-%s", SERVER_CERT_ROLE_POLICY_NAME_PREFIX, accountNumber, lbName, certName); try { EucalyptusActivityTasks.getInstance().deleteRolePolicy(roleName, policyName, lb.useSystemAccount()); } catch (final Exception ex) { LOG.warn(String.format("Failed to delete role (%s) policy (%s)", roleName, policyName), ex); } } } @Override public void deleteListenerRevokeIngressRule(String accountNumber, String lbName, List<Integer> portsToDelete) throws LoadBalancingActivityException { LoadBalancer lb; String groupName = null; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); final LoadBalancerSecurityGroupCoreView group = lb.getGroup(); if (group != null) groupName = group.getName(); } catch (Exception ex) { throw new LoadBalancingActivityException("could not find the loadbalancer", ex); } if (groupName == null) { return; } String[] protocols = new String[] { "tcp" }; /// Loadbalancer listeners protocols: HTTP, HTTPS, TCP, SSL -> all tcp for (String protocol : protocols) { for (Integer port : portsToDelete) { try { EucalyptusActivityTasks.getInstance().revokeSystemSecurityGroup(groupName, protocol, port, lb.useSystemAccount()); LOG.debug(String.format("group rule revoked (%s-%d)", groupName, port)); } catch (Exception ex) { LOG.warn("Unable to revoke the security group", ex); } } } } @Override public List<String> disableAvailabilityZonesPersistRetiredServoInstances(String accountNumber, String lbName, List<String> zonesToDisable) throws LoadBalancingActivityException { LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (NoSuchElementException ex) { throw new LoadBalancingActivityException("Could not find the loadbalancer with name=" + lbName, ex); } catch (Exception ex) { throw new LoadBalancingActivityException("Error while looking for loadbalancer with name=" + lbName, ex); } List<String> retiredInstances = Lists.newArrayList(); final List<LoadBalancerZoneCoreView> currentZones = Lists.newArrayList(lb.getZones()); for (final LoadBalancerZoneCoreView zoneView : currentZones) { if (zonesToDisable.contains(zoneView.getName())) { // the zone will be disabled LoadBalancerZone zone; try { zone = LoadBalancerZoneEntityTransform.INSTANCE.apply(zoneView); } catch (final Exception ex) { LOG.error("unable to transform zone from the view", ex); continue; } for (final LoadBalancerServoInstanceCoreView instanceView : zone.getServoInstances()) { LoadBalancerServoInstance instance; try { instance = LoadBalancerServoInstanceEntityTransform.INSTANCE.apply(instanceView); } catch (final Exception ex) { LOG.error("unable to transfrom servo-instance from the view", ex); continue; } try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { final LoadBalancerServoInstance update = Entities.uniqueResult(instance); update.setState(LoadBalancerServoInstance.STATE.Retired); update.setDnsState(LoadBalancerServoInstance.DNS_STATE.Deregistered); Entities.persist(update); db.commit(); retiredInstances.add(update.getInstanceId()); } catch (final NoSuchElementException ex) { LOG.warn("Failed to update the servo instance's state: no such instance found"); } catch (final Exception ex) { LOG.warn("Failed to update the servo instance's state", ex); } } } } return retiredInstances; } @Override public void disableAvailabilityZonesPersistRetiredServoInstancesRollback(String accountNumber, String lbName, List<String> updatedInstanceIds) { if (updatedInstanceIds == null || updatedInstanceIds.size() <= 0) return; for (final String instanceId : updatedInstanceIds) { try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { final LoadBalancerServoInstance sample = LoadBalancerServoInstance.named(instanceId); final LoadBalancerServoInstance update = Entities.uniqueResult(sample); update.setState(LoadBalancerServoInstance.STATE.InService); Entities.persist(update); db.commit(); } catch (final NoSuchElementException ex) { LOG.warn("Failed to update the servo instance's state: no such instance found"); } catch (final Exception ex) { LOG.warn("Failed to update the servo instance's state", ex); } } } @Override public List<String> disableAvailabilityZonesUpdateAutoScalingGroup(String accountNumber, String lbName, List<String> zonesToDisable) throws LoadBalancingActivityException { final List<String> updatedZones = Lists.newArrayList(); LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (NoSuchElementException ex) { throw new LoadBalancingActivityException("Could not find the loadbalancer with name=" + lbName, ex); } catch (Exception ex) { throw new LoadBalancingActivityException("Error while looking for loadbalancer with name=" + lbName, ex); } final Collection<LoadBalancerAutoScalingGroupCoreView> groups = lb.getAutoScaleGroups(); for (final LoadBalancerAutoScalingGroupCoreView group : groups) { if (!zonesToDisable.contains(group.getAvailabilityZone())) continue; final String groupName = group.getName(); final int capacity = 0; try { EucalyptusActivityTasks.getInstance().updateAutoScalingGroup(groupName, null, capacity, lb.useSystemAccount()); } catch (final Exception ex) { LOG.error("Failed to change the capacity of ELB's autoscaling group", ex); } try (final TransactionResource db = Entities.transactionFor(LoadBalancerAutoScalingGroup.class)) { final LoadBalancerAutoScalingGroup update = Entities .uniqueResult(LoadBalancerAutoScalingGroup.named(lb, group.getAvailabilityZone())); update.setCapacity(capacity); Entities.persist(update); db.commit(); } catch (NoSuchElementException ex) { LOG.error("failed to find the autoscaling group record", ex); } catch (Exception ex) { LOG.error("failed to update the autoscaling group record", ex); } updatedZones.add(group.getAvailabilityZone()); } return updatedZones; } @Override public void disableAvailabilityZonesUpdateAutoScalingGroupRollback(String accountNumber, String lbName, List<String> updatedZones) { if (updatedZones == null || updatedZones.size() <= 0) return; LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (NoSuchElementException ex) { LOG.error("Could not find the loadbalancer with name=" + lbName, ex); return; } catch (Exception ex) { LOG.error("Error while looking for loadbalancer with name=" + lbName, ex); return; } final Collection<LoadBalancerAutoScalingGroupCoreView> groups = lb.getAutoScaleGroups(); for (final LoadBalancerAutoScalingGroupCoreView group : groups) { if (!updatedZones.contains(group.getAvailabilityZone())) continue; final String groupName = group.getName(); final int capacity = LoadBalancingServiceProperties.getCapacityPerZone(); try { EucalyptusActivityTasks.getInstance().updateAutoScalingGroup(groupName, null, capacity, lb.useSystemAccount()); } catch (final Exception ex) { LOG.error("Failed to change the capacity of ELB's autoscaling group", ex); } try (final TransactionResource db = Entities.transactionFor(LoadBalancerAutoScalingGroup.class)) { final LoadBalancerAutoScalingGroup update = Entities .uniqueResult(LoadBalancerAutoScalingGroup.named(lb, group.getAvailabilityZone())); update.setCapacity(capacity); Entities.persist(update); db.commit(); } catch (NoSuchElementException ex) { LOG.error("failed to find the autoscaling group record", ex); } catch (Exception ex) { LOG.error("failed to update the autoscaling group record", ex); } } } @Override public void disableAvailabilityZonesPersistUpdatedZones(String accountNumber, String lbName, List<String> zonesToDisable) throws LoadBalancingActivityException { LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (NoSuchElementException ex) { throw new LoadBalancingActivityException("Could not find the loadbalancer with name=" + lbName, ex); } catch (Exception ex) { throw new LoadBalancingActivityException("Error while looking for loadbalancer with name=" + lbName, ex); } if (zonesToDisable == null) return; for (final String zone : zonesToDisable) { try (final TransactionResource db = Entities.transactionFor(LoadBalancerZone.class)) { final LoadBalancerZone update = Entities.uniqueResult(LoadBalancerZone.named(lb, zone)); update.setState(LoadBalancerZone.STATE.OutOfService); db.commit(); } catch (final Exception ex) { LOG.debug("Error updating state for load balancer zone", ex); } } } @Override public void disableAvailabilityZonesPersistBackendInstanceState(String accountNumber, String lbName, List<String> zonesToDisable) throws LoadBalancingActivityException { LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (NoSuchElementException ex) { throw new LoadBalancingActivityException("Could not find the loadbalancer with name=" + lbName, ex); } catch (Exception ex) { throw new LoadBalancingActivityException("Error while looking for loadbalancer with name=" + lbName, ex); } if (zonesToDisable == null || zonesToDisable.size() <= 0) return; for (final String removedZone : zonesToDisable) { final LoadBalancerZone zone = LoadBalancers.findZone(lb, removedZone); for (final LoadBalancerBackendInstanceCoreView instance : zone.getBackendInstances()) { try (TransactionResource db = Entities.transactionFor(LoadBalancerBackendInstance.class)) { final LoadBalancerBackendInstance update = Entities .uniqueResult(LoadBalancerBackendInstance.named(lb, instance.getInstanceId())); final LoadBalancerBackendInstanceStates azDisabled = LoadBalancerBackendInstanceStates.AvailabilityZoneDisabled; update.setState(azDisabled.getState()); update.setReasonCode(azDisabled.getReasonCode()); update.setDescription(azDisabled.getDescription()); Entities.persist(update); db.commit(); } catch (final NoSuchElementException ex) { LOG.warn("failed to find the backend instance"); } catch (final Exception ex) { LOG.warn("failed to query the backend instance", ex); } } } } @Override public void deleteLoadBalancerDeactivateDns(String accountNumber, String lbName) { LoadBalancer lb; final List<LoadBalancerServoInstanceCoreView> servos = Lists.newArrayList(); try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); if (lb.getZones() != null) { for (final LoadBalancerZoneCoreView zoneView : lb.getZones()) { LoadBalancerZone zone; try { zone = LoadBalancerZoneEntityTransform.INSTANCE.apply(zoneView); } catch (final Exception ex) { continue; } servos.addAll(zone.getServoInstances()); } } } catch (NoSuchElementException ex) { return; } catch (Exception ex) { LOG.warn("Failed to find the loadbalancer", ex); return; } for (final LoadBalancerServoInstanceCoreView instance : servos) { final String address = instance.getAddress(); if (address == null || address.length() <= 0) continue; try { try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { try { final LoadBalancerServoInstance entity = Entities .uniqueResult(LoadBalancerServoInstance.named(instance.getInstanceId())); entity.setDnsState(LoadBalancerServoInstance.DNS_STATE.Deregistered); Entities.persist(entity); db.commit(); } catch (final Exception ex) { LOG.error(String.format("failed to set servo instance(%s)'s dns state to deregistered", instance.getInstanceId()), ex); } } } catch (Exception ex) { LOG.error("Error updating DNS registration state for balancer " + lbName, ex); } } } @Override public void deleteLoadBalancerDeleteScalingGroup(String accountNumber, String lbName) throws LoadBalancingActivityException { LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (NoSuchElementException ex) { return; } catch (Exception ex) { LOG.warn("Failed to find the loadbalancer named " + lbName, ex); return; } final Collection<LoadBalancerAutoScalingGroupCoreView> groups = lb.getAutoScaleGroups(); if (groups == null || groups.isEmpty()) { LOG.warn(String.format("Loadbalancer %s had no autoscale group associated with it", lb.getDisplayName())); return; } for (final LoadBalancerAutoScalingGroupCoreView group : groups) { final String groupName = group.getName(); String launchConfigName = null; try { final DescribeAutoScalingGroupsResponseType resp = EucalyptusActivityTasks.getInstance() .describeAutoScalingGroups(Lists.newArrayList(groupName), lb.useSystemAccount()); final AutoScalingGroupType asgType = resp.getDescribeAutoScalingGroupsResult() .getAutoScalingGroups().getMember().get(0); launchConfigName = asgType.getLaunchConfigurationName(); } catch (final Exception ex) { LOG.warn(String.format("Unable to find the launch config associated with %s", groupName)); } try { EucalyptusActivityTasks.getInstance().updateAutoScalingGroup(groupName, null, 0, lb.useSystemAccount()); } catch (final Exception ex) { LOG.warn(String.format("Unable to set desired capacity for %s", groupName), ex); } boolean error = false; final int NUM_DELETE_ASG_RETRY = 4; for (int i = 0; i < NUM_DELETE_ASG_RETRY; i++) { try { EucalyptusActivityTasks.getInstance().deleteAutoScalingGroup(groupName, true, lb.useSystemAccount()); error = false; // will terminate all instances } catch (final Exception ex) { error = true; LOG.warn(String.format("Failed to delete autoscale group (%d'th attempt): %s", (i + 1), groupName)); try { long sleepMs = (i + 1) * 500; Thread.sleep(sleepMs); } catch (final Exception ex2) { } } if (!error) break; } if (error) { throw new LoadBalancingActivityException( "Failed to delete autoscaling group; retry in a few seconds"); } if (launchConfigName != null) { try { EucalyptusActivityTasks.getInstance().deleteLaunchConfiguration(launchConfigName, lb.useSystemAccount()); } catch (Exception ex) { LOG.warn("Failed to delete launch configuration " + launchConfigName, ex); } } LoadBalancerAutoScalingGroup scaleGroup = null; try { scaleGroup = LoadBalancerAutoScalingGroupEntityTransform.INSTANCE.apply(group); } catch (final Exception ex) { LOG.error("falied to update servo instance record", ex); } if (scaleGroup == null) return; try (TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { for (final LoadBalancerServoInstanceCoreView instanceView : scaleGroup.getServos()) { LoadBalancerServoInstance instance; try { instance = LoadBalancerServoInstanceEntityTransform.INSTANCE.apply(instanceView); } catch (final Exception ex) { continue; } final LoadBalancerServoInstance found = Entities.uniqueResult(instance); found.setAvailabilityZone(null); found.setAutoScalingGroup(null); // InService --> Retired // Pending --> Retired // OutOfService --> Retired // Error --> Retired found.setState(LoadBalancerServoInstance.STATE.Retired); Entities.persist(found); } db.commit(); } catch (final Exception ex) { LOG.error("Failed to update servo instance record", ex); } } // AutoScalingGroup record will be deleted later by clean-up workflow } @Override public void deleteLoadBalancerDeleteInstanceProfile(String accountNumber, String lbName) { final String instanceProfileName = getInstanceProfileName(accountNumber, lbName); final String roleName = getRoleName(accountNumber, lbName); LoadBalancer lb = null; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (NoSuchElementException ex) { return; } catch (Exception ex) { LOG.warn("Failed to find the loadbalancer named " + lbName, ex); return; } try { EucalyptusActivityTasks.getInstance().removeRoleFromInstanceProfile(instanceProfileName, roleName, lb.useSystemAccount()); } catch (final Exception ex) { LOG.error(String.format("Failed to remove role(%s) from the instance profile(%s)", roleName, instanceProfileName), ex); } // remove instance profile try { EucalyptusActivityTasks.getInstance().deleteInstanceProfile(instanceProfileName, lb.useSystemAccount()); } catch (final Exception ex) { LOG.error(String.format("Failed to delete instance profile (%s)", instanceProfileName), ex); } } @Override public void deleteLoadBalancerDeleteIamRole(String accountNumber, String lbName) { final String roleName = getRoleName(accountNumber, lbName); LoadBalancer lb = null; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (NoSuchElementException ex) { return; } catch (Exception ex) { LOG.warn("Failed to find the loadbalancer named " + lbName, ex); return; } List<String> rolePolicies = null; try { rolePolicies = EucalyptusActivityTasks.getInstance().listRolePolicies(roleName); } catch (final Exception ex) { LOG.warn("Failed to list role policies to delete", ex); } if (rolePolicies != null) { for (final String policy : rolePolicies) { // delete role policy try { EucalyptusActivityTasks.getInstance().deleteRolePolicy(roleName, policy, lb.useSystemAccount()); } catch (final Exception ex) { LOG.error("Failed to delete role policy: " + policy, ex); } } } // delete role try { EucalyptusActivityTasks.getInstance().deleteRole(roleName, lb.useSystemAccount()); } catch (final Exception ex) { LOG.error("failed to delete role: " + roleName, ex); } } @Override public void deleteLoadBalancerDeleteSecurityGroup(String accountNumber, String lbName) { LoadBalancer lb; LoadBalancerSecurityGroupCoreView groupView = null; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); if (lb.getGroup() != null) { groupView = lb.getGroup(); } } catch (NoSuchElementException ex) { return; } catch (Exception ex) { LOG.error("Error while looking for loadbalancer with name=" + lbName, ex); return; } if (lb.getVpcId() == null) { LoadBalancerSecurityGroup group; try { group = LoadBalancerSecurityGroupEntityTransform.INSTANCE.apply(groupView); } catch (final Exception ex) { LOG.error("Erorr while looking for loadbalancer group", ex); return; } try (TransactionResource db = Entities.transactionFor(LoadBalancerSecurityGroup.class)) { final LoadBalancerSecurityGroup update = Entities.uniqueResult(group); update.setLoadBalancer(null); // this allows the loadbalancer to be deleted update.setState(LoadBalancerSecurityGroup.STATE.OutOfService); Entities.persist(update); db.commit(); } catch (Exception ex) { LOG.warn("Could not disassociate the group from loadbalancer"); } // the actual security group is delete during the clean-up workflow } } private static EucaS3Client getS3Client(final String roleName) throws AuthException { try { final Role lbRole = Accounts.lookupRoleByName( Accounts.lookupAccountIdByAlias(AccountIdentifiers.ELB_SYSTEM_ACCOUNT), roleName); final SecurityTokenAWSCredentialsProvider roleCredentialProvider = SecurityTokenAWSCredentialsProvider .forUserOrRole(Accounts.lookupPrincipalByRoleId(lbRole.getRoleId())); return EucaS3ClientFactory.getEucaS3Client(roleCredentialProvider); } catch (AuthException ex) { LOG.error("Failed to get credentials for loadbalancing role", ex); } catch (Exception ex) { LOG.error("Failed to get credentials for loadbalancing role", ex); } return null; } final String ACCESSLOG_ROLE_POLICY_NAME = "euca-internal-loadbalancer-vm-policy-accesslog"; @Override public AccessLogPolicyActivityResult modifyLoadBalancerAttributesCreateAccessLogPolicy( final String accountNumber, final String lbName, final Boolean accessLogEnabled, final String s3BucketName, final String s3BucketPrefix, final Integer emitInterval) throws LoadBalancingActivityException { final String ACCESSLOG_ROLE_POLICY_DOCUMENT = "{\"Statement\":" + "[ {" + "\"Action\": [\"s3:PutObject\"]," + "\"Effect\": \"Allow\"," + "\"Resource\": [\"arn:aws:s3:::BUCKETNAME_PLACEHOLDER/BUCKETPREFIX_PLACEHOLDER\"]" + "}]}"; AccessLogPolicyActivityResult result = new AccessLogPolicyActivityResult(); result.setShouldRollback(false); if (!accessLogEnabled) return result; final String bucketName = s3BucketName; final String bucketPrefix = com.google.common.base.Objects.firstNonNull(s3BucketPrefix, ""); final String roleName = getRoleName(accountNumber, lbName); final String policyName = ACCESSLOG_ROLE_POLICY_NAME; try { final List<String> policies = EucalyptusActivityTasks.getInstance().listRolePolicies(roleName); if (policies.contains(policyName)) { EucalyptusActivityTasks.getInstance().deleteRolePolicy(roleName, policyName); } } catch (final Exception ex) { ; } String policyDocument = ACCESSLOG_ROLE_POLICY_DOCUMENT.replace("BUCKETNAME_PLACEHOLDER", bucketName); if (bucketPrefix.length() > 0) { policyDocument = policyDocument.replace("BUCKETPREFIX_PLACEHOLDER", bucketPrefix + "/*"); } else { policyDocument = policyDocument.replace("BUCKETPREFIX_PLACEHOLDER", "*"); } try { EucalyptusActivityTasks.getInstance().putRolePolicy(roleName, policyName, policyDocument); result.setRoleName(roleName); result.setPolicyName(policyName); result.setShouldRollback(true); } catch (final Exception ex) { throw new LoadBalancingActivityException( "failed to put role policy for loadbalancer vm's access to S3 buckets"); } try { final EucaS3Client s3c = getS3Client(roleName); final String key = s3BucketPrefix != null && !s3BucketPrefix.isEmpty() ? String.format("%s/AWSLogs/%s/ELBAccessLogTestFile", s3BucketPrefix, accountNumber) : String.format("AWSLogs/%s/ELBAccessLogTestFile", accountNumber); final DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS"); final String content = String.format("Enable AccessLog for ELB: %s at %s", lbName, df.format(new Date())); final PutObjectRequest req = new PutObjectRequest(bucketName, key, new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)), new ObjectMetadata()) .withCannedAcl(CannedAccessControlList.BucketOwnerFullControl); s3c.putObject(req); } catch (final Exception ex) { LOG.warn("Failed to put test key to the access log bucket"); } return result; } @Override public void modifyLoadBalancerAttributesCreateAccessLogPolicyRollback(final String accountNumber, final String lbName, final AccessLogPolicyActivityResult result) { if (!result.getShouldRollback()) return; try { EucalyptusActivityTasks.getInstance().deleteRolePolicy(result.getRoleName(), result.getPolicyName()); } catch (final Exception ex) { ; } } @Override public void modifyLoadBalancerAttributesDeleteAccessLogPolicy(final String accountNumber, final String lbName, final Boolean accessLogEnabled, final String s3BucketName, final String s3BucketPrefix, final Integer emitInterval) throws LoadBalancingActivityException { if (accessLogEnabled) return; final String roleName = getRoleName(accountNumber, lbName); final String policyName = ACCESSLOG_ROLE_POLICY_NAME; try { EucalyptusActivityTasks.getInstance().deleteRolePolicy(roleName, policyName); } catch (final Exception ex) { ; } } @Override public void modifyLoadBalancerAttributesPersistAttributes(final String accountNumber, final String lbName, final Boolean accessLogEnabled, final String s3BucketName, final String s3BucketPrefix, Integer emitInterval) throws LoadBalancingActivityException { String bucketName = null; String bucketPrefix = null; if (accessLogEnabled) { bucketName = s3BucketName; bucketPrefix = com.google.common.base.Objects.firstNonNull(s3BucketPrefix, ""); emitInterval = com.google.common.base.Objects.firstNonNull(emitInterval, 60); } else { bucketName = ""; bucketPrefix = ""; emitInterval = 60; } try (final TransactionResource db = Entities.transactionFor(LoadBalancer.class)) { final LoadBalancer lb = Entities.uniqueResult(LoadBalancer.namedByAccountId(accountNumber, lbName)); lb.setAccessLogEnabled(accessLogEnabled); lb.setAccessLogEmitInterval(emitInterval); lb.setAccessLogS3BucketName(bucketName); lb.setAccessLogS3BucketPrefix(bucketPrefix); Entities.persist(lb); db.commit(); } catch (final NoSuchElementException ex) { throw new LoadBalancingActivityException("No such loadbalancer is found"); } catch (final Exception ex) { throw new LoadBalancingActivityException("Unknown error occured while saving entity", ex); } } private String lookupSecondaryNetworkInterface(final String instanceId) { try { final Optional<InstanceNetworkInterfaceSetItemType> optEni = LoadBalancingSystemVpcs .getUserVpcInterface(instanceId); if (optEni.isPresent()) { return optEni.get().getNetworkInterfaceId(); } return null; } catch (final Exception ex) { LOG.error("Failed to lookup secondary network interface for instance: " + instanceId); return null; } } @Override public void modifyServicePropertiesValidateRequest(final String emi, final String instanceType, final String keyname, final String initScript) throws LoadBalancingActivityException { if (emi != null) { try { final List<ImageDetails> images = EucalyptusActivityTasks.getInstance() .describeImagesWithVerbose(Lists.newArrayList(emi)); if (images == null || images.size() <= 0) throw new LoadBalancingActivityException("No such EMI is found in the system"); if (!images.get(0).getImageId().toLowerCase().equals(emi.toLowerCase())) throw new LoadBalancingActivityException("No such EMI is found in the system"); } catch (final LoadBalancingActivityException ex) { throw ex; } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to verify EMI in the system"); } } // validate instance type if (instanceType != null) { try { final List<VmTypeDetails> vmTypes = EucalyptusActivityTasks.getInstance() .describeInstanceTypes(Lists.newArrayList(instanceType)); if (vmTypes.size() <= 0) throw new LoadBalancingActivityException("Invalid instance type -- " + instanceType); } catch (final LoadBalancingActivityException ex) { throw ex; } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to verify instance type -- " + instanceType); } } } @Override public void applySecurityGroupUpdateSecurityGroup(final String accountNumber, final String lbName, final Map<String, String> groupIdToNames) throws LoadBalancingActivityException { final LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (NoSuchElementException ex) { throw new LoadBalancingActivityException("Failed to find the loadbalancer " + lbName, ex); } catch (Exception ex) { throw new LoadBalancingActivityException("Unable to access loadbalancer metadata", ex); } for (final LoadBalancerAutoScalingGroupCoreView group : lb.getAutoScaleGroups()) { final String groupName = group.getName(); final DescribeAutoScalingGroupsResponseType response = EucalyptusActivityTasks.getInstance() .describeAutoScalingGroups(Lists.newArrayList(groupName), lb.useSystemAccount()); final DescribeAutoScalingGroupsResult describeAutoScalingGroupsResult = response .getDescribeAutoScalingGroupsResult(); if (describeAutoScalingGroupsResult != null) { final AutoScalingGroupsType autoScalingGroupsType = describeAutoScalingGroupsResult .getAutoScalingGroups(); if (autoScalingGroupsType != null && autoScalingGroupsType.getMember() != null && !autoScalingGroupsType.getMember().isEmpty() && autoScalingGroupsType.getMember().get(0).getInstances() != null) { for (final Instance instance : autoScalingGroupsType.getMember().get(0).getInstances() .getMember()) { final String userVpcEni = lookupSecondaryNetworkInterface(instance.getInstanceId()); if (userVpcEni == null) { throw new LoadBalancingActivityException("Failed to lookup user VPC network interface"); } try { final List<String> sgroupIds = Lists.newArrayList(groupIdToNames.keySet()); EucalyptusActivityTasks.getInstance().modifyNetworkInterfaceSecurityGroups(userVpcEni, sgroupIds); } catch (final Exception ex) { throw new LoadBalancingActivityException( "Failed to set security groups to network interface", ex); } } } } } } @Override public void modifyServicePropertiesUpdateScalingGroup(final String emi, final String instanceType, final String keyname, final String initScript) throws LoadBalancingActivityException { final List<LoadBalancer> lbs = LoadBalancers.listLoadbalancers(); for (final LoadBalancer lb : lbs) { final Collection<LoadBalancerAutoScalingGroupCoreView> groups = lb.getAutoScaleGroups(); for (final LoadBalancerAutoScalingGroupCoreView asg : groups) { if (asg == null || asg.getName() == null) continue; final String asgName = asg.getName(); try { AutoScalingGroupType asgType = null; try { final DescribeAutoScalingGroupsResponseType resp = EucalyptusActivityTasks.getInstance() .describeAutoScalingGroups(Lists.newArrayList(asgName), lb.useSystemAccount()); if (resp.getDescribeAutoScalingGroupsResult() != null && resp.getDescribeAutoScalingGroupsResult().getAutoScalingGroups() != null && resp.getDescribeAutoScalingGroupsResult().getAutoScalingGroups() .getMember() != null && resp.getDescribeAutoScalingGroupsResult().getAutoScalingGroups().getMember() .size() > 0) { asgType = resp.getDescribeAutoScalingGroupsResult().getAutoScalingGroups().getMember() .get(0); } } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to find the scaling group: " + asgName); } if (asgType != null) { final LaunchConfigurationType lc = EucalyptusActivityTasks.getInstance() .describeLaunchConfiguration(asgType.getLaunchConfigurationName(), lb.useSystemAccount()); String launchConfigName; do { launchConfigName = getLaunchConfigName(lb.getOwnerAccountNumber(), lb.getDisplayName(), asg.getAvailabilityZone()); } while (launchConfigName.equals(asgType.getLaunchConfigurationName())); final String newEmi = emi != null ? emi : lc.getImageId(); final String newType = instanceType != null ? instanceType : lc.getInstanceType(); String newKeyname = keyname != null ? keyname : lc.getKeyName(); final String newUserdata = B64.standard .encString(String.format("%s\n%s", getCredentialsString(), getLoadBalancerUserData(initScript, lb.getOwnerAccountNumber()))); try { EucalyptusActivityTasks.getInstance().createLaunchConfiguration(newEmi, newType, lc.getIamInstanceProfile(), launchConfigName, lc.getSecurityGroups().getMember(), newKeyname, newUserdata, lc.getAssociatePublicIpAddress(), lb.useSystemAccount()); } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to create new launch config", ex); } try { EucalyptusActivityTasks.getInstance().updateAutoScalingGroup(asgName, null, asgType.getDesiredCapacity(), launchConfigName, lb.useSystemAccount()); } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to update the autoscaling group", ex); } try { EucalyptusActivityTasks.getInstance().deleteLaunchConfiguration( asgType.getLaunchConfigurationName(), lb.useSystemAccount()); } catch (final Exception ex) { LOG.warn("unable to delete the old launch configuration", ex); } // copy all tags from new image to ASG if (emi != null) { try { final List<ImageDetails> images = EucalyptusActivityTasks.getInstance() .describeImagesWithVerbose(Lists.newArrayList(emi)); // image should exist at this point for (ResourceTag tag : images.get(0).getTagSet()) { EucalyptusActivityTasks.getInstance().createOrUpdateAutoscalingTags( tag.getKey(), tag.getValue(), asgName, lb.useSystemAccount()); } } catch (final Exception ex) { LOG.warn("unable to propagate tags from image to ASG", ex); } } LOG.debug(String.format("autoscaling group '%s' was updated", asgName)); } } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to apply ELB property changes", ex); } } // for all autoscaling groups of LB } // for all LBs } private LoadBalancerServoInstance newInstance(final Instance instance, final LoadBalancerAutoScalingGroup group) throws Exception { final String instanceId = instance.getInstanceId(); final LoadBalancerCoreView lbView = group.getLoadBalancer(); LoadBalancer lb; try { lb = LoadBalancerEntityTransform.INSTANCE.apply(lbView); } catch (final Exception ex) { LOG.error("unable to transfrom loadbalancer from the viewer", ex); throw ex; } LoadBalancerZoneCoreView zoneView = null; for (final LoadBalancerZoneCoreView z : lb.getZones()) { if (z.getName().equals(instance.getAvailabilityZone())) { zoneView = z; break; } } if (zoneView == null) throw new Exception("No availability zone with name=" + instance.getAvailabilityZone() + " found for loadbalancer " + lb.getDisplayName()); final LoadBalancerSecurityGroupCoreView sgroupView = lb.getGroup(); if (sgroupView == null && lb.getVpcId() == null) throw new Exception("No security group is found for loadbalancer " + lb.getDisplayName()); LoadBalancerZone zone; LoadBalancerSecurityGroup sgroup; try { zone = LoadBalancerZoneEntityTransform.INSTANCE.apply(zoneView); sgroup = sgroupView == null ? null : LoadBalancerSecurityGroupEntityTransform.INSTANCE.apply(sgroupView); } catch (final Exception ex) { LOG.error("Unable to transform entity", ex); throw ex; } // for faster inclusion into DNS response, update status as soon as servo is running String ipAddr = null; String privateIpAddr = null; try { List<RunningInstancesItemType> result = null; result = EucalyptusActivityTasks.getInstance() .describeSystemInstancesWithVerbose(Lists.newArrayList(instance.getInstanceId())); if (result != null && result.size() > 0) { ipAddr = result.get(0).getIpAddress(); privateIpAddr = result.get(0).getPrivateIpAddress(); } } catch (Exception ex) { LOG.warn("failed to run describe-instances", ex); } final LoadBalancerServoInstance newInstance = LoadBalancerServoInstance.newInstance(zone, sgroup, group, Integer.parseInt(LoadBalancingWorkerProperties.EXPIRATION_DAYS), instanceId); if ("Healthy".equals(instance.getHealthStatus()) && "InService".equals(instance.getLifecycleState())) newInstance.setState(LoadBalancerServoInstance.STATE.InService); newInstance.setAddress(ipAddr); newInstance.setPrivateIp(privateIpAddr); if (!(ipAddr == null && privateIpAddr == null)) newInstance.setDnsState(LoadBalancerServoInstance.DNS_STATE.Registered); return newInstance; } private void updateIpAddressesInVpc(final String instanceId) { final List<Optional<String>> userVpcInterfaceAddresses = LoadBalancingSystemVpcs .getUserVpcInterfaceIps(instanceId); if (userVpcInterfaceAddresses != null) { final Optional<String> publicIp = userVpcInterfaceAddresses.get(0); final Optional<String> privateIp = userVpcInterfaceAddresses.get(1); try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { final LoadBalancerServoInstance update = Entities .uniqueResult(LoadBalancerServoInstance.named(instanceId)); if (publicIp.isPresent()) update.setAddress(publicIp.get()); if (privateIp.isPresent()) update.setPrivateIp(privateIp.get()); Entities.persist(update); db.commit(); } catch (final Exception ex) { LOG.error("Failed to update instance's IP addresses", ex); } } } @Override public void checkServoInstances() throws LoadBalancingActivityException { final int NUM_ASGS_TO_DESCRIBE = 8; // lookup all LoadBalancerAutoScalingGroup records List<LoadBalancerAutoScalingGroup> groups = Lists.newArrayList(); Map<String, LoadBalancerAutoScalingGroup> allGroupMap = new ConcurrentHashMap<>(); try (final TransactionResource db = Entities.transactionFor(LoadBalancerAutoScalingGroup.class)) { groups = Entities.query(LoadBalancerAutoScalingGroup.named(), true); for (LoadBalancerAutoScalingGroup g : groups) { allGroupMap.put(g.getName(), g); } } catch (Exception ex) { throw new LoadBalancingActivityException("Failed to query loadbalancer autoscaing groups", ex); } final Map<String, LoadBalancerAutoScalingGroup> groupToQuery = allGroupMap; // describe as group and find the unknown instance Ids List<AutoScalingGroupType> queriedGroups = Lists.newArrayList(); for (final List<String> partition : Iterables.partition(groupToQuery.keySet(), NUM_ASGS_TO_DESCRIBE)) { try { DescribeAutoScalingGroupsResponseType response = EucalyptusActivityTasks.getInstance() .describeAutoScalingGroupsWithVerbose(partition); DescribeAutoScalingGroupsResult result = response.getDescribeAutoScalingGroupsResult(); AutoScalingGroupsType asgroups = result.getAutoScalingGroups(); queriedGroups.addAll(asgroups.getMember()); } catch (Exception ex) { throw new LoadBalancingActivityException("Failed to describe autoscaling groups", ex); } } /// lookup all servoInstances in the DB Map<String, LoadBalancerServoInstance> servoMap = new ConcurrentHashMap<>(); try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { final List<LoadBalancerServoInstance> result = Entities.query(LoadBalancerServoInstance.named(), true); for (LoadBalancerServoInstance inst : result) { servoMap.put(inst.getInstanceId(), inst); } } catch (Exception ex) { throw new LoadBalancingActivityException("Failed to lookup existing servo instances in DB", ex); } /// for all found instances that's not in the servo instance DB /// create servo record final List<LoadBalancerServoInstance> newServos = Lists.newArrayList(); final Map<String, Instance> foundInstances = new ConcurrentHashMap<>(); for (final AutoScalingGroupType asg : queriedGroups) { Instances instances = asg.getInstances(); if (instances != null && instances.getMember() != null && instances.getMember().size() > 0) { for (final Instance instance : instances.getMember()) { final String instanceId = instance.getInstanceId(); foundInstances.put(instanceId, instance); if (!servoMap.containsKey(instanceId)) { /// new instance found try { final LoadBalancerAutoScalingGroup group = allGroupMap .get(asg.getAutoScalingGroupName()); if (group == null) throw new IllegalArgumentException("The group with name " + asg.getAutoScalingGroupName() + " not found in the database"); final LoadBalancerServoInstance newInstance = newInstance(instance, group); newServos.add(newInstance); /// persist later } catch (final Exception ex) { LOG.error("Failed to construct servo instance entity", ex); continue; } } } } } // CASE 1: NEW INSTANCES WITHIN THE AS GROUP FOUND if (newServos.size() > 0) { try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { for (LoadBalancerServoInstance instance : newServos) { Entities.persist(instance); } db.commit(); } catch (Exception ex) { LOG.error("Failed to persist the servo instance record", ex); } if (LoadBalancingSystemVpcs.isCloudVpc().isPresent() && LoadBalancingSystemVpcs.isCloudVpc().get()) { try { newServos.stream().filter( instance -> LoadBalancerServoInstance.STATE.InService.equals(instance.getState())) .forEach(instance -> LoadBalancingSystemVpcs .setupUserVpcInterface(instance.getInstanceId())); } catch (final Exception ex) { LOG.error("Failed to attach secondary network interface to ELB instances", ex); } try { // if servo is in VPC, update ip addresses using the secondary interface's address newServos.stream().filter( instance -> LoadBalancerServoInstance.STATE.InService.equals(instance.getState())) .forEach(instance -> { updateIpAddressesInVpc(instance.getInstanceId()); }); } catch (final Exception ex) { LOG.error("Failed to retrieve IP addresses of secondary network interface"); } } } List<LoadBalancerServoInstanceCoreView> servoRecords = Lists.newArrayList(); for (String groupName : groupToQuery.keySet()) { final LoadBalancerAutoScalingGroup group = groupToQuery.get(groupName); servoRecords.addAll(group.getServos()); } //final List<LoadBalancerServoInstance> registerDnsARec = Lists.newArrayList(); for (LoadBalancerServoInstanceCoreView instanceView : servoRecords) { /// CASE 2: EXISTING SERVO INSTANCES ARE NOT FOUND IN THE ASG QUERY RESPONSE if (!foundInstances.containsKey(instanceView.getInstanceId()) && !instanceView.getState().equals(LoadBalancerServoInstance.STATE.Retired)) { LoadBalancerServoInstance instance; try { instance = LoadBalancerServoInstanceEntityTransform.INSTANCE.apply(instanceView); } catch (final Exception ex) { LOG.error("Failed to transform servo instance from the view", ex); continue; } try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { final LoadBalancerServoInstance update = Entities.uniqueResult(instance); update.setState(LoadBalancerServoInstance.STATE.Error); Entities.persist(update); db.commit(); } catch (Exception ex) { LOG.error(String.format("Failed to mark the servo instance's state to ERROR (%s)", instance.getInstanceId())); } } else if (foundInstances.containsKey(instanceView.getInstanceId())) { /// CASE 3: INSTANCE STATE UPDATED Instance instanceCurrent = foundInstances.get(instanceView.getInstanceId()); final String healthState = instanceCurrent.getHealthStatus(); final String lifecycleState = instanceCurrent.getLifecycleState(); LoadBalancerServoInstance.STATE curState = instanceView.getState(); LoadBalancerServoInstance.STATE newState = curState; if (healthState != null && !healthState.equals("Healthy")) { newState = LoadBalancerServoInstance.STATE.Error; } else if (lifecycleState != null) { switch (lifecycleState) { case "Pending": newState = LoadBalancerServoInstance.STATE.Pending; break; case "Quarantined": newState = LoadBalancerServoInstance.STATE.Error; break; case "InService": newState = LoadBalancerServoInstance.STATE.InService; break; case "Terminating": case "Terminated": newState = LoadBalancerServoInstance.STATE.OutOfService; break; } } if (!curState.equals(LoadBalancerServoInstance.STATE.Retired) && !curState.equals(newState)) { LoadBalancerServoInstance instance; try { instance = LoadBalancerServoInstanceEntityTransform.INSTANCE.apply(instanceView); } catch (final Exception ex) { LOG.error("Failed to transform servo instance from the view", ex); continue; } try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { final LoadBalancerServoInstance update = Entities.uniqueResult(instance); update.setState(newState); Entities.persist(update); db.commit(); } catch (Exception ex) { LOG.error(String.format("Failed to commit the servo instance's state change (%s)", instance.getInstanceId())); } if (LoadBalancerServoInstance.STATE.InService.equals(newState)) { try { if (LoadBalancingSystemVpcs.isCloudVpc().isPresent() && LoadBalancingSystemVpcs.isCloudVpc().get()) { LoadBalancingSystemVpcs.setupUserVpcInterface(instance.getInstanceId()); updateIpAddressesInVpc(instance.getInstanceId()); } } catch (final Exception ex) { LOG.error("Failed to attach secondary network interface to ELB instances", ex); } } } } } } // make sure InService servo instance has its IP registered to DNS // also make sure Error or OutOfService servo instance has its IP deregistered from DNS @Override public void checkServoInstanceDns() throws LoadBalancingActivityException { /// determine the servo instances to query final List<LoadBalancerServoInstance> allInstances = Lists.newArrayList(); //final List<LoadBalancerServoInstance> stateOutdated = Lists.newArrayList(); try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { allInstances.addAll(Entities.query(LoadBalancerServoInstance.named())); } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to query servo instances in DB"); } final List<LoadBalancerServoInstance> stateOutdated = allInstances; for (final LoadBalancerServoInstance instance : stateOutdated) { if (LoadBalancerServoInstance.STATE.InService.equals(instance.getState())) { if (!LoadBalancerServoInstance.DNS_STATE.Registered.equals(instance.getDnsState())) { String ipAddr = null; String privateIpAddr = null; final Optional<Boolean> vpcTest = LoadBalancingSystemVpcs.isCloudVpc(); if (vpcTest.isPresent() && vpcTest.get()) { /// in vpc mode final List<Optional<String>> userVpcInterfaceAddresses = LoadBalancingSystemVpcs .getUserVpcInterfaceIps(instance.getInstanceId()); if (userVpcInterfaceAddresses != null) { final Optional<String> optPublicIp = userVpcInterfaceAddresses.get(0); final Optional<String> optPrivateIp = userVpcInterfaceAddresses.get(1); if (optPublicIp.isPresent()) ipAddr = optPublicIp.get(); if (optPrivateIp.isPresent()) privateIpAddr = optPrivateIp.get(); } } else { /// in classic mode if (instance.getAddress() == null) { try { List<RunningInstancesItemType> result = null; result = EucalyptusActivityTasks.getInstance().describeSystemInstancesWithVerbose( Lists.newArrayList(instance.getInstanceId())); if (result != null && result.size() > 0) { ipAddr = result.get(0).getIpAddress(); privateIpAddr = result.get(0).getPrivateIpAddress(); } } catch (Exception ex) { LOG.warn("Failed to run describe-instances", ex); continue; } } else { ipAddr = instance.getAddress(); privateIpAddr = instance.getPrivateIp(); } } try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { final LoadBalancerServoInstance update = Entities.uniqueResult(instance); update.setAddress(ipAddr); update.setPrivateIp(privateIpAddr); if (!(ipAddr == null && privateIpAddr == null)) update.setDnsState(LoadBalancerServoInstance.DNS_STATE.Registered); db.commit(); } catch (NoSuchElementException ex) { LOG.warn("Failed to find the servo instance named " + instance.getInstanceId(), ex); } catch (Exception ex) { LOG.warn("Failed to update servo instance's ip address", ex); } } } else if (LoadBalancerServoInstance.STATE.OutOfService.equals(instance.getState()) || LoadBalancerServoInstance.STATE.Error.equals(instance.getState())) { if (!LoadBalancerServoInstance.DNS_STATE.Deregistered.equals(instance.getDnsState())) { try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { final LoadBalancerServoInstance update = Entities.uniqueResult(instance); update.setDnsState(LoadBalancerServoInstance.DNS_STATE.Deregistered); db.commit(); } catch (NoSuchElementException ex) { LOG.warn("Failed to find the servo instance named " + instance.getInstanceId(), ex); } catch (Exception ex) { LOG.warn("Failed to update servo instance's ip address", ex); } } } } } @Override public void checkServoElasticIp() throws LoadBalancingActivityException { // EUCA-12956 if (!LoadBalancingWorkerProperties.useElasticIp()) { return; } final List<LoadBalancerServoInstance> allInstances = Lists.newArrayList(); try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { allInstances.addAll(Entities.query(LoadBalancerServoInstance.named())); } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to query servo instances in DB"); } final Map<String, LoadBalancerServoInstance> inServiceInstances = allInstances.stream() .filter(vm -> LoadBalancerServoInstance.STATE.InService.equals(vm.getState())) .filter(vm -> LoadBalancerServoInstance.DNS_STATE.Registered.equals(vm.getDnsState())) .collect(Collectors.toMap(LoadBalancerServoInstance::getInstanceId, vm -> vm)); final Optional<Boolean> vpcTest = LoadBalancingSystemVpcs.isCloudVpc(); for (final String instanceId : inServiceInstances.keySet()) { String ipAddr = null; String privateIpAddr = null; try { if (vpcTest.isPresent() && vpcTest.get()) { /// in vpc mode final List<Optional<String>> userVpcInterfaceAddresses = LoadBalancingSystemVpcs .getUserVpcInterfaceIps(instanceId); if (userVpcInterfaceAddresses != null) { final Optional<String> optPublicIp = userVpcInterfaceAddresses.get(0); final Optional<String> optPrivateIp = userVpcInterfaceAddresses.get(1); if (optPublicIp.isPresent()) ipAddr = optPublicIp.get(); if (optPrivateIp.isPresent()) privateIpAddr = optPrivateIp.get(); } } else { /// in classic mode final List<RunningInstancesItemType> result = EucalyptusActivityTasks.getInstance() .describeSystemInstancesWithVerbose(Lists.newArrayList(instanceId)); if (result != null && result.size() > 0) { ipAddr = result.get(0).getIpAddress(); privateIpAddr = result.get(0).getPrivateIpAddress(); } } } catch (final Exception ex) { LOG.warn("Failed to describe loadbalancer worker instances", ex); continue; } // there's an external change in elastic or private IP if ((ipAddr != null && !ipAddr.equals(inServiceInstances.get(instanceId).getAddress())) || privateIpAddr != null && !privateIpAddr.equals(inServiceInstances.get(instanceId).getPrivateIp())) { try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { final LoadBalancerServoInstance update = Entities .uniqueResult(LoadBalancerServoInstance.named(instanceId)); if (ipAddr != null) { update.setAddress(ipAddr); } if (privateIpAddr != null) { update.setPrivateIp(ipAddr); } db.commit(); } catch (final NoSuchElementException ex) { LOG.warn("Failed to find the servo instance named " + instanceId, ex); } catch (final Exception ex) { LOG.warn("Failed to update servo instance's ip address", ex); } } } } /* * Note that the backend instance check does not affect the health check result of the instances. * the health check is left to the "ping" mechanism by the servo. the state update here is the mean * by which to include only the non-faulty instances in the list delivered to servo. */ @Override public void checkBackendInstances() throws LoadBalancingActivityException { final int NUM_INSTANCES_TO_DESCRIBE = 8; /// determine backend instances to query (an instance can be registered to multiple ELBs) final Map<String, List<LoadBalancerBackendInstance>> allInstances = Maps.newHashMap(); try (final TransactionResource db = Entities.transactionFor(LoadBalancerBackendInstance.class)) { final List<LoadBalancerBackendInstance> instances = Entities.query(LoadBalancerBackendInstance.named()); for (final LoadBalancerBackendInstance instance : instances) { if (!allInstances.containsKey(instance.getInstanceId())) { allInstances.put(instance.getInstanceId(), Lists.newArrayList()); } allInstances.get(instance.getInstanceId()).add(instance); } } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to query backend instances", ex); } final List<RunningInstancesItemType> queryResult = Lists.newArrayList(); for (final List<String> partition : Iterables.partition(allInstances.keySet(), NUM_INSTANCES_TO_DESCRIBE)) { try { queryResult.addAll( EucalyptusActivityTasks.getInstance().describeSystemInstancesWithVerbose(partition)); } catch (final Exception ex) { LOG.warn("Failed to query instances", ex); break; } } //EUCA-9919: remove registered instances when terminated final Set<String> terminatedInstances = Sets.newHashSet(); final Map<String, LoadBalancerBackendInstanceStates> stateMap = new HashMap<>(); final Map<String, RunningInstancesItemType> runningInstances = new HashMap<String, RunningInstancesItemType>(); for (final RunningInstancesItemType instance : queryResult) { final String state = instance.getStateName(); if ("pending".equals(state)) stateMap.put(instance.getInstanceId(), LoadBalancerBackendInstanceStates.InitialRegistration); else if ("running".equals(state)) { runningInstances.put(instance.getInstanceId(), instance); } else if ("shutting-down".equals(state)) stateMap.put(instance.getInstanceId(), LoadBalancerBackendInstanceStates.InstanceInvalidState); else if ("terminated".equals(state)) { stateMap.put(instance.getInstanceId(), LoadBalancerBackendInstanceStates.InstanceInvalidState); terminatedInstances.add(instance.getInstanceId()); } else if ("stopping".equals(state)) stateMap.put(instance.getInstanceId(), LoadBalancerBackendInstanceStates.InstanceStopped); else if ("stopped".equals(state)) stateMap.put(instance.getInstanceId(), LoadBalancerBackendInstanceStates.InstanceStopped); } final Set<LoadBalancerBackendInstance> backendsToDelete = Sets.newHashSet(); for (final String instanceId : allInstances.keySet()) { for (final LoadBalancerBackendInstance be : allInstances.get(instanceId)) { if (terminatedInstances.contains(instanceId)) { // case 1: instance terminated backendsToDelete.add(be); continue; } if (stateMap.containsKey(instanceId)) { // case 2: instance not in running state try (final TransactionResource db = Entities .transactionFor(LoadBalancerBackendInstance.class)) { final LoadBalancerBackendInstanceStates trueState = stateMap.get(be.getInstanceId()); final LoadBalancerBackendInstance update = Entities.uniqueResult(be); update.setBackendState(trueState.getState()); update.setReasonCode(trueState.getReasonCode()); update.setDescription(trueState.getDescription()); Entities.persist(update); db.commit(); } catch (final Exception ex) { ; } } else if (runningInstances.containsKey(instanceId)) { // case 3: instance running // case 3.a: check if instance was re-started (EUCA-11859) if (LoadBalancerBackendInstanceStates.InstanceStopped.isInstanceState(be)) { final LoadBalancerBackendInstanceStates registration = LoadBalancerBackendInstanceStates.InitialRegistration; try (final TransactionResource db = Entities .transactionFor(LoadBalancerBackendInstance.class)) { final LoadBalancerBackendInstance update = Entities.uniqueResult(be); update.setBackendState(registration.getState()); update.setReasonCode(registration.getReasonCode()); update.setDescription(registration.getDescription()); Entities.persist(update); db.commit(); } catch (final Exception ex) { ; } } // case 3.b: check instance's IP address change String instanceIpAddress = null; if (be.getLoadBalancer().getVpcId() == null) instanceIpAddress = runningInstances.get(instanceId).getIpAddress(); else instanceIpAddress = runningInstances.get(instanceId).getPrivateIpAddress(); if (instanceIpAddress == null) { LOG.warn(String.format("Failed to determine ELB backend instance's IP address: %s", instanceId)); } else if (!instanceIpAddress.equals(be.getIpAddress())) { try (final TransactionResource db = Entities .transactionFor(LoadBalancerBackendInstance.class)) { final LoadBalancerBackendInstance update = Entities.uniqueResult(be); update.setIpAddress(instanceIpAddress); update.setPartition(runningInstances.get(instanceId).getPlacement()); Entities.persist(update); db.commit(); } catch (final Exception ex) { ; } } } } } for (final LoadBalancerBackendInstance be : backendsToDelete) { try (final TransactionResource db = Entities.transactionFor(LoadBalancerBackendInstance.class)) { final LoadBalancerBackendInstance entity = Entities.uniqueResult(be); Entities.delete(entity); LOG.info("Instance " + be.getInstanceId() + " is terminated and removed from ELB"); db.commit(); } catch (final Exception ex) { ; } } /// mark outdated instances as Error final int HealthUpdateTimeoutSec = 3 * MAX_HEALTHCHECK_INTERVAL_SEC; /// 6 minutes final Predicate<LoadBalancerBackendInstance> unreachableLoadbalancer = (instance) -> { if (LoadBalancerBackendInstanceStates.UnrechableLoadBalancer.isInstanceState(instance)) return false; if (LoadBalancerBackendInstanceStates.InstanceStopped.isInstanceState(instance)) return false; final long currentTime = System.currentTimeMillis(); Date lastUpdated = instance.instanceStateLastUpdated(); if (lastUpdated == null) lastUpdated = instance.getCreationTimestamp(); final int diffSec = (int) ((currentTime - lastUpdated.getTime()) / 1000.0); return diffSec > HealthUpdateTimeoutSec; }; final Set<LoadBalancerBackendInstance> outdatedInstances = allInstances.values().stream() .flatMap(Collection::stream).filter(v -> !backendsToDelete.contains(v)) // DB records deleted already .filter(v -> unreachableLoadbalancer.apply(v)).collect(Collectors.toSet()); if (!outdatedInstances.isEmpty()) { final LoadBalancerBackendInstanceStates unreachable = LoadBalancerBackendInstanceStates.UnrechableLoadBalancer; try (TransactionResource db = Entities.transactionFor(LoadBalancerBackendInstance.class)) { for (final LoadBalancerBackendInstance instance : outdatedInstances) { final LoadBalancerBackendInstance update = Entities.uniqueResult(instance); update.setState(unreachable.getState()); update.setReasonCode(unreachable.getReasonCode()); update.setDescription(unreachable.getDescription()); Entities.persist(update); } db.commit(); } catch (final Exception ex) { ; } } } @Override public void cleanupSecurityGroups() throws LoadBalancingActivityException { /// find all security group whose member instances are empty List<LoadBalancerSecurityGroup> allGroups = null; try (TransactionResource db = Entities.transactionFor(LoadBalancerSecurityGroup.class)) { allGroups = Entities .query(LoadBalancerSecurityGroup.withState(LoadBalancerSecurityGroup.STATE.OutOfService)); } catch (Exception ex) { /* retry later */ } if (allGroups == null || allGroups.size() <= 0) return; final List<LoadBalancerSecurityGroup> toDelete = Lists.newArrayList(); for (LoadBalancerSecurityGroup group : allGroups) { Collection<LoadBalancerServoInstanceCoreView> instances = group.getServoInstances(); if (instances == null || instances.size() <= 0) toDelete.add(group); } /// delete them from euca for (final LoadBalancerSecurityGroup group : toDelete) { boolean deleted = false; try { final List<SecurityGroupItemType> existingGroups = EucalyptusActivityTasks.getInstance() .describeSystemSecurityGroups(Lists.newArrayList(group.getName()), true); if (existingGroups == null || existingGroups.size() <= 0) deleted = true; else { EucalyptusActivityTasks.getInstance().deleteSystemSecurityGroup(group.getName(), true); LOG.info("Deleted security group: " + group.getName()); deleted = true; } } catch (final Exception ex) { try { final List<SecurityGroupItemType> existingGroups = EucalyptusActivityTasks.getInstance() .describeSystemSecurityGroups(Lists.newArrayList(group.getName()), false); if (existingGroups == null || existingGroups.size() <= 0) deleted = true; else { EucalyptusActivityTasks.getInstance().deleteSystemSecurityGroup(group.getName(), false); LOG.info("Deleted security group: " + group.getName()); deleted = true; } } catch (final Exception ex2) { LOG.warn("Failed to delete the security group from eucalyptus", ex2); } } if (deleted) { try (final TransactionResource db = Entities.transactionFor(LoadBalancerSecurityGroup.class)) { final LoadBalancerSecurityGroup g = Entities.uniqueResult(group); Entities.delete(g); db.commit(); } catch (NoSuchElementException ex) { ; } catch (Exception ex) { LOG.warn("Failed to delete the securty group records from database", ex); } } } } @Override public void cleanupServoInstances() throws LoadBalancingActivityException { // find all OutOfService instances List<LoadBalancerServoInstance> retired = null; try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { LoadBalancerServoInstance sample = LoadBalancerServoInstance .withState(LoadBalancerServoInstance.STATE.Retired.name()); retired = Entities.query(sample); sample = LoadBalancerServoInstance.withState(LoadBalancerServoInstance.STATE.Error.name()); retired.addAll(Entities.query(sample)); } catch (Exception ex) { LOG.warn("Failed to query loadbalancer servo instance", ex); } if (retired == null || retired.size() <= 0) return; /// for each: // describe instances final List<String> param = Lists.newArrayList(); final Map<String, String> latestState = Maps.newHashMap(); for (final LoadBalancerServoInstance instance : retired) { /// call describe instance String instanceId = instance.getInstanceId(); if (instanceId == null) continue; param.clear(); param.add(instanceId); String instanceState; try { List<RunningInstancesItemType> result = null; result = EucalyptusActivityTasks.getInstance().describeSystemInstancesWithVerbose(param); if (result.isEmpty()) instanceState = "terminated"; else instanceState = result.get(0).getStateName(); } catch (final Exception ex) { LOG.warn("Failed to query instances", ex); continue; } latestState.put(instanceId, instanceState); } // if state==terminated or describe instances return no result, // delete the database record for (String instanceId : latestState.keySet()) { String state = latestState.get(instanceId); if (state.equals("terminated")) { try (final TransactionResource db2 = Entities.transactionFor(LoadBalancerServoInstance.class)) { LoadBalancerServoInstance toDelete = Entities .uniqueResult(LoadBalancerServoInstance.named(instanceId)); Entities.delete(toDelete); db2.commit(); } catch (Exception ex) { LOG.warn("Unable to delete load balancer servo instance: ", ex); } } } } @Override public void recycleFailedServoInstances() throws LoadBalancingActivityException { /// when SWF activity fails on the VM more than the threshold (property), the VM is terminated /// and the autoscaling group replaces it with the new VM. /// The terminated Vms will be cleaned up later by checkServoInstances() and cleanupServoInstances() final int failureThreshold = Integer.parseInt(LoadBalancingWorkerProperties.FAILURE_THRESHOLD_FOR_RECYCLE); if (failureThreshold <= 0) { return; } try { final List<LoadBalancerServoInstance> inServiceInstances = Lists.newArrayList(); try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { LoadBalancerServoInstance sample = LoadBalancerServoInstance .withState(LoadBalancerServoInstance.STATE.InService.name()); inServiceInstances.addAll(Entities.query(sample)); } final Date oneHourAgo = new Date(System.currentTimeMillis() - (1 * 60 * 60 * 1000)); final List<String> unhealthyInstances = inServiceInstances.stream() .filter(vm -> vm.getActivityFailureCount() >= failureThreshold).map(vm -> vm.getInstanceId()) .collect(Collectors.toList()); final List<String> newInstances = inServiceInstances.stream() .filter(vm -> vm.getActivityFailureUpdateTime() == null).map(vm -> vm.getInstanceId()) .collect(Collectors.toList()); final List<String> temporallyFailedInstances = inServiceInstances.stream() .filter(vm -> vm.getActivityFailureCount() < failureThreshold && vm.getActivityFailureCount() > 0 && (vm.getActivityFailureUpdateTime() != null && vm.getActivityFailureUpdateTime().before(oneHourAgo))) .map(vm -> vm.getInstanceId()).collect(Collectors.toList()); try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { for (final String instanceId : newInstances) { final LoadBalancerServoInstance update = Entities .uniqueResult(LoadBalancerServoInstance.named(instanceId)); update.setActivityFailureUpdateTime(new Date(System.currentTimeMillis())); Entities.persist(update); } for (final String instanceId : temporallyFailedInstances) { final LoadBalancerServoInstance update = Entities .uniqueResult(LoadBalancerServoInstance.named(instanceId)); update.setActivityFailureCount(0); update.setActivityFailureUpdateTime(new Date(System.currentTimeMillis())); Entities.persist(update); } db.commit(); } for (final String instanceId : unhealthyInstances) { EucalyptusActivityTasks.getInstance().terminateInstances(Lists.newArrayList(instanceId)); LOG.debug(String.format("Unhealthy loadbalancer VM is detected and terminated (%s)", instanceId)); } } catch (final Exception ex) { LOG.error("Failed to recycle unhealthy worker VMs", ex); } } @Override public void runContinousWorkflows() throws LoadBalancingActivityException { List<LoadBalancer> loadbalancers = null; try { loadbalancers = LoadBalancers.listLoadbalancers(); } catch (final Exception ex) { LOG.error("Failed to list all loadbalancers", ex); return; } for (final LoadBalancer lb : loadbalancers) { final String accountId = lb.getOwnerAccountNumber(); final String lbName = lb.getDisplayName(); try { LoadBalancingWorkflows.runUpdateLoadBalancer(accountId, lbName); LoadBalancingWorkflows.runInstanceStatusPolling(accountId, lbName); LoadBalancingWorkflows.runCloudWatchPutMetric(accountId, lbName); } catch (final Exception ex) { LOG.error("Failed to run continous workflows for loadbalancers", ex); } } } @Override public List<String> lookupServoInstances(final String accountNumber, final String lbName) throws LoadBalancingActivityException { final LoadBalancer lb; try { lb = LoadBalancers.getLoadbalancer(accountNumber, lbName); } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to lookup loadbalancer"); } final List<String> instances = Lists.newArrayList(); try { for (final LoadBalancerZoneCoreView zoneView : lb.getZones()) { if (LoadBalancerZone.STATE.OutOfService.equals(zoneView.getState())) continue; final LoadBalancerZone zone = LoadBalancerZoneEntityTransform.INSTANCE.apply(zoneView); instances.addAll(Collections2.transform(zone.getServoInstances(), new Function<LoadBalancerServoInstanceCoreView, String>() { @Override public String apply(LoadBalancerServoInstanceCoreView arg0) { return arg0.getInstanceId(); } })); } } catch (final Exception ex) { throw new LoadBalancingActivityException("Failed to lookup servo instance records", ex); } return instances; } // to make sure that all ELB VMs have the right role policy @Override public void upgrade4_4() throws LoadBalancingActivityException { final List<LoadBalancer> oldLBs = LoadBalancers.listLoadbalancers().stream() .filter(lb -> !LoadBalancers.v4_4_0.apply(lb)).collect(Collectors.toList()); for (final LoadBalancer lb : oldLBs) { final String accountNumber = lb.getOwnerAccountNumber(); final String lbName = lb.getDisplayName(); final String roleName = getRoleName(accountNumber, lbName); try { GetRolePolicyResult policy = null; final List<String> policies = EucalyptusActivityTasks.getInstance().listRolePolicies(roleName); if (policies.contains(SERVO_ROLE_POLICY_NAME)) { policy = EucalyptusActivityTasks.getInstance().getRolePolicy(roleName, SERVO_ROLE_POLICY_NAME); } final boolean policyAllowsSwf = true ? policy != null && SERVO_ROLE_POLICY_DOCUMENT.toLowerCase() .equals(policy.getPolicyDocument().toLowerCase()) : false; if (!policyAllowsSwf) { if (policy != null) { EucalyptusActivityTasks.getInstance().deleteRolePolicy(roleName, policy.getPolicyName()); } EucalyptusActivityTasks.getInstance().putRolePolicy(roleName, SERVO_ROLE_POLICY_NAME, SERVO_ROLE_POLICY_DOCUMENT); LOG.info(String.format("IAM role policy was updated for loadbalancer (%s-%s)", accountNumber, lbName)); } } catch (final Exception ex) { LOG.warn(String.format("Failed to upgrade old loadbalancer (%s-%s) to 4.4", accountNumber, lbName), ex); } } } @Override public void recordInstanceTaskFailure(final String instanceId) throws LoadBalancingActivityException { try { try (final TransactionResource db = Entities.transactionFor(LoadBalancerServoInstance.class)) { final LoadBalancerServoInstance sample = LoadBalancerServoInstance.named(instanceId); final LoadBalancerServoInstance entity = Entities.uniqueResult(sample); entity.setActivityFailureCount(entity.getActivityFailureCount() + 1); Entities.persist(entity); db.commit(); } } catch (final Exception ex) { LOG.warn(String.format("Failed to mark the VM (%s) as failed", instanceId), ex); } } }