Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.jclouds.ec2.compute; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Strings.emptyToNull; import static com.google.common.collect.Iterables.concat; import static com.google.common.collect.Iterables.transform; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.jclouds.compute.config.ComputeServiceProperties.RESOURCENAME_DELIMITER; import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_RUNNING; import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_SUSPENDED; import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_TERMINATED; import static org.jclouds.compute.util.ComputeServiceUtils.addMetadataAndParseTagsFromValuesOfEmptyString; import static org.jclouds.compute.util.ComputeServiceUtils.metadataAndTagsAsValuesOfEmptyString; import static org.jclouds.ec2.reference.EC2Constants.PROPERTY_EC2_GENERATE_INSTANCE_NAMES; import static org.jclouds.ec2.util.Tags.resourceToTagsAsMap; import static org.jclouds.util.Predicates2.retry; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicReference; import javax.inject.Named; import javax.inject.Provider; import javax.inject.Singleton; import org.jclouds.Constants; import org.jclouds.aws.util.AWSUtils; import org.jclouds.collect.Memoized; import org.jclouds.compute.ComputeServiceContext; import org.jclouds.compute.RunNodesException; import org.jclouds.compute.callables.RunScriptOnNode; import org.jclouds.compute.domain.Hardware; import org.jclouds.compute.domain.Image; import org.jclouds.compute.domain.NodeMetadata; import org.jclouds.compute.domain.NodeMetadataBuilder; import org.jclouds.compute.domain.Template; import org.jclouds.compute.domain.TemplateBuilder; import org.jclouds.compute.extensions.ImageExtension; import org.jclouds.compute.extensions.SecurityGroupExtension; import org.jclouds.compute.functions.GroupNamingConvention; import org.jclouds.compute.functions.GroupNamingConvention.Factory; import org.jclouds.compute.internal.BaseComputeService; import org.jclouds.compute.internal.PersistNodeCredentials; import org.jclouds.compute.options.TemplateOptions; import org.jclouds.compute.reference.ComputeServiceConstants.Timeouts; import org.jclouds.compute.strategy.CreateNodesInGroupThenAddToSet; import org.jclouds.compute.strategy.DestroyNodeStrategy; import org.jclouds.compute.strategy.GetImageStrategy; import org.jclouds.compute.strategy.GetNodeMetadataStrategy; import org.jclouds.compute.strategy.InitializeRunScriptOnNodeOrPlaceInBadMap; import org.jclouds.compute.strategy.ListNodesStrategy; import org.jclouds.compute.strategy.RebootNodeStrategy; import org.jclouds.compute.strategy.ResumeNodeStrategy; import org.jclouds.compute.strategy.SuspendNodeStrategy; import org.jclouds.domain.Credentials; import org.jclouds.domain.Location; import org.jclouds.ec2.EC2Api; import org.jclouds.ec2.compute.domain.RegionAndName; import org.jclouds.ec2.compute.domain.RegionNameAndIngressRules; import org.jclouds.ec2.compute.options.EC2TemplateOptions; import org.jclouds.ec2.domain.InstanceState; import org.jclouds.ec2.domain.KeyPair; import org.jclouds.ec2.domain.RunningInstance; import org.jclouds.ec2.domain.Tag; import org.jclouds.ec2.util.TagFilterBuilder; import org.jclouds.scriptbuilder.functions.InitAdminAccess; import org.jclouds.util.Strings2; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Predicate; import com.google.common.base.Supplier; import com.google.common.cache.LoadingCache; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableMultimap.Builder; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.inject.Inject; /** * @author Adrian Cole */ @Singleton public class EC2ComputeService extends BaseComputeService { private final EC2Api client; private final ConcurrentMap<RegionAndName, KeyPair> credentialsMap; private final LoadingCache<RegionAndName, String> securityGroupMap; private final Factory namingConvention; private final boolean generateInstanceNames; @Inject protected EC2ComputeService(ComputeServiceContext context, Map<String, Credentials> credentialStore, @Memoized Supplier<Set<? extends Image>> images, @Memoized Supplier<Set<? extends Hardware>> sizes, @Memoized Supplier<Set<? extends Location>> locations, ListNodesStrategy listNodesStrategy, GetImageStrategy getImageStrategy, GetNodeMetadataStrategy getNodeMetadataStrategy, CreateNodesInGroupThenAddToSet runNodesAndAddToSetStrategy, RebootNodeStrategy rebootNodeStrategy, DestroyNodeStrategy destroyNodeStrategy, ResumeNodeStrategy startNodeStrategy, SuspendNodeStrategy stopNodeStrategy, Provider<TemplateBuilder> templateBuilderProvider, @Named("DEFAULT") Provider<TemplateOptions> templateOptionsProvider, @Named(TIMEOUT_NODE_RUNNING) Predicate<AtomicReference<NodeMetadata>> nodeRunning, @Named(TIMEOUT_NODE_TERMINATED) Predicate<AtomicReference<NodeMetadata>> nodeTerminated, @Named(TIMEOUT_NODE_SUSPENDED) Predicate<AtomicReference<NodeMetadata>> nodeSuspended, InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory, RunScriptOnNode.Factory runScriptOnNodeFactory, InitAdminAccess initAdminAccess, PersistNodeCredentials persistNodeCredentials, Timeouts timeouts, @Named(Constants.PROPERTY_USER_THREADS) ListeningExecutorService userExecutor, EC2Api client, ConcurrentMap<RegionAndName, KeyPair> credentialsMap, @Named("SECURITY") LoadingCache<RegionAndName, String> securityGroupMap, Optional<ImageExtension> imageExtension, GroupNamingConvention.Factory namingConvention, @Named(PROPERTY_EC2_GENERATE_INSTANCE_NAMES) boolean generateInstanceNames, Optional<SecurityGroupExtension> securityGroupExtension) { super(context, credentialStore, images, sizes, locations, listNodesStrategy, getImageStrategy, getNodeMetadataStrategy, runNodesAndAddToSetStrategy, rebootNodeStrategy, destroyNodeStrategy, startNodeStrategy, stopNodeStrategy, templateBuilderProvider, templateOptionsProvider, nodeRunning, nodeTerminated, nodeSuspended, initScriptRunnerFactory, initAdminAccess, runScriptOnNodeFactory, persistNodeCredentials, timeouts, userExecutor, imageExtension, securityGroupExtension); this.client = client; this.credentialsMap = credentialsMap; this.securityGroupMap = securityGroupMap; this.namingConvention = namingConvention; this.generateInstanceNames = generateInstanceNames; } @Override public Set<? extends NodeMetadata> createNodesInGroup(String group, int count, final Template template) throws RunNodesException { Set<? extends NodeMetadata> nodes = super.createNodesInGroup(group, count, template); String region = AWSUtils.getRegionFromLocationOrNull(template.getLocation()); if (client.getTagApiForRegion(region).isPresent()) { Map<String, String> common = metadataAndTagsAsValuesOfEmptyString(template.getOptions()); if (generateInstanceNames || !common.isEmpty() || !template.getOptions().getNodeNames().isEmpty()) { return addTagsAndNamesToInstancesInRegion(common, template.getOptions().getNodeNames(), nodes, region, group); } } return nodes; } private static final Function<NodeMetadata, String> instanceId = new Function<NodeMetadata, String>() { @Override public String apply(NodeMetadata in) { return in.getProviderId(); } }; private Set<NodeMetadata> addTagsAndNamesToInstancesInRegion(Map<String, String> common, Set<String> nodeNames, Set<? extends NodeMetadata> input, String region, String group) { Map<String, ? extends NodeMetadata> instancesById = Maps.uniqueIndex(input, instanceId); ImmutableSet.Builder<NodeMetadata> builder = ImmutableSet.<NodeMetadata>builder(); if (generateInstanceNames && !common.containsKey("Name")) { for (Map.Entry<String, ? extends NodeMetadata> entry : instancesById.entrySet()) { String id = entry.getKey(); String name; if (!nodeNames.isEmpty()) { name = Iterables.get(nodeNames, 0); } else { name = id.replaceAll(".*-", group + "-"); } Map<String, String> tags = ImmutableMap.<String, String>builder().putAll(common).put("Name", name) .build(); logger.debug(">> applying tags %s to instance %s in region %s", tags, id, region); client.getTagApiForRegion(region).get().applyToResources(tags, ImmutableSet.of(id)); builder.add(addTagsForInstance(tags, instancesById.get(id))); } } else { Iterable<String> ids = instancesById.keySet(); logger.debug(">> applying tags %s to instances %s in region %s", common, ids, region); client.getTagApiForRegion(region).get().applyToResources(common, ids); for (NodeMetadata in : input) builder.add(addTagsForInstance(common, in)); } if (logger.isDebugEnabled()) { Multimap<String, String> filter = new TagFilterBuilder().resourceIds(instancesById.keySet()).build(); FluentIterable<Tag> tags = client.getTagApiForRegion(region).get().filter(filter); logger.debug("<< applied tags in region %s: %s", region, resourceToTagsAsMap(tags)); } return builder.build(); } private static NodeMetadata addTagsForInstance(Map<String, String> tags, NodeMetadata input) { NodeMetadataBuilder builder = NodeMetadataBuilder.fromNodeMetadata(input).name(tags.get("Name")); return addMetadataAndParseTagsFromValuesOfEmptyString(builder, tags).build(); } @Inject(optional = true) @Named(RESOURCENAME_DELIMITER) char delimiter = '#'; /** * @throws IllegalStateException If the security group was in use */ @VisibleForTesting void deleteSecurityGroup(String region, String group) { checkNotNull(emptyToNull(region), "region must be defined"); checkNotNull(emptyToNull(group), "group must be defined"); String groupName = namingConvention.create().sharedNameForGroup(group); if (client.getSecurityGroupApi().get().describeSecurityGroupsInRegion(region, groupName).size() > 0) { logger.debug(">> deleting securityGroup(%s)", groupName); client.getSecurityGroupApi().get().deleteSecurityGroupInRegion(region, groupName); // TODO: test this clear happens securityGroupMap.invalidate(new RegionNameAndIngressRules(region, groupName, null, false)); logger.debug("<< deleted securityGroup(%s)", groupName); } } @VisibleForTesting void deleteKeyPair(String region, String group) { for (KeyPair keyPair : client.getKeyPairApi().get() .describeKeyPairsInRegionWithFilter(region, ImmutableMultimap.<String, String>builder() .put("key-name", Strings2.urlEncode( String.format("jclouds#%s#%s*", group, region).replace('#', delimiter))) .build())) { String keyName = keyPair.getKeyName(); Predicate<String> keyNameMatcher = namingConvention.create().containsGroup(group); String oldKeyNameRegex = String.format("jclouds#%s#%s#%s", group, region, "[0-9a-f]+").replace('#', delimiter); // old keypair pattern too verbose as it has an unnecessary region qualifier if (keyNameMatcher.apply(keyName) || keyName.matches(oldKeyNameRegex)) { Set<String> instancesUsingKeyPair = extractIdsFromInstances( concat(client.getInstanceApi().get().describeInstancesInRegionWithFilter(region, ImmutableMultimap.<String, String>builder() .put("instance-state-name", InstanceState.TERMINATED.toString()) .put("instance-state-name", InstanceState.SHUTTING_DOWN.toString()) .put("key-name", keyPair.getKeyName()).build()))); if (instancesUsingKeyPair.size() > 0) { logger.debug("<< inUse keyPair(%s), by (%s)", keyPair.getKeyName(), instancesUsingKeyPair); } else { logger.debug(">> deleting keyPair(%s)", keyPair.getKeyName()); client.getKeyPairApi().get().deleteKeyPairInRegion(region, keyPair.getKeyName()); // TODO: test this clear happens credentialsMap.remove(new RegionAndName(region, keyPair.getKeyName())); credentialsMap.remove(new RegionAndName(region, group)); logger.debug("<< deleted keyPair(%s)", keyPair.getKeyName()); } } } } protected ImmutableSet<String> extractIdsFromInstances(Iterable<? extends RunningInstance> deadOnes) { return ImmutableSet.copyOf(transform(deadOnes, new Function<RunningInstance, String>() { @Override public String apply(RunningInstance input) { return input.getId(); } })); } /** * Cleans implicit keypairs and security groups. */ @Override protected void cleanUpIncidentalResourcesOfDeadNodes(Set<? extends NodeMetadata> deadNodes) { Builder<String, String> regionGroups = ImmutableMultimap.builder(); for (NodeMetadata nodeMetadata : deadNodes) { if (nodeMetadata.getGroup() != null) regionGroups.put(AWSUtils.parseHandle(nodeMetadata.getId())[0], nodeMetadata.getGroup()); } for (Entry<String, String> regionGroup : regionGroups.build().entries()) { cleanUpIncidentalResources(regionGroup.getKey(), regionGroup.getValue()); } } protected void cleanUpIncidentalResources(final String region, final String group) { // For issue #445, tries to delete security groups first: ec2 throws exception if in use, but // deleting a key pair does not. // This is "belt-and-braces" because deleteKeyPair also does extractIdsFromInstances & usingKeyPairAndNotDead // for us to check if any instances are using the key-pair before we delete it. // There is (probably?) still a race if someone is creating instances at the same time as deleting them: // we may delete the key-pair just when the node-being-created was about to rely on the incidental // resources existing. // Also in #445, in aws-ec2 the deleteSecurityGroup sometimes fails after terminating the final VM using a // given security group, if called very soon after the VM's state reports terminated. Empirically, it seems that // waiting a small time (e.g. enabling logging or debugging!) then the tests pass. We therefore retry. // TODO: this could be moved to a config module, also the narrative above made more concise retry(new Predicate<RegionAndName>() { public boolean apply(RegionAndName input) { try { logger.debug(">> deleting incidentalResources(%s)", input); deleteSecurityGroup(input.getRegion(), input.getName()); deleteKeyPair(input.getRegion(), input.getName()); // not executed if securityGroup was in use logger.debug("<< deleted incidentalResources(%s)", input); return true; } catch (IllegalStateException e) { logger.debug("<< inUse incidentalResources(%s)", input); return false; } } }, SECONDS.toMillis(3), 50, 1000, MILLISECONDS).apply(new RegionAndName(region, group)); } /** * returns template options, except of type {@link EC2TemplateOptions}. */ @Override public EC2TemplateOptions templateOptions() { return EC2TemplateOptions.class.cast(super.templateOptions()); } }