List of usage examples for java.util.concurrent Semaphore getQueueLength
public final int getQueueLength()
From source file:com.thoughtworks.studios.shine.cruise.stage.details.LazyStageGraphLoaderTest.java
@Test public void shouldNotReuseTransformerAcrossConcurrentInvocations() throws InterruptedException { final StageIdentifier stageId = new StageIdentifier("pipeline-foo", 23, "stage-1", "1"); final Semaphore invocationBlocker = new Semaphore(1); final DummyStageResourceImporter realLoader = new DummyStageResourceImporter(realGraph(), stageId, invocationBlocker);//w w w.java 2 s . c o m final LazyStageGraphLoader loader = new LazyStageGraphLoader(realLoader, stageStorage, 2); final XSLTTransformerRegistry[] transformerRegistryUsed = new XSLTTransformerRegistry[2]; invocationBlocker.acquire(); Thread firstThd = new Thread(new Runnable() { public void run() { loader.load(stageId); invocationBlocker.release(); } }); firstThd.start(); while (invocationBlocker.getQueueLength() == 0) { Thread.sleep(10); } transformerRegistryUsed[0] = realLoader.transformerRegistry; Thread secondThd = new Thread(new Runnable() { public void run() { stageStorage.clear(); loader.load(stageId); } }); secondThd.start(); while (invocationBlocker.getQueueLength() == 1) { Thread.sleep(10); } transformerRegistryUsed[1] = realLoader.transformerRegistry; invocationBlocker.release(); firstThd.join(); secondThd.join(); assertThat(transformerRegistryUsed[0], not(sameInstance(transformerRegistryUsed[1]))); }
From source file:org.apache.brooklyn.location.jclouds.JcloudsLocation.java
protected MachineLocation obtainOnce(ConfigBag setup) throws NoMachinesAvailableException { AccessController.Response access = getManagementContext().getAccessController().canProvisionLocation(this); if (!access.isAllowed()) { throw new IllegalStateException( "Access controller forbids provisioning in " + this + ": " + access.getMsg()); }/*w ww .j a v a2 s .co m*/ setCreationString(setup); boolean waitForSshable = !"false".equalsIgnoreCase(setup.get(WAIT_FOR_SSHABLE)); boolean waitForWinRmable = !"false".equalsIgnoreCase(setup.get(WAIT_FOR_WINRM_AVAILABLE)); boolean usePortForwarding = setup.get(USE_PORT_FORWARDING); boolean skipJcloudsSshing = Boolean.FALSE.equals(setup.get(USE_JCLOUDS_SSH_INIT)) || usePortForwarding; JcloudsPortForwarderExtension portForwarder = setup.get(PORT_FORWARDER); if (usePortForwarding) checkNotNull(portForwarder, "portForwarder, when use-port-forwarding enabled"); final ComputeService computeService = getConfig(COMPUTE_SERVICE_REGISTRY).findComputeService(setup, true); CloudMachineNamer cloudMachineNamer = getCloudMachineNamer(setup); String groupId = elvis(setup.get(GROUP_ID), cloudMachineNamer.generateNewGroupId(setup)); NodeMetadata node = null; JcloudsMachineLocation machineLocation = null; Duration semaphoreTimestamp = null; Duration templateTimestamp = null; Duration provisionTimestamp = null; Duration usableTimestamp = null; Duration customizedTimestamp = null; Stopwatch provisioningStopwatch = Stopwatch.createStarted(); try { LOG.info("Creating VM " + setup.getDescription() + " in " + this); Semaphore machineCreationSemaphore = getMachineCreationSemaphore(); boolean acquired = machineCreationSemaphore.tryAcquire(0, TimeUnit.SECONDS); if (!acquired) { LOG.info("Waiting in {} for machine-creation permit ({} other queuing requests already)", new Object[] { this, machineCreationSemaphore.getQueueLength() }); Stopwatch blockStopwatch = Stopwatch.createStarted(); machineCreationSemaphore.acquire(); LOG.info("Acquired in {} machine-creation permit, after waiting {}", this, Time.makeTimeStringRounded(blockStopwatch)); } else { LOG.debug("Acquired in {} machine-creation permit immediately", this); } semaphoreTimestamp = Duration.of(provisioningStopwatch); LoginCredentials userCredentials = null; Set<? extends NodeMetadata> nodes; Template template; try { // Setup the template template = buildTemplate(computeService, setup); boolean expectWindows = isWindows(template, setup); if (!skipJcloudsSshing) { if (expectWindows) { // TODO Was this too early to look at template.getImage? e.g. customizeTemplate could subsequently modify it. LOG.warn("Ignoring invalid configuration for Windows provisioning of " + template.getImage() + ": " + USE_JCLOUDS_SSH_INIT.getName() + " should be false"); skipJcloudsSshing = true; } else if (waitForSshable) { userCredentials = initTemplateForCreateUser(template, setup); } } templateTimestamp = Duration.of(provisioningStopwatch); // "Name" metadata seems to set the display name; at least in AWS // TODO it would be nice if this salt comes from the location's ID (but we don't know that yet as the ssh machine location isn't created yet) // TODO in softlayer we want to control the suffix of the hostname which is 3 random hex digits template.getOptions().getUserMetadata().put("Name", cloudMachineNamer.generateNewMachineUniqueNameFromGroupId(setup, groupId)); if (setup.get(JcloudsLocationConfig.INCLUDE_BROOKLYN_USER_METADATA)) { template.getOptions().getUserMetadata().put("brooklyn-user", System.getProperty("user.name")); Object context = setup.get(CALLER_CONTEXT); if (context instanceof Entity) { Entity entity = (Entity) context; template.getOptions().getUserMetadata().put("brooklyn-app-id", entity.getApplicationId()); template.getOptions().getUserMetadata().put("brooklyn-app-name", entity.getApplication().getDisplayName()); template.getOptions().getUserMetadata().put("brooklyn-entity-id", entity.getId()); template.getOptions().getUserMetadata().put("brooklyn-entity-name", entity.getDisplayName()); template.getOptions().getUserMetadata().put("brooklyn-server-creation-date", Time.makeDateSimpleStampString()); } } customizeTemplate(setup, computeService, template); LOG.debug("jclouds using template {} / options {} to provision machine in {}", new Object[] { template, template.getOptions(), setup.getDescription() }); if (!setup.getUnusedConfig().isEmpty()) if (LOG.isDebugEnabled()) LOG.debug("NOTE: unused flags passed to obtain VM in " + setup.getDescription() + ": " + Sanitizer.sanitize(setup.getUnusedConfig())); nodes = computeService.createNodesInGroup(groupId, 1, template); provisionTimestamp = Duration.of(provisioningStopwatch); } finally { machineCreationSemaphore.release(); } node = Iterables.getOnlyElement(nodes, null); LOG.debug("jclouds created {} for {}", node, setup.getDescription()); if (node == null) throw new IllegalStateException( "No nodes returned by jclouds create-nodes in " + setup.getDescription()); boolean windows = isWindows(node, setup); if (windows) { int newLoginPort = node.getLoginPort() == 22 ? 5985 : node.getLoginPort(); String newLoginUser = "root".equals(node.getCredentials().getUser()) ? "Administrator" : node.getCredentials().getUser(); LOG.debug( "jclouds created Windows VM {}; transforming connection details: loginPort from {} to {}; loginUser from {} to {}", new Object[] { node, node.getLoginPort(), newLoginPort, node.getCredentials().getUser(), newLoginUser }); node = NodeMetadataBuilder.fromNodeMetadata(node).loginPort(newLoginPort) .credentials(LoginCredentials.builder(node.getCredentials()).user(newLoginUser).build()) .build(); } // FIXME How do we influence the node.getLoginPort, so it is set correctly for Windows? // Setup port-forwarding, if required Optional<HostAndPort> sshHostAndPortOverride; if (usePortForwarding) { sshHostAndPortOverride = Optional.of(portForwarder.openPortForwarding(node, node.getLoginPort(), Optional.<Integer>absent(), Protocol.TCP, Cidr.UNIVERSAL)); } else { sshHostAndPortOverride = Optional.absent(); } LoginCredentials initialCredentials = node.getCredentials(); if (skipJcloudsSshing) { boolean waitForConnectable = (windows) ? waitForWinRmable : waitForSshable; if (waitForConnectable) { if (windows) { // TODO Does jclouds support any windows user setup? initialCredentials = waitForWinRmAvailable(computeService, node, sshHostAndPortOverride, setup); } else { initialCredentials = waitForSshable(computeService, node, sshHostAndPortOverride, setup); } userCredentials = createUser(computeService, node, sshHostAndPortOverride, initialCredentials, setup); } } // Figure out which login-credentials to use LoginCredentials customCredentials = setup.get(CUSTOM_CREDENTIALS); if (customCredentials != null) { userCredentials = customCredentials; //set userName and other data, from these credentials Object oldUsername = setup.put(USER, customCredentials.getUser()); LOG.debug("node {} username {} / {} (customCredentials)", new Object[] { node, customCredentials.getUser(), oldUsername }); if (customCredentials.getOptionalPassword().isPresent()) setup.put(PASSWORD, customCredentials.getOptionalPassword().get()); if (customCredentials.getOptionalPrivateKey().isPresent()) setup.put(PRIVATE_KEY_DATA, customCredentials.getOptionalPrivateKey().get()); } if (userCredentials == null || (!userCredentials.getOptionalPassword().isPresent() && !userCredentials.getOptionalPrivateKey().isPresent())) { // We either don't have any userCredentials, or it is missing both a password/key. // TODO See waitForSshable, which now handles if the node.getLoginCredentials has both a password+key userCredentials = extractVmCredentials(setup, node, initialCredentials); } if (userCredentials == null) { // TODO See waitForSshable, which now handles if the node.getLoginCredentials has both a password+key userCredentials = extractVmCredentials(setup, node, initialCredentials); } if (userCredentials != null) { node = NodeMetadataBuilder.fromNodeMetadata(node).credentials(userCredentials).build(); } else { // only happens if something broke above... userCredentials = LoginCredentials.fromCredentials(node.getCredentials()); } // store the credentials, in case they have changed setup.putIfNotNull(JcloudsLocationConfig.PASSWORD, userCredentials.getOptionalPassword().orNull()); setup.putIfNotNull(JcloudsLocationConfig.PRIVATE_KEY_DATA, userCredentials.getOptionalPrivateKey().orNull()); // Wait for the VM to be reachable over SSH if (waitForSshable && !windows) { waitForSshable(computeService, node, sshHostAndPortOverride, ImmutableList.of(userCredentials), setup); } else { LOG.debug("Skipping ssh check for {} ({}) due to config waitForSshable=false", node, setup.getDescription()); } usableTimestamp = Duration.of(provisioningStopwatch); // JcloudsSshMachineLocation jcloudsSshMachineLocation = null; // WinRmMachineLocation winRmMachineLocation = null; // Create a JcloudsSshMachineLocation, and register it if (windows) { machineLocation = registerWinRmMachineLocation(computeService, node, userCredentials, sshHostAndPortOverride, setup); } else { machineLocation = registerJcloudsSshMachineLocation(computeService, node, Optional.fromNullable(template), userCredentials, sshHostAndPortOverride, setup); } if (usePortForwarding && sshHostAndPortOverride.isPresent()) { // Now that we have the sshMachineLocation, we can associate the port-forwarding address with it. PortForwardManager portForwardManager = setup.get(PORT_FORWARDING_MANAGER); if (portForwardManager != null) { portForwardManager.associate(node.getId(), sshHostAndPortOverride.get(), machineLocation, node.getLoginPort()); } else { LOG.warn("No port-forward manager for {} so could not associate {} -> {} for {}", new Object[] { this, node.getLoginPort(), sshHostAndPortOverride, machineLocation }); } } if ("docker".equals(this.getProvider())) { if (windows) { throw new UnsupportedOperationException("Docker not supported on Windows"); } Map<Integer, Integer> portMappings = JcloudsUtil.dockerPortMappingsFor(this, node.getId()); PortForwardManager portForwardManager = setup.get(PORT_FORWARDING_MANAGER); if (portForwardManager != null) { for (Integer containerPort : portMappings.keySet()) { Integer hostPort = portMappings.get(containerPort); String dockerHost = ((JcloudsSshMachineLocation) machineLocation).getSshHostAndPort() .getHostText(); portForwardManager.associate(node.getId(), HostAndPort.fromParts(dockerHost, hostPort), machineLocation, containerPort); } } else { LOG.warn("No port-forward manager for {} so could not associate docker port-mappings for {}", this, machineLocation); } } List<String> customisationForLogging = new ArrayList<String>(); // Apply same securityGroups rules to iptables, if iptables is running on the node if (waitForSshable) { String setupScript = setup.get(JcloudsLocationConfig.CUSTOM_MACHINE_SETUP_SCRIPT_URL); List<String> setupScripts = setup.get(JcloudsLocationConfig.CUSTOM_MACHINE_SETUP_SCRIPT_URL_LIST); Collection<String> allScripts = new MutableList<String>().appendIfNotNull(setupScript) .appendAll(setupScripts); for (String setupScriptItem : allScripts) { if (Strings.isNonBlank(setupScriptItem)) { customisationForLogging.add("custom setup script " + setupScriptItem); String setupVarsString = setup.get(JcloudsLocationConfig.CUSTOM_MACHINE_SETUP_SCRIPT_VARS); Map<String, String> substitutions = (setupVarsString != null) ? Splitter.on(",").withKeyValueSeparator(":").split(setupVarsString) : ImmutableMap.<String, String>of(); String scriptContent = ResourceUtils.create(this).getResourceAsString(setupScriptItem); String script = TemplateProcessor.processTemplateContents(scriptContent, getManagementContext(), substitutions); if (windows) { ((WinRmMachineLocation) machineLocation) .executeCommand(ImmutableList.copyOf((script.replace("\r", "").split("\n")))); } else { ((SshMachineLocation) machineLocation).execCommands("Customizing node " + this, ImmutableList.of(script)); } } } if (setup.get(JcloudsLocationConfig.MAP_DEV_RANDOM_TO_DEV_URANDOM)) { if (windows) { LOG.warn("Ignoring flag MAP_DEV_RANDOM_TO_DEV_URANDOM on Windows location {}", machineLocation); } else { customisationForLogging.add("point /dev/random to urandom"); ((SshMachineLocation) machineLocation).execCommands("using urandom instead of random", Arrays.asList("sudo mv /dev/random /dev/random-real", "sudo ln -s /dev/urandom /dev/random")); } } if (setup.get(GENERATE_HOSTNAME)) { if (windows) { // TODO: Generate Windows Hostname LOG.warn("Ignoring flag GENERATE_HOSTNAME on Windows location {}", machineLocation); } else { customisationForLogging.add("configure hostname"); ((SshMachineLocation) machineLocation).execCommands("Generate hostname " + node.getName(), Arrays.asList("sudo hostname " + node.getName(), "sudo sed -i \"s/HOSTNAME=.*/HOSTNAME=" + node.getName() + "/g\" /etc/sysconfig/network", "sudo bash -c \"echo 127.0.0.1 `hostname` >> /etc/hosts\"")); } } if (setup.get(OPEN_IPTABLES)) { if (windows) { LOG.warn("Ignoring DEPRECATED flag OPEN_IPTABLES on Windows location {}", machineLocation); } else { LOG.warn( "Using DEPRECATED flag OPEN_IPTABLES (will not be supported in future versions) for {} at {}", machineLocation, this); @SuppressWarnings("unchecked") Iterable<Integer> inboundPorts = (Iterable<Integer>) setup.get(INBOUND_PORTS); if (inboundPorts == null || Iterables.isEmpty(inboundPorts)) { LOG.info("No ports to open in iptables (no inbound ports) for {} at {}", machineLocation, this); } else { customisationForLogging.add("open iptables"); List<String> iptablesRules = Lists.newArrayList(); if (isLocationFirewalldEnabled((SshMachineLocation) machineLocation)) { for (Integer port : inboundPorts) { iptablesRules.add(IptablesCommands.addFirewalldRule(Chain.INPUT, Protocol.TCP, port, Policy.ACCEPT)); } } else { iptablesRules = createIptablesRulesForNetworkInterface(inboundPorts); iptablesRules.add(IptablesCommands.saveIptablesRules()); } List<String> batch = Lists.newArrayList(); // Some entities, such as Riak (erlang based) have a huge range of ports, which leads to a script that // is too large to run (fails with a broken pipe). Batch the rules into batches of 50 for (String rule : iptablesRules) { batch.add(rule); if (batch.size() == 50) { ((SshMachineLocation) machineLocation) .execCommands("Inserting iptables rules, 50 command batch", batch); batch.clear(); } } if (batch.size() > 0) { ((SshMachineLocation) machineLocation).execCommands("Inserting iptables rules", batch); } ((SshMachineLocation) machineLocation).execCommands("List iptables rules", ImmutableList.of(IptablesCommands.listIptablesRule())); } } } if (setup.get(STOP_IPTABLES)) { if (windows) { LOG.warn("Ignoring DEPRECATED flag OPEN_IPTABLES on Windows location {}", machineLocation); } else { LOG.warn( "Using DEPRECATED flag STOP_IPTABLES (will not be supported in future versions) for {} at {}", machineLocation, this); customisationForLogging.add("stop iptables"); List<String> cmds = ImmutableList.<String>of(); if (isLocationFirewalldEnabled((SshMachineLocation) machineLocation)) { cmds = ImmutableList.of(IptablesCommands.firewalldServiceStop(), IptablesCommands.firewalldServiceStatus()); } else { cmds = ImmutableList.of(IptablesCommands.iptablesServiceStop(), IptablesCommands.iptablesServiceStatus()); } ((SshMachineLocation) machineLocation).execCommands("Stopping iptables", cmds); } } List<String> extraKeyUrlsToAuth = setup.get(EXTRA_PUBLIC_KEY_URLS_TO_AUTH); if (extraKeyUrlsToAuth != null && !extraKeyUrlsToAuth.isEmpty()) { if (windows) { LOG.warn("Ignoring flag EXTRA_PUBLIC_KEY_URLS_TO_AUTH on Windows location", machineLocation); } else { List<String> extraKeyDataToAuth = MutableList.of(); for (String keyUrl : extraKeyUrlsToAuth) { extraKeyDataToAuth.add(ResourceUtils.create().getResourceAsString(keyUrl)); } ((SshMachineLocation) machineLocation).execCommands("Authorizing ssh keys", ImmutableList.of(new AuthorizeRSAPublicKeys(extraKeyDataToAuth) .render(org.jclouds.scriptbuilder.domain.OsFamily.UNIX))); } } } else { // Otherwise we have deliberately not waited to be ssh'able, so don't try now to // ssh to exec these commands! } // Apply any optional app-specific customization. for (JcloudsLocationCustomizer customizer : getCustomizers(setup)) { LOG.debug("Customizing machine {}, using customizer {}", machineLocation, customizer); customizer.customize(this, computeService, machineLocation); } for (MachineLocationCustomizer customizer : getMachineCustomizers(setup)) { LOG.debug("Customizing machine {}, using customizer {}", machineLocation, customizer); customizer.customize(machineLocation); } customizedTimestamp = Duration.of(provisioningStopwatch); try { String logMessage = "Finished VM " + setup.getDescription() + " creation:" + " " + machineLocation.getUser() + "@" + machineLocation.getAddress() + ":" + machineLocation.getPort() + (Boolean.TRUE.equals(setup.get(LOG_CREDENTIALS)) ? "password=" + userCredentials.getOptionalPassword().or("<absent>") + " && key=" + userCredentials.getOptionalPrivateKey().or("<absent>") : "") + " ready after " + Duration.of(provisioningStopwatch).toStringRounded() + " (" + "semaphore obtained in " + Duration.of(semaphoreTimestamp).toStringRounded() + ";" + template + " template built in " + Duration.of(templateTimestamp).subtract(semaphoreTimestamp).toStringRounded() + ";" + " " + node + " provisioned in " + Duration.of(provisionTimestamp).subtract(templateTimestamp).toStringRounded() + ";" + " " + machineLocation + " connection usable in " + Duration.of(usableTimestamp).subtract(provisionTimestamp).toStringRounded() + ";" + " and os customized in " + Duration.of(customizedTimestamp).subtract(usableTimestamp).toStringRounded() + " - " + Joiner.on(", ").join(customisationForLogging) + ")"; LOG.info(logMessage); } catch (Exception e) { // TODO Remove try-catch! @Nakomis: why did you add it? What exception happened during logging? Exceptions.propagateIfFatal(e); LOG.warn("Problem generating log message summarising completion of jclouds machine provisioning " + machineLocation + " by " + this, e); } return machineLocation; } catch (Exception e) { if (e instanceof RunNodesException && ((RunNodesException) e).getNodeErrors().size() > 0) { node = Iterables.get(((RunNodesException) e).getNodeErrors().keySet(), 0); } // sometimes AWS nodes come up busted (eg ssh not allowed); just throw it back (and maybe try for another one) boolean destroyNode = (node != null) && Boolean.TRUE.equals(setup.get(DESTROY_ON_FAILURE)); if (e.toString().contains("VPCResourceNotSpecified")) { LOG.error( "Detected that your EC2 account is a legacy 'classic' account, but the recommended instance type requires VPC. " + "You can specify the 'eu-central-1' region to avoid this problem, or you can specify a classic-compatible instance type, " + "or you can specify a subnet to use with 'networkName' " + "(taking care that the subnet auto-assigns public IP's and allows ingress on all ports, " + "as Brooklyn does not currently configure security groups for non-default VPC's; " + "or setting up Brooklyn to be in the subnet or have a jump host or other subnet access configuration). " + "For more information on VPC vs classic see http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-vpc.html."); } LOG.error( "Failed to start VM for " + setup.getDescription() + (destroyNode ? " (destroying)" : "") + (node != null ? "; node " + node : "") + " after " + Duration.of(provisioningStopwatch).toStringRounded() + (semaphoreTimestamp != null ? " (" + "semaphore obtained in " + Duration.of(semaphoreTimestamp).toStringRounded() + ";" + (templateTimestamp != null && semaphoreTimestamp != null ? " template built in " + Duration.of(templateTimestamp) .subtract(semaphoreTimestamp).toStringRounded() + ";" : "") + (provisionTimestamp != null && templateTimestamp != null ? " node provisioned in " + Duration.of(provisionTimestamp) .subtract(templateTimestamp).toStringRounded() + ";" : "") + (usableTimestamp != null && provisioningStopwatch != null ? " connection usable in " + Duration.of(usableTimestamp) .subtract(provisionTimestamp).toStringRounded() + ";" : "") + (customizedTimestamp != null && usableTimestamp != null ? " and OS customized in " + Duration.of(customizedTimestamp) .subtract(usableTimestamp).toStringRounded() : "") + ")" : "") + ": " + e.getMessage()); LOG.debug(Throwables.getStackTraceAsString(e)); if (destroyNode) { Stopwatch destroyingStopwatch = Stopwatch.createStarted(); if (machineLocation != null) { releaseSafely(machineLocation); } else { releaseNodeSafely(node); } LOG.info("Destroyed " + (machineLocation != null ? "machine " + machineLocation : "node " + node) + " in " + Duration.of(destroyingStopwatch).toStringRounded()); } throw Exceptions.propagate(e); } }