Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.brooklyn.entity.nosql.cassandra; import java.util.Collection; import java.util.Map; import java.util.Set; import org.apache.brooklyn.api.entity.Entity; import org.apache.brooklyn.api.entity.EntitySpec; import org.apache.brooklyn.api.location.Location; import org.apache.brooklyn.api.policy.PolicySpec; import org.apache.brooklyn.api.sensor.SensorEvent; import org.apache.brooklyn.api.sensor.SensorEventListener; import org.apache.brooklyn.core.entity.Attributes; import org.apache.brooklyn.core.entity.Entities; import org.apache.brooklyn.core.entity.EntityInternal; import org.apache.brooklyn.core.entity.EntityPredicates; import org.apache.brooklyn.core.entity.lifecycle.Lifecycle; import org.apache.brooklyn.entity.group.AbstractMembershipTrackingPolicy; import org.apache.brooklyn.entity.group.DynamicFabricImpl; import org.apache.brooklyn.entity.group.DynamicGroup; import org.apache.brooklyn.util.collections.CollectionFunctionals; import org.apache.brooklyn.util.collections.MutableMap; import org.apache.brooklyn.util.collections.MutableSet; import org.apache.brooklyn.util.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Function; import com.google.common.base.Objects; import com.google.common.base.Optional; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; /** * Implementation of {@link CassandraDatacenter}. * <p> * Serveral subtleties to note: * - a node may take some time after it is running and serving JMX to actually be contactable on its thrift port * (so we wait for thrift port to be contactable) * - sometimes new nodes take a while to peer, and/or take a while to get a consistent schema * (each up to 1m; often very close to the 1m) */ public class CassandraFabricImpl extends DynamicFabricImpl implements CassandraFabric { private static final Logger log = LoggerFactory.getLogger(CassandraFabricImpl.class); // Mutex for synchronizing during re-size operations private final Object mutex = new Object[0]; private final Supplier<Set<Entity>> defaultSeedSupplier = new Supplier<Set<Entity>>() { @Override public Set<Entity> get() { // TODO Remove duplication from CassandraClusterImpl.defaultSeedSupplier Set<Entity> seeds = getAttribute(CURRENT_SEEDS); boolean hasPublishedSeeds = Boolean.TRUE.equals(getAttribute(HAS_PUBLISHED_SEEDS)); int quorumSize = getSeedQuorumSize(); // update seeds if we're not quorate; note this may not work for dynamically adding new datacenters // as we do not take a new seed from the new datacenter if (seeds == null || seeds.size() < quorumSize || containsDownEntity(seeds)) { Set<Entity> newseeds; Map<CassandraDatacenter, Set<Entity>> potentialSeeds = MutableMap.of(); int potentialSeedCount = 0; for (CassandraDatacenter member : Iterables.filter(getMembers(), CassandraDatacenter.class)) { Set<Entity> dcPotentialSeeds = member.gatherPotentialSeeds(); potentialSeeds.put(member, dcPotentialSeeds); potentialSeedCount += dcPotentialSeeds.size(); } if (hasPublishedSeeds) { Set<Entity> currentSeeds = getAttribute(CURRENT_SEEDS); Lifecycle serviceState = getAttribute(SERVICE_STATE_ACTUAL); if (serviceState == Lifecycle.STARTING) { if (Sets.intersection(currentSeeds, ImmutableSet.copyOf(Iterables.concat(potentialSeeds.values()))).isEmpty()) { log.warn( "Fabric {} lost all its seeds while starting! Subsequent failure likely, but changing seeds during startup would risk split-brain: seeds={}", new Object[] { CassandraFabricImpl.this, currentSeeds }); } newseeds = currentSeeds; } else if (serviceState == Lifecycle.STOPPING || serviceState == Lifecycle.STOPPED) { if (log.isTraceEnabled()) log.trace("Fabric {} ignoring any potential seed-changes, because {}: seeds={}", new Object[] { CassandraFabricImpl.this, serviceState, currentSeeds }); newseeds = currentSeeds; } else if (potentialSeedCount == 0) { // TODO Could be race where nodes have only just returned from start() and are about to // transition to serviceUp; so don't just abandon all our seeds! log.warn( "Fabric {} has no seeds (after startup); leaving seeds as-is; but risks split-brain if these seeds come back up!", new Object[] { CassandraFabricImpl.this }); newseeds = currentSeeds; } else if (!allNonEmpty(potentialSeeds.values())) { log.warn( "Fabric {} has datacenter with no seeds (after startup); leaving seeds as-is; but risks split-brain if these seeds come back up!", new Object[] { CassandraFabricImpl.this }); newseeds = currentSeeds; } else { Set<Entity> result = selectSeeds(quorumSize, potentialSeeds); if (log.isDebugEnabled() && !Objects.equal(seeds, result)) { log.debug("Fabric {} updating seeds: chosen={}; potential={}", new Object[] { CassandraFabricImpl.this, result, potentialSeeds }); } newseeds = result; } } else if (potentialSeedCount < quorumSize) { if (log.isDebugEnabled()) log.debug( "Not setting seeds of fabric {} yet, because still waiting for quorum (need {}; have {} potentials from {} members)", new Object[] { CassandraFabricImpl.this, quorumSize, potentialSeedCount, getMembers() }); newseeds = ImmutableSet.of(); } else if (!allNonEmpty(potentialSeeds.values())) { if (log.isDebugEnabled()) { Map<CassandraDatacenter, Integer> datacenterCounts = Maps.transformValues(potentialSeeds, CollectionFunctionals.sizeFunction()); log.debug( "Not setting seeds of fabric {} yet, because not all datacenters have seeds (sizes are {})", new Object[] { CassandraFabricImpl.this, datacenterCounts }); } newseeds = ImmutableSet.of(); } else { // yay, we're quorate Set<Entity> result = selectSeeds(quorumSize, potentialSeeds); log.info("Fabric {} has reached seed quorum: seeds={}", new Object[] { CassandraFabricImpl.this, result }); newseeds = result; } if (!Objects.equal(seeds, newseeds)) { sensors().set(CURRENT_SEEDS, newseeds); if (newseeds != null && newseeds.size() > 0) { sensors().set(HAS_PUBLISHED_SEEDS, true); // Need to tell every datacenter that seeds are ready. // Otherwise a datacenter might get no more changes (e.g. to nodes' hostnames etc), // and not call seedSupplier.get() again. for (CassandraDatacenter member : Iterables.filter(getMembers(), CassandraDatacenter.class)) { member.update(); } } return newseeds; } else { return seeds; } } else { if (log.isTraceEnabled()) log.trace( "Not refresheed seeds of fabric {}, because have quorum {} (of {} members), and none are down: seeds={}", new Object[] { CassandraFabricImpl.class, quorumSize, getMembers().size(), seeds }); return seeds; } } private boolean allNonEmpty(Collection<? extends Collection<Entity>> contenders) { for (Collection<Entity> contender : contenders) if (contender.isEmpty()) return false; return true; } private Set<Entity> selectSeeds(int num, Map<CassandraDatacenter, ? extends Collection<Entity>> contenders) { // Prefer existing seeds wherever possible; // otherwise prefer a seed from each sub-cluster; // otherwise accept any other contenders Set<Entity> currentSeeds = (getAttribute(CURRENT_SEEDS) != null) ? getAttribute(CURRENT_SEEDS) : ImmutableSet.<Entity>of(); MutableSet<Entity> result = MutableSet.of(); result.addAll(Sets.intersection(currentSeeds, ImmutableSet.copyOf(contenders.values()))); for (CassandraDatacenter cluster : contenders.keySet()) { Set<Entity> contendersInCluster = Sets.newLinkedHashSet(contenders.get(cluster)); if (contendersInCluster.size() > 0 && Sets.intersection(result, contendersInCluster).isEmpty()) { result.add(Iterables.getFirst(contendersInCluster, null)); } } result.addAll(Iterables.concat(contenders.values())); return ImmutableSet.copyOf(Iterables.limit(result, num)); } private boolean containsDownEntity(Set<Entity> seeds) { for (Entity seed : seeds) { if (!isViableSeed(seed)) { return true; } } return false; } public boolean isViableSeed(Entity member) { // TODO remove duplication from CassandraClusterImpl.SeedTracker.isViableSeed boolean managed = Entities.isManaged(member); String hostname = member.getAttribute(Attributes.HOSTNAME); boolean serviceUp = Boolean.TRUE.equals(member.getAttribute(Attributes.SERVICE_UP)); Lifecycle serviceState = member.getAttribute(Attributes.SERVICE_STATE_ACTUAL); boolean hasFailed = !managed || (serviceState == Lifecycle.ON_FIRE) || (serviceState == Lifecycle.RUNNING && !serviceUp) || (serviceState == Lifecycle.STOPPED); boolean result = (hostname != null && !hasFailed); if (log.isTraceEnabled()) log.trace( "Node {} in Fabric {}: viableSeed={}; hostname={}; serviceUp={}; serviceState={}; hasFailed={}", new Object[] { member, CassandraFabricImpl.this, result, hostname, serviceUp, serviceState, hasFailed }); return result; } }; public CassandraFabricImpl() { } @Override public void init() { super.init(); if (!config().getRaw(CassandraDatacenter.SEED_SUPPLIER).isPresentAndNonNull()) config().set(CassandraDatacenter.SEED_SUPPLIER, getSeedSupplier()); // track members policies().add(PolicySpec.create(MemberTrackingPolicy.class).displayName("Cassandra Fabric Tracker") .configure("group", this)); // Track first node's startup subscriptions().subscribeToMembers(this, CassandraDatacenter.FIRST_NODE_STARTED_TIME_UTC, new SensorEventListener<Long>() { @Override public void onEvent(SensorEvent<Long> event) { Long oldval = getAttribute(CassandraDatacenter.FIRST_NODE_STARTED_TIME_UTC); Long newval = event.getValue(); if (oldval == null && newval != null) { sensors().set(CassandraDatacenter.FIRST_NODE_STARTED_TIME_UTC, newval); for (CassandraDatacenter member : Iterables.filter(getMembers(), CassandraDatacenter.class)) { ((EntityInternal) member).sensors() .set(CassandraDatacenter.FIRST_NODE_STARTED_TIME_UTC, newval); } } } }); // Track the datacenters for this cluster subscriptions().subscribeToMembers(this, CassandraDatacenter.DATACENTER_USAGE, new SensorEventListener<Multimap<String, Entity>>() { @Override public void onEvent(SensorEvent<Multimap<String, Entity>> event) { Multimap<String, Entity> usage = calculateDatacenterUsage(); sensors().set(DATACENTER_USAGE, usage); sensors().set(DATACENTERS, usage.keySet()); } }); subscriptions().subscribe(this, DynamicGroup.MEMBER_REMOVED, new SensorEventListener<Entity>() { @Override public void onEvent(SensorEvent<Entity> event) { Multimap<String, Entity> usage = calculateDatacenterUsage(); sensors().set(DATACENTER_USAGE, usage); sensors().set(DATACENTERS, usage.keySet()); } }); } public static class MemberTrackingPolicy extends AbstractMembershipTrackingPolicy { @Override protected void onEntityChange(Entity member) { if (log.isDebugEnabled()) log.debug("Location {} updated in Fabric {}", member, entity); ((CassandraFabricImpl) entity).update(); } @Override protected void onEntityAdded(Entity member) { if (log.isDebugEnabled()) log.debug("Location {} added to Fabric {}", member, entity); ((CassandraFabricImpl) entity).update(); } @Override protected void onEntityRemoved(Entity member) { if (log.isDebugEnabled()) log.debug("Location {} removed from Fabric {}", member, entity); ((CassandraFabricImpl) entity).update(); } }; protected int getSeedQuorumSize() { Integer quorumSize = getConfig(INITIAL_QUORUM_SIZE); if (quorumSize != null && quorumSize > 0) return quorumSize; int initialSizeSum = 0; for (CassandraDatacenter cluster : Iterables.filter(getMembers(), CassandraDatacenter.class)) { initialSizeSum += cluster.getConfig(CassandraDatacenter.INITIAL_SIZE); } if (initialSizeSum > 5) initialSizeSum /= 2; else if (initialSizeSum > 3) initialSizeSum -= 2; else if (initialSizeSum > 2) initialSizeSum -= 1; return Math.min(Math.max(initialSizeSum, 1), CassandraFabric.DEFAULT_SEED_QUORUM); } /** * Sets the default {@link #MEMBER_SPEC} to describe the Cassandra sub-clusters. */ @Override protected EntitySpec<?> getMemberSpec() { // Need to set the seedSupplier, even if the caller has overridden the CassandraCluster config // (unless they've explicitly overridden the seedSupplier as well!) // TODO probably don't need to anymore, as it is set on the Fabric here -- just make sure there is a default! EntitySpec<?> custom = getConfig(MEMBER_SPEC); if (custom == null) { return EntitySpec.create(CassandraDatacenter.class).configure(CassandraDatacenter.SEED_SUPPLIER, getSeedSupplier()); } else if (custom.getConfig().containsKey(CassandraDatacenter.SEED_SUPPLIER) || custom.getFlags().containsKey("seedSupplier")) { return custom; } else { return EntitySpec.create(custom).configure(CassandraDatacenter.SEED_SUPPLIER, getSeedSupplier()); } } @Override protected Entity createCluster(Location location, Map flags) { Function<Location, String> dataCenterNamer = getConfig(DATA_CENTER_NAMER); if (dataCenterNamer != null) { flags = ImmutableMap.builder().putAll(flags) .put(CassandraNode.DATACENTER_NAME, dataCenterNamer.apply(location)).build(); } return super.createCluster(location, flags); } /** * Prefers one node per location, and then others from anywhere. * Then trims result down to the "quorumSize". */ public Supplier<Set<Entity>> getSeedSupplier() { return defaultSeedSupplier; } @Override public void start(Collection<? extends Location> locations) { super.start(locations); connectSensors(); // TODO wait until all nodes which we think are up are consistent // i.e. all known nodes use the same schema, as reported by // SshEffectorTasks.ssh("echo \"describe cluster;\" | /bin/cassandra-cli"); // once we've done that we can revert to using 2 seed nodes. // see CassandraCluster.DEFAULT_SEED_QUORUM Time.sleep(getConfig(CassandraDatacenter.DELAY_BEFORE_ADVERTISING_CLUSTER)); update(); } protected void connectSensors() { connectEnrichers(); } protected void connectEnrichers() { // TODO Aggregate across sub-clusters subscriptions().subscribeToMembers(this, SERVICE_UP, new SensorEventListener<Boolean>() { @Override public void onEvent(SensorEvent<Boolean> event) { sensors().set(SERVICE_UP, calculateServiceUp()); } }); } @Override public void stop() { disconnectSensors(); super.stop(); } protected void disconnectSensors() { } protected boolean calculateServiceUp() { Optional<Entity> upNode = Iterables.tryFind(getMembers(), EntityPredicates.attributeEqualTo(SERVICE_UP, Boolean.TRUE)); return upNode.isPresent(); } protected Multimap<String, Entity> calculateDatacenterUsage() { Multimap<String, Entity> result = LinkedHashMultimap.<String, Entity>create(); for (CassandraDatacenter member : Iterables.filter(getMembers(), CassandraDatacenter.class)) { Multimap<String, Entity> memberUsage = member.getAttribute(CassandraDatacenter.DATACENTER_USAGE); if (memberUsage != null) result.putAll(memberUsage); } return result; } @Override public void update() { synchronized (mutex) { for (CassandraDatacenter member : Iterables.filter(getMembers(), CassandraDatacenter.class)) { member.update(); } calculateServiceUp(); // Choose the first available location to set host and port (and compute one-up) Optional<Entity> upNode = Iterables.tryFind(getMembers(), EntityPredicates.attributeEqualTo(SERVICE_UP, Boolean.TRUE)); if (upNode.isPresent()) { sensors().set(HOSTNAME, upNode.get().getAttribute(Attributes.HOSTNAME)); sensors().set(THRIFT_PORT, upNode.get().getAttribute(CassandraNode.THRIFT_PORT)); } } } }