Java tutorial
/** * diqube: Distributed Query Base. * * Copyright (C) 2015 Bastian Gloeckle * * This file is part of diqube. * * diqube is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ package org.diqube.cluster; import java.io.IOException; import java.lang.Thread.UncaughtExceptionHandler; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.inject.Inject; import org.apache.thrift.TException; import org.diqube.cluster.ClusterLayoutStateMachine.RemoveNode; import org.diqube.cluster.ClusterLayoutStateMachine.SetTablesOfNode; import org.diqube.config.Config; import org.diqube.config.ConfigKey; import org.diqube.connection.ClusterNodeStatusDetailListener; import org.diqube.connection.Connection; import org.diqube.connection.ConnectionException; import org.diqube.connection.ConnectionPool; import org.diqube.connection.NodeAddress; import org.diqube.connection.OurNodeAddressProvider; import org.diqube.consensus.ConsensusClient; import org.diqube.consensus.ConsensusClient.ClosableProvider; import org.diqube.consensus.ConsensusClient.ConsensusClusterUnavailableException; import org.diqube.consensus.ConsensusClusterNodeAddressProvider; import org.diqube.consensus.ConsensusIsLeaderProvider; import org.diqube.consensus.ConsensusServer; import org.diqube.consensus.ConsensusStateMachineClientInterruptedException; import org.diqube.context.AutoInstatiate; import org.diqube.context.InjectOptional; import org.diqube.context.shutdown.ContextShutdownListener; import org.diqube.context.shutdown.ShutdownBefore; import org.diqube.listeners.ClusterManagerListener; import org.diqube.listeners.ServingListener; import org.diqube.listeners.TableLoadListener; import org.diqube.listeners.providers.LoadedTablesProvider; import org.diqube.listeners.providers.OurNodeAddressStringProvider; import org.diqube.remote.cluster.thrift.ClusterManagementService; import org.diqube.threads.ExecutorManager; import org.diqube.thrift.base.thrift.RNodeAddress; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Iterables; /** * Manages state of the diqube-server cluster, this nodes state in other cluster nodes and shares information about * that. * * <p> * This class ensures that the information other nodes have about this node is correct, it manages our nodes address. * * @author Bastian Gloeckle */ @AutoInstatiate public class ClusterManager implements ServingListener, TableLoadListener, OurNodeAddressStringProvider, ClusterNodeStatusDetailListener, OurNodeAddressProvider, ConsensusClusterNodeAddressProvider, ContextShutdownListener { private static final Logger logger = LoggerFactory.getLogger(ClusterManager.class); private static final String OUR_HOST_AUTOMATIC = "*"; @Config(ConfigKey.OUR_HOST) private String ourHost; @Config(ConfigKey.PORT) private int ourPort; private NodeAddress ourHostAddr; @Config(ConfigKey.CLUSTER_NODES) private String clusterNodesConfigString; @Inject private ConnectionPool connectionPool; @InjectOptional private List<ClusterManagerListener> clusterManagerListeners; /** will contain "this", too! */ @InjectOptional private List<ClusterNodeStatusDetailListener> clusterNodeDiedListeners; private List<NodeAddress> consensusClusterNodes = new ArrayList<>(); @Inject private ClusterLayout clusterLayout; @Inject private ConsensusClient consensusClient; @Inject private ConsensusIsLeaderProvider consensusIsLeaderProvider; @Inject private LoadedTablesProvider loadedTablesProvider; @Inject private ExecutorManager executorManager; /** * Disable the methods of {@link ClusterNodeStatusDetailListener} on startup, until we have initially found some * cluster nodes. */ private boolean clusterNodeStatusDetailListenerDisabled = true; private ExecutorService executorService; @PostConstruct public void initialize() { if (ourHost.equals(OUR_HOST_AUTOMATIC)) { try { InetAddress foundAddr = InetAddress.getLocalHost(); ourHost = foundAddr.getHostAddress(); logger.info( "Using {} as our host address. We expect that other cluster nodes will be able to reach this " + "node under that address. If not, define a different host in the configuration!", ourHost); } catch (UnknownHostException e) { logger.error("Configuration said to identify our host automatically, " + "but was not able to inspect network interfaces.", e); throw new RuntimeException("Configuration said to identify our host automatically, " + "but was not able to inspect network interfaces.", e); } } else logger.info( "Using {} as our host address. We expect that other cluster nodes will be able to reach this node " + "under that address!", ourHost); ourHostAddr = new NodeAddress(ourHost, (short) ourPort); executorService = executorManager.newCachedThreadPool("clustermanager-%d", new UncaughtExceptionHandler() { @Override public void uncaughtException(Thread t, Throwable e) { logger.error("Error while executing asynchronous ClusterManager task", e); // swallow otherwise, as we'd like to continue as well as possible. } }); } @PreDestroy public void cleanup() { if (executorService != null) executorService.shutdownNow(); } @Override @ShutdownBefore({ ConsensusClient.class, ConsensusServer.class }) public void contextAboutToShutdown() { // try to gracefully tell the ClusterLayout that we're gone. If it does not work within a second, skip it. The other // nodes might then try to submit stuff to our node, but will soon discover that we're down and remove us from the // ClusterLayout themselves. try { logger.debug("Trying to remove ourselves from the cluster layout, as we're shutting down..."); executorService.submit(() -> { try (ClosableProvider<ClusterLayoutStateMachine> p = consensusClient .getStateMachineClient(ClusterLayoutStateMachine.class)) { p.getClient().removeNode(RemoveNode.local(ourHostAddr)); } catch (ConsensusClusterUnavailableException e) { logger.warn("Could not access consensus cluster to remove ourselves from cluster layout."); } }).get(1, TimeUnit.SECONDS); } catch (TimeoutException | InterruptedException | ExecutionException e) { logger.warn( "Could not deregister from cluster layout gracefully. The other cluster nodes might show exceptions " + "about this soon, but the cluster should recover.", e); } } private List<NodeAddress> parseClusterNodes(String clusterNodes) { List<NodeAddress> res = new ArrayList<>(); for (String clusterNodeString : clusterNodes.split(",")) { int lastColon = clusterNodeString.lastIndexOf(":"); if (lastColon == -1) { logger.warn("No port specified in '{}'. Ignoring.", clusterNodeString); continue; } if (lastColon == 0) { logger.warn("No host specified in '{}'. Ignoring.", clusterNodeString); continue; } short port; try { port = Short.valueOf(clusterNodeString.substring(lastColon + 1)); } catch (NumberFormatException e) { logger.warn("Could not parse port in '{}'. Ignoring.", clusterNodeString); continue; } String host = clusterNodeString.substring(0, lastColon); res.add(new NodeAddress(host, port)); } if (res.isEmpty()) return null; return res; } @Override public void localServerStartedServing() { if (clusterNodesConfigString == null || "".equals(clusterNodesConfigString)) { logger.info("There are no cluster nodes configured, will therefore not connect anywhere."); if (clusterManagerListeners != null) clusterManagerListeners.forEach(l -> l.clusterInitialized()); return; } List<NodeAddress> initialClusterNodes = parseClusterNodes(this.clusterNodesConfigString); if (initialClusterNodes == null) { logger.warn("There are no cluster nodes configured, will therefore not connect anywhere."); if (clusterManagerListeners != null) clusterManagerListeners.forEach(l -> l.clusterInitialized()); return; } logger.debug("Starting to communicate to cluster using the configured hosts ({})...", initialClusterNodes); try { // use the first node we can contact to fetch a list of all cluster nodes it knows. That list will later be used // to startup the consensus node. Set<RNodeAddress> allClusterNodes = new HashSet<>(); for (NodeAddress nodeAddr : initialClusterNodes) { try (Connection<ClusterManagementService.Iface> conn = reserveConnection(nodeAddr)) { allClusterNodes.addAll(conn.getService().getAllKnownClusterNodes()); } catch (ConnectionException | TException | IOException e) { logger.warn("Could not contact cluster node at {}.", nodeAddr, e); } } if (allClusterNodes.isEmpty()) { logger.warn("There are no cluster nodes alive, will therefore not connect anywhere."); if (clusterManagerListeners != null) clusterManagerListeners.forEach(l -> l.clusterInitialized()); return; } allClusterNodes.forEach(remoteAddr -> consensusClusterNodes.add(new NodeAddress(remoteAddr))); } catch (InterruptedException e) { logger.error("Interrupted while starting to communicate with cluster", e); return; } logger.info("Gathered {} node addresses of the cluster (limit): {}", consensusClusterNodes.size(), Iterables.limit(consensusClusterNodes, 100)); // enable activity when dead or alive nodes are identified. clusterNodeStatusDetailListenerDisabled = false; if (clusterManagerListeners != null) clusterManagerListeners.forEach(l -> l.clusterInitialized()); } private Connection<ClusterManagementService.Iface> reserveConnection(NodeAddress addr) throws ConnectionException, InterruptedException { return connectionPool.reserveConnection(ClusterManagementService.Iface.class, addr.createRemote(), null /* node will be removed automatically from ClusterManager, therefore no separate listener needed */); } @Override public void localServerStoppedServing() { // noop. } @Override public void nodeDied(RNodeAddress diedAddr) { if (clusterNodeStatusDetailListenerDisabled) // Disable during startup, as we do not want to act on "dead" nodes of the config file. return; // This will typically be called when a connection to a node fails. We will not remove the node from the consensus // cluster (as that would allow split-brains), but we ensure that its information is removed from the clusterLayout // across the consensus cluster. That way, no connections will be opened to that cluster node for queries any more // etc. We have to ensure that we integrate the current information again as soon as the node gets back online (=the // node gets restarted which would be a normal join to the consensus cluster or if the e.g. network partition ends // and we can communicate with the node again without it re-joining the cluster). if (diedAddr.isSetDefaultAddr()) { NodeAddress addr = new NodeAddress(diedAddr); logger.trace( "Cluster node died. Checking consensus cluster if we need to distribute that information..."); // execute asynchronously, as this might take some time and we might even still be in startup (e.g. internal // consensus cluster server startup). executorService.execute(() -> { try { if (clusterLayout.isNodeKnown(addr)) { logger.info( "Cluster node died: {}. Distributing information on changed cluster layout in consensus cluster.", addr); // This might actually be executed by multiple cluster nodes in parallel, but that does not hurt that much, // as node deaths should be rare. try (ClosableProvider<ClusterLayoutStateMachine> p = consensusClient .getStateMachineClient(ClusterLayoutStateMachine.class)) { p.getClient().removeNode(RemoveNode.local(addr)); } catch (ConsensusClusterUnavailableException e) { logger.warn( "Could not remove node {} from cluster layout since consensus cluster is unavailable", addr); } } else logger.trace( "Cluster node died. No need to distribute information since that node was unknown to the " + "consensus cluster anyway."); } catch (InterruptedException | ConsensusStateMachineClientInterruptedException | ConsensusClusterUnavailableException e) { // exit quietly. } }); } } @Override public void nodeAlive(RNodeAddress remoteNodeAddr) throws InterruptedException { if (clusterNodeStatusDetailListenerDisabled) // Disable during startup, as we are not yet interesting in "alive" nodes - we will receive cluster layout // information automatically if we join a cluster (= our consensus log will be filled) or if we're a single node // setup, there are no nodes anyway. return; // This will typically be called on the consensus master node when a new node joined or became alive again, as the // consensus master periodically sends keepAlives to all nodes. We ensure here that we get current information about // that new node. if (!consensusIsLeaderProvider.isLeader()) // Only let the consensus leader find new alive nodes. This is to reduce the number of times a new node is asked // to "publishLoadedTables" and also to limit the number of times "clusterLayout.isNodeKnown" is called: This is // pretty slow on non-leader nodes, but we will receive a lot of "nodeAlive" calls. return; if (remoteNodeAddr.isSetDefaultAddr()) { NodeAddress addr = new NodeAddress(remoteNodeAddr); try { if (!clusterLayout.isNodeKnown(addr)) { logger.info( "Cluster node seems to be accessible now: {}. As we do not have information on the tables this " + "new node serves, we ask it to publicize that.", addr); try (Connection<ClusterManagementService.Iface> conn = reserveConnection(addr)) { conn.getService().publishLoadedTablesInConsensus(); } catch (ConnectionException | TException | IOException e) { logger.warn("Could not contact cluster node at {}.", addr, e); } } } catch (ConsensusClusterUnavailableException e) { logger.warn("Could not inform cluster about the node {} becoming alive, since the consensus " + "cluster is not reachable", e); } } } @Override public synchronized void tableLoaded(String newTableName) throws AbortTableLoadException { logger.info("Informing consensus cluster of our updated table list."); try (ClosableProvider<ClusterLayoutStateMachine> p = consensusClient .getStateMachineClient(ClusterLayoutStateMachine.class)) { p.getClient().setTablesOfNode( SetTablesOfNode.local(ourHostAddr, loadedTablesProvider.getNamesOfLoadedTables())); } catch (ConsensusClusterUnavailableException e) { logger.error("Table cannot be loaded because consensus cluster is not available", e); throw new AbortTableLoadException("Table cannot be loaded because consensus cluster is not available", e); } logger.trace("Informed consensus cluster of our updated table list."); } @Override public void tableUnloaded(String tableName) { logger.info("Informing consensus cluster of our updated table list."); try (ClosableProvider<ClusterLayoutStateMachine> p = consensusClient .getStateMachineClient(ClusterLayoutStateMachine.class)) { p.getClient().setTablesOfNode( SetTablesOfNode.local(ourHostAddr, loadedTablesProvider.getNamesOfLoadedTables())); logger.trace("Informed consensus cluster of our updated table list."); } catch (ConsensusClusterUnavailableException e) { logger.warn("Could not inform consensus cluster that we do not serve the table any more.", e); } } @Override public NodeAddress getOurNodeAddress() { return ourHostAddr; } @Override public String getOurNodeAddressAsString() { return ourHostAddr.toString(); } @Override public List<NodeAddress> getClusterNodeAddressesForConsensus() { return consensusClusterNodes; } }