Java tutorial
/* * Copyright 2014 WANdisco * * WANdisco licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations * under the License. */ package c5db.discovery; import c5db.codec.UdpProtostuffDecoder; import c5db.codec.UdpProtostuffEncoder; import c5db.discovery.generated.Availability; import c5db.discovery.generated.ModuleDescriptor; import c5db.interfaces.DiscoveryModule; import c5db.interfaces.ModuleInformationProvider; import c5db.interfaces.discovery.NewNodeVisible; import c5db.interfaces.discovery.NodeInfo; import c5db.interfaces.discovery.NodeInfoReply; import c5db.interfaces.discovery.NodeInfoRequest; import c5db.messages.generated.ModuleType; import c5db.util.C5Futures; import c5db.util.FiberOnly; import c5db.util.FiberSupplier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.net.InetAddresses; import com.google.common.util.concurrent.AbstractService; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelOption; import io.netty.channel.ChannelPipeline; import io.netty.channel.EventLoopGroup; import io.netty.channel.SimpleChannelInboundHandler; import io.netty.channel.socket.DatagramChannel; import io.netty.channel.socket.nio.NioDatagramChannel; import org.jetbrains.annotations.NotNull; import org.jetlang.channels.MemoryChannel; import org.jetlang.channels.MemoryRequestChannel; import org.jetlang.channels.Request; import org.jetlang.channels.RequestChannel; import org.jetlang.channels.Subscriber; import org.jetlang.fibers.Fiber; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.NetworkInterface; import java.net.SocketException; import java.util.ArrayList; import java.util.Enumeration; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import static c5db.codec.UdpProtostuffEncoder.UdpProtostuffMessage; /** * Uses broadcast UDP packets to discover 'adjacent' nodes in the cluster. Maintains * a state table for them, and provides information to other modules as they request it. * <p> * Currently UDP broadcast has some issues on Mac OSX vs Linux. The big question, * specifically, is what happens when multiple processes bind to 255.255.255.255:PORT * and send packets? Which processes receive such packets? * <ul> * <li>On Mac OSX 10.8/9, all processes reliably receive all packets including * the originating process</li> * <li>On Linux (Ubuntu, modern) a variety of things appear to occur: * <ul> * <li>First to bind receives all packets</li> * <li>All processes receives all packets</li> * <li>No one receives any packets</li> * <li>Please fill this doc in!</li> * </ul></li> * </ul> * <p> * The beacon service needs to be refactored and different discovery methods need to be * pluggable but all behind the discovery module interface. */ public class BeaconService extends AbstractService implements DiscoveryModule { private static final Logger LOG = LoggerFactory.getLogger(BeaconService.class); private static final InetAddress BROADCAST_ADDRESS = InetAddresses.forString("255.255.255.255"); private static final int BEACON_SERVICE_INITIAL_BROADCAST_DELAY_MILLISECONDS = 2000; private static final int BEACON_SERVICE_BROADCAST_PERIOD_MILLISECONDS = 10000; @Override public ModuleType getModuleType() { return ModuleType.Discovery; } @Override public boolean hasPort() { return true; } @Override public int port() { return discoveryPort; } @Override public String acceptCommand(String commandString) { return null; } private final RequestChannel<NodeInfoRequest, NodeInfoReply> nodeInfoRequests = new MemoryRequestChannel<>(); @Override public RequestChannel<NodeInfoRequest, NodeInfoReply> getNodeInfo() { return nodeInfoRequests; } @Override public ListenableFuture<NodeInfoReply> getNodeInfo(long nodeId, ModuleType module) { SettableFuture<NodeInfoReply> future = SettableFuture.create(); fiber.execute(() -> { NodeInfo peer = peerNodeInfoMap.get(nodeId); if (peer == null) { future.set(NodeInfoReply.NO_REPLY); } else { Integer servicePort = peer.modules.get(module); if (servicePort == null) { future.set(NodeInfoReply.NO_REPLY); } else { List<String> peerAddresses = peer.availability.getAddressesList(); future.set(new NodeInfoReply(true, peerAddresses, servicePort)); } } }); return future; } @FiberOnly private void handleNodeInfoRequest(Request<NodeInfoRequest, NodeInfoReply> message) { NodeInfoRequest req = message.getRequest(); NodeInfo peer = peerNodeInfoMap.get(req.nodeId); if (peer == null) { message.reply(NodeInfoReply.NO_REPLY); return; } Integer servicePort = peer.modules.get(req.moduleType); if (servicePort == null) { message.reply(NodeInfoReply.NO_REPLY); return; } List<String> peerAddresses = peer.availability.getAddressesList(); if (peerAddresses == null || peerAddresses.isEmpty()) { message.reply(NodeInfoReply.NO_REPLY); return; } // does this module run on that peer? message.reply(new NodeInfoReply(true, peerAddresses, servicePort)); } @Override public String toString() { return "BeaconService{" + "discoveryPort=" + discoveryPort + ", nodeId=" + nodeId + '}'; } // For main system modules/pubsub stuff. private final long nodeId; private final int discoveryPort; private final EventLoopGroup eventLoopGroup; private final InetSocketAddress broadcastAddress; private final InetSocketAddress loopbackAddress; private final Map<Long, NodeInfo> peerNodeInfoMap = new HashMap<>(); private final org.jetlang.channels.Channel<Availability> incomingMessages = new MemoryChannel<>(); private final org.jetlang.channels.Channel<NewNodeVisible> newNodeVisibleChannel = new MemoryChannel<>(); private final ModuleInformationProvider moduleInformationProvider; private final FiberSupplier fiberSupplier; // These should be final, but they are initialized in doStart(). private Channel broadcastChannel = null; private Bootstrap bootstrap = null; private List<String> localIPs; private Fiber fiber; // This field is updated when modules' availability changes. It must only be accessed from the fiber. private ImmutableMap<ModuleType, Integer> onlineModuleToPortMap = ImmutableMap.of(); private class BeaconMessageHandler extends SimpleChannelInboundHandler<Availability> { @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { LOG.warn("Exception, ignoring datagram", cause); } @Override protected void channelRead0(ChannelHandlerContext ctx, Availability msg) throws Exception { incomingMessages.publish(msg); } } /** * @param nodeId the id of this node. * @param discoveryPort the port to send discovery beacon messages on, and to listen to * for messages from others * @param eventLoopGroup An EventLoopGroup that's not shut down. * @param moduleInformationProvider Used to receive module availability updates */ public BeaconService(long nodeId, int discoveryPort, EventLoopGroup eventLoopGroup, ModuleInformationProvider moduleInformationProvider, FiberSupplier fiberSupplier) { this.nodeId = nodeId; this.discoveryPort = discoveryPort; this.eventLoopGroup = eventLoopGroup; this.moduleInformationProvider = moduleInformationProvider; this.fiberSupplier = fiberSupplier; this.broadcastAddress = new InetSocketAddress(BROADCAST_ADDRESS, discoveryPort); this.loopbackAddress = new InetSocketAddress(InetAddress.getLoopbackAddress(), discoveryPort); } @Override public ListenableFuture<ImmutableMap<Long, NodeInfo>> getState() { final SettableFuture<ImmutableMap<Long, NodeInfo>> future = SettableFuture.create(); fiber.execute(() -> { future.set(getCopyOfState()); }); return future; } @Override public Subscriber<NewNodeVisible> getNewNodeNotifications() { return newNodeVisibleChannel; } private ImmutableMap<Long, NodeInfo> getCopyOfState() { return ImmutableMap.copyOf(peerNodeInfoMap); } @FiberOnly private void sendBeacon() { if (broadcastChannel == null) { LOG.debug("Channel not available yet, deferring beacon send"); return; } LOG.trace("Sending beacon broadcast message to {}", broadcastAddress); List<ModuleDescriptor> msgModules = new ArrayList<>(onlineModuleToPortMap.size()); for (ModuleType moduleType : onlineModuleToPortMap.keySet()) { msgModules.add(new ModuleDescriptor(moduleType, onlineModuleToPortMap.get(moduleType))); } if (!localIPs.isEmpty()) { Availability beaconMessage = new Availability(nodeId, 0, localIPs, msgModules); broadcastChannel.writeAndFlush(new UdpProtostuffMessage<>(broadcastAddress, beaconMessage)) .addListener(future -> { if (!future.isSuccess()) { LOG.warn("node {} error sending message {} to broadcast address {}", nodeId, beaconMessage, broadcastAddress); } }); } List<String> loopbackIps = Lists.newArrayList(loopbackAddress.getAddress().getHostAddress()); Availability localMessage = new Availability(nodeId, 0, loopbackIps, msgModules); broadcastChannel.writeAndFlush(new UdpProtostuffMessage<>(loopbackAddress, localMessage)) .addListener(future -> { if (!future.isSuccess()) { LOG.warn("node {} error sending message {} to loopback address", nodeId, localMessage); } }); // Fix issue #76, feed back the beacon Message to our own database: processWireMessage(localMessage); } @FiberOnly private void processWireMessage(Availability message) { LOG.trace("Got incoming message {}", message); if (message.getNodeId() == 0) { // if (!message.hasNodeId()) { LOG.error("Incoming availability message does not have node id, ignoring!"); return; } // Always just overwrite what was already there for now. // TODO consider a more sophisticated merge strategy? NodeInfo nodeInfo = new NodeInfo(message); if (!peerNodeInfoMap.containsKey(message.getNodeId())) { newNodeVisibleChannel.publish(new NewNodeVisible(message.getNodeId(), nodeInfo)); } peerNodeInfoMap.put(message.getNodeId(), nodeInfo); } @Override protected void doStart() { eventLoopGroup.next().execute(() -> { bootstrap = new Bootstrap(); bootstrap.group(eventLoopGroup).channel(NioDatagramChannel.class) .option(ChannelOption.SO_BROADCAST, true).option(ChannelOption.SO_REUSEADDR, true) .handler(new ChannelInitializer<DatagramChannel>() { @Override protected void initChannel(DatagramChannel ch) throws Exception { ChannelPipeline p = ch.pipeline(); p.addLast("protobufDecoder", new UdpProtostuffDecoder<>(Availability.getSchema(), false)); p.addLast("protobufEncoder", new UdpProtostuffEncoder<>(Availability.getSchema(), false)); p.addLast("beaconMessageHandler", new BeaconMessageHandler()); } }); // Wait, this is why we are in a new executor... //noinspection RedundantCast bootstrap.bind(discoveryPort).addListener((ChannelFutureListener) future -> { if (future.isSuccess()) { broadcastChannel = future.channel(); } else { LOG.error("Unable to bind! ", future.cause()); notifyFailed(future.cause()); } }); try { localIPs = getLocalIPs(); } catch (SocketException e) { LOG.error("SocketException:", e); notifyFailed(e); return; } fiber = fiberSupplier.getNewFiber(this::notifyFailed); fiber.start(); // Schedule fiber tasks and subscriptions. incomingMessages.subscribe(fiber, this::processWireMessage); nodeInfoRequests.subscribe(fiber, this::handleNodeInfoRequest); moduleInformationProvider.moduleChangeChannel().subscribe(fiber, this::updateCurrentModulePorts); if (localIPs.isEmpty()) { LOG.warn( "Found no IP addresses to broadcast to other nodes; as a result, only sending to loopback"); } fiber.scheduleAtFixedRate(this::sendBeacon, BEACON_SERVICE_INITIAL_BROADCAST_DELAY_MILLISECONDS, BEACON_SERVICE_BROADCAST_PERIOD_MILLISECONDS, TimeUnit.MILLISECONDS); C5Futures.addCallback(moduleInformationProvider.getOnlineModules(), (ImmutableMap<ModuleType, Integer> onlineModuleToPortMap) -> { updateCurrentModulePorts(onlineModuleToPortMap); notifyStarted(); }, this::notifyFailed, fiber); }); } @Override protected void doStop() { fiber.dispose(); fiber = null; eventLoopGroup.next().execute(this::notifyStopped); } @FiberOnly private void updateCurrentModulePorts(ImmutableMap<ModuleType, Integer> onlineModuleToPortMap) { if (onlineModuleToPortMap == null) { notifyFailed( new NullPointerException("received null instead of a map of online modules to their ports")); return; } this.onlineModuleToPortMap = onlineModuleToPortMap; } @NotNull private List<String> getLocalIPs() throws SocketException { List<String> ips = new LinkedList<>(); for (Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces(); interfaces .hasMoreElements();) { NetworkInterface networkInterface = interfaces.nextElement(); if (networkInterface.isPointToPoint()) { continue; //ignore tunnel type interfaces } for (Enumeration<InetAddress> addrs = networkInterface.getInetAddresses(); addrs.hasMoreElements();) { InetAddress addr = addrs.nextElement(); if (addr.isLoopbackAddress() || addr.isLinkLocalAddress() || addr.isAnyLocalAddress()) { continue; } ips.add(addr.getHostAddress()); } } return ips; } }