Java tutorial
/* * Copyright (c) 2015 The Jupiter Project * * Licensed under the Apache License, version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.jupiter.registry; import io.netty.bootstrap.Bootstrap; import io.netty.buffer.ByteBuf; import io.netty.channel.*; import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.codec.MessageToByteEncoder; import io.netty.handler.codec.ReplayingDecoder; import io.netty.util.Attribute; import io.netty.util.AttributeKey; import io.netty.util.ReferenceCountUtil; import org.jupiter.common.concurrent.collection.ConcurrentSet; import org.jupiter.common.util.*; import org.jupiter.common.util.internal.logging.InternalLogger; import org.jupiter.common.util.internal.logging.InternalLoggerFactory; import org.jupiter.serialization.Serializer; import org.jupiter.serialization.SerializerFactory; import org.jupiter.serialization.SerializerType; import org.jupiter.transport.*; import org.jupiter.transport.exception.ConnectFailedException; import org.jupiter.transport.exception.IoSignals; import org.jupiter.transport.netty.NettyTcpConnector; import org.jupiter.transport.netty.TcpChannelProvider; import org.jupiter.transport.netty.handler.AcknowledgeEncoder; import org.jupiter.transport.netty.handler.IdleStateChecker; import org.jupiter.transport.netty.handler.connector.ConnectionWatchdog; import org.jupiter.transport.netty.handler.connector.ConnectorIdleStateTrigger; import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.List; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import static org.jupiter.common.util.Preconditions.checkNotNull; import static org.jupiter.common.util.StackTraceUtil.stackTrace; /** * The client of registration center. * * jupiter * org.jupiter.registry * * @author jiachun.fjc */ public final class DefaultRegistry extends NettyTcpConnector { private static final InternalLogger logger = InternalLoggerFactory.getInstance(DefaultRegistry.class); private static final AttributeKey<ConcurrentSet<RegisterMeta.ServiceMeta>> C_SUBSCRIBE_KEY = AttributeKey .valueOf("client.subscribed"); private static final AttributeKey<ConcurrentSet<RegisterMeta>> C_PUBLISH_KEY = AttributeKey .valueOf("client.published"); // ack, ???? private final ConcurrentMap<Long, MessageNonAck> messagesNonAck = Maps.newConcurrentMap(); // handlers private final ConnectorIdleStateTrigger idleStateTrigger = new ConnectorIdleStateTrigger(); private final MessageHandler handler = new MessageHandler(); private final MessageEncoder encoder = new MessageEncoder(); private final AcknowledgeEncoder ackEncoder = new AcknowledgeEncoder(); // ?/???? private final SerializerType serializerType; { SerializerType expected = SerializerType .parse(SystemPropertyUtil.get("jupiter.registry.default.serializer_type")); serializerType = expected == null ? SerializerType.getDefault() : expected; } private final AbstractRegistryService registryService; // ?ConfigClient??channel private volatile Channel channel; public DefaultRegistry(AbstractRegistryService registryService) { this(registryService, 1); } public DefaultRegistry(AbstractRegistryService registryService, int nWorkers) { super(nWorkers); this.registryService = checkNotNull(registryService, "registryService"); } @Override protected void doInit() { // child options config().setOption(JOption.SO_REUSEADDR, true); config().setOption(JOption.CONNECT_TIMEOUT_MILLIS, (int) TimeUnit.SECONDS.toMillis(3)); // channel factory bootstrap().channelFactory(TcpChannelProvider.NIO_CONNECTOR); } /** * ConfigClient??, async? */ @Override public JConnection connect(UnresolvedAddress address, boolean async) { setOptions(); final Bootstrap boot = bootstrap(); final SocketAddress socketAddress = InetSocketAddress.createUnresolved(address.getHost(), address.getPort()); // ?watchdog final ConnectionWatchdog watchdog = new ConnectionWatchdog(boot, timer, socketAddress, null) { @Override public ChannelHandler[] handlers() { return new ChannelHandler[] { this, new IdleStateChecker(timer, 0, JConstants.WRITER_IDLE_TIME_SECONDS, 0), idleStateTrigger, new MessageDecoder(), encoder, ackEncoder, handler }; } }; watchdog.start(); try { ChannelFuture future; synchronized (bootstrapLock()) { boot.handler(new ChannelInitializer<NioSocketChannel>() { @Override protected void initChannel(NioSocketChannel ch) throws Exception { ch.pipeline().addLast(watchdog.handlers()); } }); future = boot.connect(socketAddress); } // ?synchronized??? future.sync(); channel = future.channel(); } catch (Throwable t) { throw new ConnectFailedException("connects to [" + address + "] fails", t); } return new JConnection(address) { @Override public void setReconnect(boolean reconnect) { if (reconnect) { watchdog.start(); } else { watchdog.stop(); } } }; } /** * Sent the subscription information to registry server. */ public void doSubscribe(RegisterMeta.ServiceMeta serviceMeta) { registryService.subscribeSet().add(serviceMeta); Message msg = new Message(serializerType.value()); msg.messageCode(JProtocolHeader.SUBSCRIBE_SERVICE); msg.data(serviceMeta); Channel ch = channel; // MessageHandler#channelActive()write if (attachSubscribeEventOnChannel(serviceMeta, ch)) { ch.writeAndFlush(msg).addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); MessageNonAck msgNonAck = new MessageNonAck(msg, ch); messagesNonAck.put(msgNonAck.id, msgNonAck); } } /** * Publishing service to registry server. */ public void doRegister(RegisterMeta meta) { registryService.registerMetaSet().add(meta); Message msg = new Message(serializerType.value()); msg.messageCode(JProtocolHeader.PUBLISH_SERVICE); msg.data(meta); Channel ch = channel; // MessageHandler#channelActive()write if (attachPublishEventOnChannel(meta, ch)) { ch.writeAndFlush(msg).addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); MessageNonAck msgNonAck = new MessageNonAck(msg, ch); messagesNonAck.put(msgNonAck.id, msgNonAck); } } /** * Notify to registry server unpublish corresponding service. */ public void doUnregister(final RegisterMeta meta) { registryService.registerMetaSet().remove(meta); Message msg = new Message(serializerType.value()); msg.messageCode(JProtocolHeader.PUBLISH_CANCEL_SERVICE); msg.data(meta); channel.writeAndFlush(msg).addListener(new ChannelFutureListener() { @Override public void operationComplete(ChannelFuture future) throws Exception { if (!future.isSuccess()) { Channel ch = future.channel(); if (ch.isActive()) { ch.pipeline().fireExceptionCaught(future.cause()); } else { if (logger.isWarnEnabled()) { logger.warn("Unregister {} fail because of channel is inactive: {}.", meta, stackTrace(future.cause())); } } } } }); MessageNonAck msgNonAck = new MessageNonAck(msg, channel); messagesNonAck.put(msgNonAck.id, msgNonAck); } private void handleAcknowledge(Acknowledge ack) { messagesNonAck.remove(ack.sequence()); } // channel(??) private static boolean attachPublishEventOnChannel(RegisterMeta meta, Channel channel) { Attribute<ConcurrentSet<RegisterMeta>> attr = channel.attr(C_PUBLISH_KEY); ConcurrentSet<RegisterMeta> registerMetaSet = attr.get(); if (registerMetaSet == null) { ConcurrentSet<RegisterMeta> newRegisterMetaSet = new ConcurrentSet<>(); registerMetaSet = attr.setIfAbsent(newRegisterMetaSet); if (registerMetaSet == null) { registerMetaSet = newRegisterMetaSet; } } return registerMetaSet.add(meta); } // channel(?) private static boolean attachSubscribeEventOnChannel(RegisterMeta.ServiceMeta serviceMeta, Channel channel) { Attribute<ConcurrentSet<RegisterMeta.ServiceMeta>> attr = channel.attr(C_SUBSCRIBE_KEY); ConcurrentSet<RegisterMeta.ServiceMeta> serviceMetaSet = attr.get(); if (serviceMetaSet == null) { ConcurrentSet<RegisterMeta.ServiceMeta> newServiceMetaSet = new ConcurrentSet<>(); serviceMetaSet = attr.setIfAbsent(newServiceMetaSet); if (serviceMetaSet == null) { serviceMetaSet = newServiceMetaSet; } } return serviceMetaSet.add(serviceMeta); } static class MessageNonAck { private final long id; private final Message msg; private final Channel channel; private final long timestamp = SystemClock.millisClock().now(); public MessageNonAck(Message msg, Channel channel) { this.msg = msg; this.channel = channel; id = msg.sequence(); } } /** * ************************************************************************************************** * Protocol * ? * 2 1 1 8 4 * * * MAGIC Sign Status Invoke Id Body Length Body Content * * * * ?16 * = 2 // magic = (short) 0xbabe * + 1 // ??, ?4???, ?4??? * + 1 // * + 8 // ? id, long * + 4 // ? body , int */ static class MessageDecoder extends ReplayingDecoder<MessageDecoder.State> { public MessageDecoder() { super(State.HEADER_MAGIC); } // ?? private final JProtocolHeader header = new JProtocolHeader(); @Override protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception { switch (state()) { case HEADER_MAGIC: checkMagic(in.readShort()); // MAGIC checkpoint(State.HEADER_SIGN); case HEADER_SIGN: header.sign(in.readByte()); // ?? checkpoint(State.HEADER_STATUS); case HEADER_STATUS: in.readByte(); // no-op checkpoint(State.HEADER_ID); case HEADER_ID: header.id(in.readLong()); // ?id checkpoint(State.HEADER_BODY_LENGTH); case HEADER_BODY_LENGTH: header.bodyLength(in.readInt()); // ? checkpoint(State.BODY); case BODY: byte s_code = header.serializerCode(); switch (header.messageCode()) { case JProtocolHeader.PUBLISH_SERVICE: case JProtocolHeader.PUBLISH_CANCEL_SERVICE: case JProtocolHeader.OFFLINE_NOTICE: { byte[] bytes = new byte[header.bodyLength()]; in.readBytes(bytes); Serializer serializer = SerializerFactory.getSerializer(s_code); Message msg = serializer.readObject(bytes, Message.class); msg.messageCode(header.messageCode()); out.add(msg); break; } case JProtocolHeader.ACK: out.add(new Acknowledge(header.id())); break; default: throw IoSignals.ILLEGAL_SIGN; } checkpoint(State.HEADER_MAGIC); } } private static void checkMagic(short magic) throws Signal { if (magic != JProtocolHeader.MAGIC) { throw IoSignals.ILLEGAL_MAGIC; } } enum State { HEADER_MAGIC, HEADER_SIGN, HEADER_STATUS, HEADER_ID, HEADER_BODY_LENGTH, BODY } } /** * ************************************************************************************************** * Protocol * ? * 2 1 1 8 4 * * * MAGIC Sign Status Invoke Id Body Length Body Content * * * * ?16 * = 2 // magic = (short) 0xbabe * + 1 // ??, ?4???, ?4??? * + 1 // * + 8 // ? id, long * + 4 // ? body , int */ @ChannelHandler.Sharable static class MessageEncoder extends MessageToByteEncoder<Message> { @Override protected void encode(ChannelHandlerContext ctx, Message msg, ByteBuf out) throws Exception { byte s_code = msg.serializerCode(); byte sign = JProtocolHeader.toSign(s_code, msg.messageCode()); Serializer serializer = SerializerFactory.getSerializer(s_code); byte[] bytes = serializer.writeObject(msg); out.writeShort(JProtocolHeader.MAGIC).writeByte(sign).writeByte(0).writeLong(0).writeInt(bytes.length) .writeBytes(bytes); } } @ChannelHandler.Sharable class MessageHandler extends ChannelInboundHandlerAdapter { @SuppressWarnings("unchecked") @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { Channel ch = ctx.channel(); if (msg instanceof Message) { Message obj = (Message) msg; switch (obj.messageCode()) { case JProtocolHeader.PUBLISH_SERVICE: { Pair<RegisterMeta.ServiceMeta, ?> data = (Pair<RegisterMeta.ServiceMeta, ?>) obj.data(); Object metaObj = data.getSecond(); if (metaObj instanceof List) { List<RegisterMeta> list = (List<RegisterMeta>) metaObj; RegisterMeta[] array = new RegisterMeta[list.size()]; list.toArray(array); registryService.notify(data.getFirst(), NotifyListener.NotifyEvent.CHILD_ADDED, obj.version(), array); } else if (metaObj instanceof RegisterMeta) { registryService.notify(data.getFirst(), NotifyListener.NotifyEvent.CHILD_ADDED, obj.version(), (RegisterMeta) metaObj); } ch.writeAndFlush(new Acknowledge(obj.sequence())) // ?ACK .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); if (logger.isInfoEnabled()) { logger.info("Publish from RegistryServer {}, metadata: {}, version: {}.", data.getFirst(), metaObj, obj.version()); } break; } case JProtocolHeader.PUBLISH_CANCEL_SERVICE: { Pair<RegisterMeta.ServiceMeta, RegisterMeta> data = (Pair<RegisterMeta.ServiceMeta, RegisterMeta>) obj .data(); registryService.notify(data.getFirst(), NotifyListener.NotifyEvent.CHILD_REMOVED, obj.version(), data.getSecond()); ch.writeAndFlush(new Acknowledge(obj.sequence())) // ?ACK .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); if (logger.isInfoEnabled()) { logger.info("Publish cancel from RegistryServer {}, metadata: {}, version: {}.", data.getFirst(), data.getSecond(), obj.version()); } break; } case JProtocolHeader.OFFLINE_NOTICE: RegisterMeta.Address address = (RegisterMeta.Address) obj.data(); logger.info("Offline notice on {}.", address); registryService.offline(address); break; } } else if (msg instanceof Acknowledge) { handleAcknowledge((Acknowledge) msg); } else { if (logger.isWarnEnabled()) { logger.warn("Unexpected message type received: {}, channel: {}.", msg.getClass(), ch); } ReferenceCountUtil.release(msg); } } @Override public void channelActive(ChannelHandlerContext ctx) throws Exception { Channel ch = (channel = ctx.channel()); // ? for (RegisterMeta.ServiceMeta serviceMeta : registryService.subscribeSet()) { // doSubscribe()write if (!attachSubscribeEventOnChannel(serviceMeta, ch)) { continue; } Message msg = new Message(serializerType.value()); msg.messageCode(JProtocolHeader.SUBSCRIBE_SERVICE); msg.data(serviceMeta); ch.writeAndFlush(msg).addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); MessageNonAck msgNonAck = new MessageNonAck(msg, ch); messagesNonAck.put(msgNonAck.id, msgNonAck); } // ??? for (RegisterMeta meta : registryService.registerMetaSet()) { // doRegister()write if (!attachPublishEventOnChannel(meta, ch)) { continue; } Message msg = new Message(serializerType.value()); msg.messageCode(JProtocolHeader.PUBLISH_SERVICE); msg.data(meta); ch.writeAndFlush(msg).addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); MessageNonAck msgNonAck = new MessageNonAck(msg, ch); messagesNonAck.put(msgNonAck.id, msgNonAck); } } @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { Channel ch = ctx.channel(); if (cause instanceof Signal) { logger.error("An I/O signal was caught: {}, force to close channel: {}.", ((Signal) cause).name(), ch); ch.close(); } else if (cause instanceof IOException) { logger.error("An I/O exception was caught: {}, force to close channel: {}.", stackTrace(cause), ch); ch.close(); } else { logger.error("An unexpected exception was caught: {}, channel: {}.", stackTrace(cause), ch); } } } private class AckTimeoutScanner implements Runnable { @SuppressWarnings("all") @Override public void run() { for (;;) { try { for (MessageNonAck m : messagesNonAck.values()) { if (SystemClock.millisClock().now() - m.timestamp > TimeUnit.SECONDS.toMillis(10)) { // if (messagesNonAck.remove(m.id) == null) { continue; } if (m.channel.isActive()) { MessageNonAck msgNonAck = new MessageNonAck(m.msg, m.channel); messagesNonAck.put(msgNonAck.id, msgNonAck); m.channel.writeAndFlush(m.msg) .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); } } } Thread.sleep(300); } catch (Throwable t) { logger.error("An exception was caught while scanning the timeout acknowledges {}.", stackTrace(t)); } } } } { Thread t = new Thread(new AckTimeoutScanner(), "ack.timeout.scanner"); t.setDaemon(true); t.start(); } }