Java tutorial
/* * Copyright (c) 2015 The Jupiter Project * * Licensed under the Apache License, version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.jupiter.registry; import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBuf; import io.netty.channel.*; import io.netty.channel.group.ChannelGroup; import io.netty.channel.group.ChannelMatcher; import io.netty.channel.group.DefaultChannelGroup; import io.netty.channel.socket.SocketChannel; import io.netty.handler.codec.MessageToByteEncoder; import io.netty.handler.codec.ReplayingDecoder; import io.netty.util.Attribute; import io.netty.util.AttributeKey; import io.netty.util.ReferenceCountUtil; import io.netty.util.concurrent.GlobalEventExecutor; import org.jupiter.common.concurrent.collection.ConcurrentSet; import org.jupiter.common.util.*; import org.jupiter.common.util.internal.logging.InternalLogger; import org.jupiter.common.util.internal.logging.InternalLoggerFactory; import org.jupiter.serialization.Serializer; import org.jupiter.serialization.SerializerFactory; import org.jupiter.serialization.SerializerType; import org.jupiter.transport.Acknowledge; import org.jupiter.transport.JConfig; import org.jupiter.transport.JOption; import org.jupiter.transport.JProtocolHeader; import org.jupiter.transport.exception.IoSignals; import org.jupiter.transport.netty.NettyTcpAcceptor; import org.jupiter.transport.netty.TcpChannelProvider; import org.jupiter.transport.netty.handler.AcknowledgeEncoder; import org.jupiter.transport.netty.handler.IdleStateChecker; import org.jupiter.transport.netty.handler.acceptor.AcceptorIdleStateTrigger; import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.List; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import static org.jupiter.common.util.StackTraceUtil.stackTrace; /** * The server of registration center. * * ??, ??. * * provider(client)provider??server, ??provider???, * server???. * * consumer(client)consumer?server, ??consumer??. * * jupiter * org.jupiter.registry * * @author jiachun.fjc */ public final class DefaultRegistryServer extends NettyTcpAcceptor implements RegistryServer { private static final InternalLogger logger = InternalLoggerFactory.getInstance(DefaultRegistryServer.class); private static final AttributeKey<ConcurrentSet<RegisterMeta.ServiceMeta>> S_SUBSCRIBE_KEY = AttributeKey .valueOf("server.subscribed"); private static final AttributeKey<ConcurrentSet<RegisterMeta>> S_PUBLISH_KEY = AttributeKey .valueOf("server.published"); // ? private final RegisterInfoContext registerInfoContext = new RegisterInfoContext(); // private final ChannelGroup subscriberChannels = new DefaultChannelGroup("subscribers", GlobalEventExecutor.INSTANCE); // ack, ???? private final ConcurrentMap<String, MessageNonAck> messagesNonAck = Maps.newConcurrentMap(); // handlers private final AcceptorIdleStateTrigger idleStateTrigger = new AcceptorIdleStateTrigger(); private final MessageHandler handler = new MessageHandler(); private final MessageEncoder encoder = new MessageEncoder(); private final AcknowledgeEncoder ackEncoder = new AcknowledgeEncoder(); // ?/???? private final SerializerType serializerType; { SerializerType expected = SerializerType .parse(SystemPropertyUtil.get("jupiter.registry.default.serializer_type")); serializerType = expected == null ? SerializerType.getDefault() : expected; } public DefaultRegistryServer(int port) { super(port, false); } public DefaultRegistryServer(SocketAddress address) { super(address, false); } public DefaultRegistryServer(int port, int nWorks) { super(port, nWorks, false); } public DefaultRegistryServer(SocketAddress address, int nWorks) { super(address, nWorks, false); } @Override protected void init() { super.init(); // parent options JConfig parent = configGroup().parent(); parent.setOption(JOption.SO_BACKLOG, 1024); parent.setOption(JOption.SO_REUSEADDR, true); // child options JConfig child = configGroup().child(); child.setOption(JOption.SO_REUSEADDR, true); } @Override public ChannelFuture bind(SocketAddress localAddress) { ServerBootstrap boot = bootstrap(); boot.channelFactory(TcpChannelProvider.NIO_ACCEPTOR).childHandler(new ChannelInitializer<SocketChannel>() { @Override protected void initChannel(SocketChannel ch) throws Exception { ch.pipeline().addLast(new IdleStateChecker(timer, JConstants.READER_IDLE_TIME_SECONDS, 0, 0), idleStateTrigger, new MessageDecoder(), encoder, ackEncoder, handler); } }); setOptions(); return boot.bind(localAddress); } @Override public List<String> listPublisherHosts() { List<RegisterMeta.Address> fromList = registerInfoContext.listPublisherHosts(); return Lists.transform(fromList, new Function<RegisterMeta.Address, String>() { @Override public String apply(RegisterMeta.Address input) { return input.getHost(); } }); } @Override public List<String> listSubscriberAddresses() { List<String> hosts = Lists.newArrayList(); for (Channel ch : subscriberChannels) { SocketAddress address = ch.remoteAddress(); if (address instanceof InetSocketAddress) { String host = ((InetSocketAddress) address).getAddress().getHostAddress(); int port = ((InetSocketAddress) address).getPort(); hosts.add(new RegisterMeta.Address(host, port).toString()); } } return hosts; } @Override public List<String> listAddressesByService(String group, String serviceProviderName, String version) { RegisterMeta.ServiceMeta serviceMeta = new RegisterMeta.ServiceMeta(group, serviceProviderName, version); List<RegisterMeta.Address> fromList = registerInfoContext.listAddressesByService(serviceMeta); return Lists.transform(fromList, new Function<RegisterMeta.Address, String>() { @Override public String apply(RegisterMeta.Address input) { return input.toString(); } }); } @Override public List<String> listServicesByAddress(String host, int port) { RegisterMeta.Address address = new RegisterMeta.Address(host, port); List<RegisterMeta.ServiceMeta> fromList = registerInfoContext.listServicesByAddress(address); return Lists.transform(fromList, new Function<RegisterMeta.ServiceMeta, String>() { @Override public String apply(RegisterMeta.ServiceMeta input) { return input.toString(); } }); } @Override public void startRegistryServer() { try { start(); } catch (InterruptedException e) { ExceptionUtil.throwException(e); } } // ?, ??? private void handlePublish(RegisterMeta meta, Channel channel) { logger.info("Publish {} on channel{}.", meta, channel); attachPublishEventOnChannel(meta, channel); final RegisterMeta.ServiceMeta serviceMeta = meta.getServiceMeta(); ConfigWithVersion<ConcurrentMap<RegisterMeta.Address, RegisterMeta>> config = registerInfoContext .getRegisterMeta(serviceMeta); synchronized (registerInfoContext.publishLock(config)) { // putIfAbsentconfig.newVersion()???, ? if (config.getConfig().putIfAbsent(meta.getAddress(), meta) == null) { registerInfoContext.getServiceMeta(meta.getAddress()).add(serviceMeta); final Message msg = new Message(serializerType.value()); msg.messageCode(JProtocolHeader.PUBLISH_SERVICE); msg.version(config.newVersion()); // ?+1 msg.data(Pair.of(serviceMeta, meta)); subscriberChannels.writeAndFlush(msg, new ChannelMatcher() { @Override public boolean matches(Channel channel) { boolean doSend = isChannelSubscribeOnServiceMeta(serviceMeta, channel); if (doSend) { MessageNonAck msgNonAck = new MessageNonAck(serviceMeta, msg, channel); // ack??key(??handleAcknowledge), ??? messagesNonAck.put(msgNonAck.id, msgNonAck); } return doSend; } }); } } } // ?, ??? private void handlePublishCancel(RegisterMeta meta, Channel channel) { logger.info("Cancel publish {} on channel{}.", meta, channel); attachPublishCancelEventOnChannel(meta, channel); final RegisterMeta.ServiceMeta serviceMeta = meta.getServiceMeta(); ConfigWithVersion<ConcurrentMap<RegisterMeta.Address, RegisterMeta>> config = registerInfoContext .getRegisterMeta(serviceMeta); if (config.getConfig().isEmpty()) { return; } synchronized (registerInfoContext.publishLock(config)) { // putIfAbsentconfig.newVersion()???, ? RegisterMeta.Address address = meta.getAddress(); RegisterMeta data = config.getConfig().remove(address); if (data != null) { registerInfoContext.getServiceMeta(address).remove(serviceMeta); final Message msg = new Message(serializerType.value()); msg.messageCode(JProtocolHeader.PUBLISH_CANCEL_SERVICE); msg.version(config.newVersion()); // ?+1 msg.data(Pair.of(serviceMeta, data)); subscriberChannels.writeAndFlush(msg, new ChannelMatcher() { @Override public boolean matches(Channel channel) { boolean doSend = isChannelSubscribeOnServiceMeta(serviceMeta, channel); if (doSend) { MessageNonAck msgNonAck = new MessageNonAck(serviceMeta, msg, channel); // ack??key(??handleAcknowledge), ??? messagesNonAck.put(msgNonAck.id, msgNonAck); } return doSend; } }); } } } // ? private void handleSubscribe(RegisterMeta.ServiceMeta serviceMeta, Channel channel) { logger.info("Subscribe {} on channel{}.", serviceMeta, channel); attachSubscribeEventOnChannel(serviceMeta, channel); subscriberChannels.add(channel); ConfigWithVersion<ConcurrentMap<RegisterMeta.Address, RegisterMeta>> config = registerInfoContext .getRegisterMeta(serviceMeta); if (config.getConfig().isEmpty()) { return; } final Message msg = new Message(serializerType.value()); msg.messageCode(JProtocolHeader.PUBLISH_SERVICE); msg.version(config.getVersion()); // ? List<RegisterMeta> registerMetaList = Lists.newArrayList(config.getConfig().values()); // ????meta?? msg.data(Pair.of(serviceMeta, registerMetaList)); MessageNonAck msgNonAck = new MessageNonAck(serviceMeta, msg, channel); // ack??key(??handleAcknowledge), ??? messagesNonAck.put(msgNonAck.id, msgNonAck); channel.writeAndFlush(msg); } // ?ack private void handleAcknowledge(Acknowledge ack, Channel channel) { messagesNonAck.remove(key(ack.sequence(), channel)); } // ?Provider private void handleOfflineNotice(RegisterMeta.Address address) { logger.info("OfflineNotice on {}.", address); Message msg = new Message(serializerType.value()); msg.messageCode(JProtocolHeader.OFFLINE_NOTICE); msg.data(address); subscriberChannels.writeAndFlush(msg); } private static String key(long sequence, Channel channel) { return String.valueOf(sequence) + '-' + channel.id().asShortText(); } // channel(??) private static boolean attachPublishEventOnChannel(RegisterMeta meta, Channel channel) { Attribute<ConcurrentSet<RegisterMeta>> attr = channel.attr(S_PUBLISH_KEY); ConcurrentSet<RegisterMeta> registerMetaSet = attr.get(); if (registerMetaSet == null) { ConcurrentSet<RegisterMeta> newRegisterMetaSet = new ConcurrentSet<>(); registerMetaSet = attr.setIfAbsent(newRegisterMetaSet); if (registerMetaSet == null) { registerMetaSet = newRegisterMetaSet; } } return registerMetaSet.add(meta); } // ?channel(??) private static boolean attachPublishCancelEventOnChannel(RegisterMeta meta, Channel channel) { Attribute<ConcurrentSet<RegisterMeta>> attr = channel.attr(S_PUBLISH_KEY); ConcurrentSet<RegisterMeta> registerMetaSet = attr.get(); if (registerMetaSet == null) { ConcurrentSet<RegisterMeta> newRegisterMetaSet = new ConcurrentSet<>(); registerMetaSet = attr.setIfAbsent(newRegisterMetaSet); if (registerMetaSet == null) { registerMetaSet = newRegisterMetaSet; } } return registerMetaSet.remove(meta); } // channel(?) private static boolean attachSubscribeEventOnChannel(RegisterMeta.ServiceMeta serviceMeta, Channel channel) { Attribute<ConcurrentSet<RegisterMeta.ServiceMeta>> attr = channel.attr(S_SUBSCRIBE_KEY); ConcurrentSet<RegisterMeta.ServiceMeta> serviceMetaSet = attr.get(); if (serviceMetaSet == null) { ConcurrentSet<RegisterMeta.ServiceMeta> newServiceMetaSet = new ConcurrentSet<>(); serviceMetaSet = attr.setIfAbsent(newServiceMetaSet); if (serviceMetaSet == null) { serviceMetaSet = newServiceMetaSet; } } return serviceMetaSet.add(serviceMeta); } // channel(??) private static boolean isChannelSubscribeOnServiceMeta(RegisterMeta.ServiceMeta serviceMeta, Channel channel) { ConcurrentSet<RegisterMeta.ServiceMeta> serviceMetaSet = channel.attr(S_SUBSCRIBE_KEY).get(); return serviceMetaSet != null && serviceMetaSet.contains(serviceMeta); } /** * ACK, ???? */ static class MessageNonAck { private final String id; private final RegisterMeta.ServiceMeta serviceMeta; private final Message msg; private final Channel channel; private final long version; private final long timestamp = SystemClock.millisClock().now(); public MessageNonAck(RegisterMeta.ServiceMeta serviceMeta, Message msg, Channel channel) { this.serviceMeta = serviceMeta; this.msg = msg; this.channel = channel; this.version = msg.version(); id = key(msg.sequence(), channel); } } /** * ************************************************************************************************** * Protocol * ? * 2 1 1 8 4 * * * MAGIC Sign Status Invoke Id Body Length Body Content * * * * ?16 * = 2 // magic = (short) 0xbabe * + 1 // ??, ?4???, ?4??? * + 1 // * + 8 // ? id, long * + 4 // ? body , int */ static class MessageDecoder extends ReplayingDecoder<MessageDecoder.State> { public MessageDecoder() { super(State.HEADER_MAGIC); } // ?? private final JProtocolHeader header = new JProtocolHeader(); @Override protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception { switch (state()) { case HEADER_MAGIC: checkMagic(in.readShort()); // MAGIC checkpoint(State.HEADER_SIGN); case HEADER_SIGN: header.sign(in.readByte()); // ?? checkpoint(State.HEADER_STATUS); case HEADER_STATUS: in.readByte(); // no-op checkpoint(State.HEADER_ID); case HEADER_ID: header.id(in.readLong()); // ?id checkpoint(State.HEADER_BODY_LENGTH); case HEADER_BODY_LENGTH: header.bodyLength(in.readInt()); // ? checkpoint(State.BODY); case BODY: byte s_code = header.serializerCode(); switch (header.messageCode()) { case JProtocolHeader.HEARTBEAT: break; case JProtocolHeader.PUBLISH_SERVICE: case JProtocolHeader.PUBLISH_CANCEL_SERVICE: case JProtocolHeader.SUBSCRIBE_SERVICE: case JProtocolHeader.OFFLINE_NOTICE: { byte[] bytes = new byte[header.bodyLength()]; in.readBytes(bytes); Serializer serializer = SerializerFactory.getSerializer(s_code); Message msg = serializer.readObject(bytes, Message.class); msg.messageCode(header.messageCode()); out.add(msg); break; } case JProtocolHeader.ACK: out.add(new Acknowledge(header.id())); break; default: throw IoSignals.ILLEGAL_SIGN; } checkpoint(State.HEADER_MAGIC); } } private static void checkMagic(short magic) throws Signal { if (magic != JProtocolHeader.MAGIC) { throw IoSignals.ILLEGAL_MAGIC; } } enum State { HEADER_MAGIC, HEADER_SIGN, HEADER_STATUS, HEADER_ID, HEADER_BODY_LENGTH, BODY } } /** * ************************************************************************************************** * Protocol * ? * 2 1 1 8 4 * * * MAGIC Sign Status Invoke Id Body Length Body Content * * * * ?16 * = 2 // magic = (short) 0xbabe * + 1 // ??, ?4???, ?4??? * + 1 // * + 8 // ? id, long * + 4 // ? body , int */ @ChannelHandler.Sharable static class MessageEncoder extends MessageToByteEncoder<Message> { @Override protected void encode(ChannelHandlerContext ctx, Message msg, ByteBuf out) throws Exception { byte s_code = msg.serializerCode(); byte sign = JProtocolHeader.toSign(s_code, msg.messageCode()); Serializer serializer = SerializerFactory.getSerializer(s_code); byte[] bytes = serializer.writeObject(msg); out.writeShort(JProtocolHeader.MAGIC).writeByte(sign).writeByte(0).writeLong(0).writeInt(bytes.length) .writeBytes(bytes); } } @ChannelHandler.Sharable class MessageHandler extends ChannelInboundHandlerAdapter { @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { Channel ch = ctx.channel(); if (msg instanceof Message) { Message obj = (Message) msg; switch (obj.messageCode()) { case JProtocolHeader.PUBLISH_SERVICE: case JProtocolHeader.PUBLISH_CANCEL_SERVICE: RegisterMeta meta = (RegisterMeta) obj.data(); if (Strings.isNullOrEmpty(meta.getHost())) { SocketAddress address = ch.remoteAddress(); if (address instanceof InetSocketAddress) { meta.setHost(((InetSocketAddress) address).getAddress().getHostAddress()); } else { logger.warn("Could not get remote host: {}, info: {}", ch, meta); return; } } if (obj.messageCode() == JProtocolHeader.PUBLISH_SERVICE) { handlePublish(meta, ch); } else if (obj.messageCode() == JProtocolHeader.PUBLISH_CANCEL_SERVICE) { handlePublishCancel(meta, ch); } ch.writeAndFlush(new Acknowledge(obj.sequence())) // ?ACK .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); break; case JProtocolHeader.SUBSCRIBE_SERVICE: handleSubscribe((RegisterMeta.ServiceMeta) obj.data(), ch); ch.writeAndFlush(new Acknowledge(obj.sequence())) // ?ACK .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); break; case JProtocolHeader.OFFLINE_NOTICE: handleOfflineNotice((RegisterMeta.Address) obj.data()); break; } } else if (msg instanceof Acknowledge) { handleAcknowledge((Acknowledge) msg, ch); } else { if (logger.isWarnEnabled()) { logger.warn("Unexpected message type received: {}, channel: {}.", msg.getClass(), ch); } ReferenceCountUtil.release(msg); } } @Override public void channelInactive(ChannelHandlerContext ctx) throws Exception { Channel ch = ctx.channel(); // ???? ConcurrentSet<RegisterMeta> registerMetaSet = ch.attr(S_PUBLISH_KEY).get(); if (registerMetaSet == null || registerMetaSet.isEmpty()) { return; } RegisterMeta.Address address = null; for (RegisterMeta meta : registerMetaSet) { if (address == null) { address = meta.getAddress(); } handlePublishCancel(meta, ch); } if (address != null) { // handleOfflineNotice(address); } } @Override public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception { Channel ch = ctx.channel(); ChannelConfig config = ch.config(); // ?: ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK // ?: ChannelOption.WRITE_BUFFER_LOW_WATER_MARK if (!ch.isWritable()) { // ?channel(OutboundBuffer)?WRITE_BUFFER_HIGH_WATER_MARK if (logger.isWarnEnabled()) { logger.warn( "{} is not writable, high water mask: {}, the number of flushed entries that are not written yet: {}.", ch, config.getWriteBufferHighWaterMark(), ch.unsafe().outboundBuffer().size()); } config.setAutoRead(false); } else { // ??OutboundBuffer?WRITE_BUFFER_LOW_WATER_MARK if (logger.isWarnEnabled()) { logger.warn( "{} is writable(rehabilitate), low water mask: {}, the number of flushed entries that are not written yet: {}.", ch, config.getWriteBufferLowWaterMark(), ch.unsafe().outboundBuffer().size()); } config.setAutoRead(true); } } @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { Channel ch = ctx.channel(); if (cause instanceof Signal) { logger.error("An I/O signal was caught: {}, force to close channel: {}.", ((Signal) cause).name(), ch); ch.close(); } else if (cause instanceof IOException) { logger.error("An I/O exception was caught: {}, force to close channel: {}.", stackTrace(cause), cause); ch.close(); } else { logger.error("An unexpected exception was caught: {}, channel: {}.", stackTrace(cause), ch); } } } @SuppressWarnings("all") private class AckTimeoutScanner implements Runnable { @Override public void run() { for (;;) { try { for (MessageNonAck m : messagesNonAck.values()) { if (SystemClock.millisClock().now() - m.timestamp > TimeUnit.SECONDS.toMillis(10)) { // if (messagesNonAck.remove(m.id) == null) { continue; } if (registerInfoContext.getRegisterMeta(m.serviceMeta).getVersion() > m.version) { // ???? continue; } if (m.channel.isActive()) { MessageNonAck msgNonAck = new MessageNonAck(m.serviceMeta, m.msg, m.channel); messagesNonAck.put(msgNonAck.id, msgNonAck); m.channel.writeAndFlush(m.msg) .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); } } } Thread.sleep(300); } catch (Throwable t) { logger.error("An exception was caught while scanning the timeout acknowledges {}.", stackTrace(t)); } } } } { Thread t = new Thread(new AckTimeoutScanner(), "ack.timeout.scanner"); t.setDaemon(true); t.start(); } }