Java tutorial
/* * Copyright (c) 2015 The Jupiter Project * * Licensed under the Apache License, version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.jupiter.registry; import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBuf; import io.netty.channel.*; import io.netty.channel.group.ChannelGroup; import io.netty.channel.group.ChannelMatcher; import io.netty.channel.group.DefaultChannelGroup; import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.handler.codec.MessageToByteEncoder; import io.netty.handler.codec.ReplayingDecoder; import io.netty.util.Attribute; import io.netty.util.AttributeKey; import io.netty.util.ReferenceCountUtil; import io.netty.util.concurrent.GlobalEventExecutor; import org.jupiter.common.concurrent.ConcurrentSet; import org.jupiter.common.util.*; import org.jupiter.common.util.internal.logging.InternalLogger; import org.jupiter.common.util.internal.logging.InternalLoggerFactory; import org.jupiter.rpc.channel.JChannel; import org.jupiter.transport.Acknowledge; import org.jupiter.transport.JConfig; import org.jupiter.transport.JOption; import org.jupiter.transport.JProtocolHeader; import org.jupiter.transport.exception.IoSignals; import org.jupiter.transport.netty.NettyTcpAcceptor; import org.jupiter.transport.netty.channel.NettyChannel; import org.jupiter.transport.netty.handler.AcknowledgeEncoder; import org.jupiter.transport.netty.handler.IdleStateChecker; import org.jupiter.transport.netty.handler.acceptor.AcceptorIdleStateTrigger; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.List; import java.util.concurrent.ConcurrentMap; import static java.util.concurrent.TimeUnit.SECONDS; import static org.jupiter.common.util.JConstants.READER_IDLE_TIME_SECONDS; import static org.jupiter.common.util.StackTraceUtil.stackTrace; import static org.jupiter.registry.RegisterMeta.Address; import static org.jupiter.registry.RegisterMeta.ServiceMeta; import static org.jupiter.serialization.SerializerHolder.serializerImpl; import static org.jupiter.transport.JProtocolHeader.*; import static org.jupiter.transport.exception.IoSignals.ILLEGAL_MAGIC; import static org.jupiter.transport.exception.IoSignals.ILLEGAL_SIGN; /** * The server of registration center. * * ??, ??. * * provider(client)provider??server, ??provider???, * server???. * * consumer(client)consumer?server, ??consumer??. * * jupiter * org.jupiter.registry * * @author jiachun.fjc */ public class ConfigServer extends NettyTcpAcceptor implements RegistryMonitor { private static final InternalLogger logger = InternalLoggerFactory.getInstance(ConfigServer.class); private static final AttributeKey<ConcurrentSet<ServiceMeta>> S_SUBSCRIBE_KEY = AttributeKey .valueOf("server.subscribed"); private static final AttributeKey<ConcurrentSet<RegisterMeta>> S_PUBLISH_KEY = AttributeKey .valueOf("server.published"); // ? private final RegisterInfoContext registerInfoContext = new RegisterInfoContext(); // private final ChannelGroup subscriberChannels = new DefaultChannelGroup("subscribers", GlobalEventExecutor.INSTANCE); // ack, ???? private final ConcurrentMap<String, MessageNonAck> messagesNonAck = Maps.newConcurrentHashMap(); // handlers private final AcceptorIdleStateTrigger idleStateTrigger = new AcceptorIdleStateTrigger(); private final MessageHandler handler = new MessageHandler(); private final MessageEncoder encoder = new MessageEncoder(); private final AcknowledgeEncoder ackEncoder = new AcknowledgeEncoder(); public ConfigServer(int port) { super(port, false); } public ConfigServer(SocketAddress address) { super(address, false); } public ConfigServer(int port, int nWorks) { super(port, nWorks, false); } public ConfigServer(SocketAddress address, int nWorks) { super(address, nWorks, false); } @Override protected void init() { super.init(); // parent options JConfig parent = configGroup().parent(); parent.setOption(JOption.SO_BACKLOG, 1024); parent.setOption(JOption.SO_REUSEADDR, true); // child options JConfig child = configGroup().child(); child.setOption(JOption.SO_REUSEADDR, true); } @Override public ChannelFuture bind(SocketAddress localAddress) { ServerBootstrap boot = bootstrap(); boot.channel(NioServerSocketChannel.class).childHandler(new ChannelInitializer<SocketChannel>() { @Override protected void initChannel(SocketChannel ch) throws Exception { ch.pipeline().addLast(new IdleStateChecker(timer, READER_IDLE_TIME_SECONDS, 0, 0), idleStateTrigger, new MessageDecoder(), encoder, ackEncoder, handler); } }); setOptions(); return boot.bind(localAddress); } @Override public List<String> listPublisherHosts() { List<Address> fromList = registerInfoContext.listPublisherHosts(); return Lists.transform(fromList, new Function<Address, String>() { @Override public String apply(Address input) { return input.getHost(); } }); } @Override public List<String> listSubscriberAddresses() { List<String> hosts = Lists.newArrayList(); for (Channel ch : subscriberChannels) { SocketAddress address = ch.remoteAddress(); if (address instanceof InetSocketAddress) { String host = ((InetSocketAddress) address).getAddress().getHostAddress(); int port = ((InetSocketAddress) address).getPort(); hosts.add(new Address(host, port).toString()); } } return hosts; } @Override public List<String> listAddressesByService(String group, String version, String serviceProviderName) { ServiceMeta serviceMeta = new ServiceMeta(group, version, serviceProviderName); List<Address> fromList = registerInfoContext.listAddressesByService(serviceMeta); return Lists.transform(fromList, new Function<Address, String>() { @Override public String apply(Address input) { return input.toString(); } }); } @Override public List<String> listServicesByAddress(String host, int port) { Address address = new Address(host, port); List<ServiceMeta> fromList = registerInfoContext.listServicesByAddress(address); return Lists.transform(fromList, new Function<ServiceMeta, String>() { @Override public String apply(ServiceMeta input) { return input.toString(); } }); } // ?, ??? private void handlePublish(RegisterMeta meta, Channel channel) { logger.info("Publish {} on channel{}.", meta, channel); attachPublishEventOnChannel(meta, channel); final ServiceMeta serviceMeta = meta.getServiceMeta(); ConfigWithVersion<ConcurrentMap<Address, RegisterMeta>> config = registerInfoContext .getRegisterMeta(serviceMeta); synchronized (registerInfoContext.publishLock(config)) { // putIfAbsentconfig.newVersion()???, ? if (config.getConfig().putIfAbsent(meta.getAddress(), meta) == null) { registerInfoContext.getServiceMeta(meta.getAddress()).add(serviceMeta); final Message msg = new Message(); msg.sign(PUBLISH_SERVICE); msg.setVersion(config.newVersion()); // ?+1 List<RegisterMeta> registerMetaList = Lists.newArrayList(config.getConfig().values()); // ????meta?? msg.data(new Pair<>(serviceMeta, registerMetaList)); subscriberChannels.writeAndFlush(msg, new ChannelMatcher() { @Override public boolean matches(Channel channel) { boolean doSend = isChannelSubscribeOnServiceMeta(serviceMeta, channel); if (doSend) { MessageNonAck msgNonAck = new MessageNonAck(serviceMeta, msg, channel); // ack??key(??handleAcknowledge), ??? messagesNonAck.put(msgNonAck.id, msgNonAck); } return doSend; } }); } } } // ?, ??? private void handlePublishCancel(RegisterMeta meta, Channel channel) { logger.info("Cancel publish {} on channel{}.", meta, channel); attachPublishCancelEventOnChannel(meta, channel); final ServiceMeta serviceMeta = meta.getServiceMeta(); ConfigWithVersion<ConcurrentMap<Address, RegisterMeta>> config = registerInfoContext .getRegisterMeta(serviceMeta); if (config.getConfig().isEmpty()) { return; } synchronized (registerInfoContext.publishLock(config)) { // putIfAbsentconfig.newVersion()???, ? Address address = meta.getAddress(); RegisterMeta data = config.getConfig().remove(address); if (data != null) { registerInfoContext.getServiceMeta(address).remove(serviceMeta); final Message msg = new Message(); msg.sign(PUBLISH_SERVICE); msg.setVersion(config.newVersion()); // ?+1 List<RegisterMeta> registerMetaList = Lists.newArrayList(config.getConfig().values()); // ????meta?? msg.data(new Pair<>(serviceMeta, registerMetaList)); subscriberChannels.writeAndFlush(msg, new ChannelMatcher() { @Override public boolean matches(Channel channel) { boolean doSend = isChannelSubscribeOnServiceMeta(serviceMeta, channel); if (doSend) { MessageNonAck msgNonAck = new MessageNonAck(serviceMeta, msg, channel); // ack??key(??handleAcknowledge), ??? messagesNonAck.put(msgNonAck.id, msgNonAck); } return doSend; } }); } } } // ? private void handleSubscribe(ServiceMeta serviceMeta, Channel channel) { logger.info("Subscribe {} on channel{}.", serviceMeta, channel); attachSubscribeEventOnChannel(serviceMeta, channel); subscriberChannels.add(channel); ConfigWithVersion<ConcurrentMap<Address, RegisterMeta>> config = registerInfoContext .getRegisterMeta(serviceMeta); if (config.getConfig().isEmpty()) { return; } final Message msg = new Message(); msg.sign(PUBLISH_SERVICE); msg.setVersion(config.getVersion()); // ? List<RegisterMeta> registerMetaList = Lists.newArrayList(config.getConfig().values()); // ????meta?? msg.data(new Pair<>(serviceMeta, registerMetaList)); MessageNonAck msgNonAck = new MessageNonAck(serviceMeta, msg, channel); // ack??key(??handleAcknowledge), ??? messagesNonAck.put(msgNonAck.id, msgNonAck); channel.writeAndFlush(msg); } // ?ack private void handleAcknowledge(Acknowledge ack, Channel channel) { messagesNonAck.remove(key(ack.sequence(), channel)); } // ?Provider private void handleOfflineNotice(Address address) { logger.info("OfflineNotice on {}.", address); Message msg = new Message(); msg.sign(OFFLINE_NOTICE); msg.data(address); subscriberChannels.writeAndFlush(msg); } private static String key(long sequence, Channel channel) { return String.valueOf(sequence) + '-' + channel.id().asShortText(); } // channel(??) private static boolean attachPublishEventOnChannel(RegisterMeta meta, Channel channel) { Attribute<ConcurrentSet<RegisterMeta>> attr = channel.attr(S_PUBLISH_KEY); ConcurrentSet<RegisterMeta> registerMetaSet = attr.get(); if (registerMetaSet == null) { ConcurrentSet<RegisterMeta> newRegisterMetaSet = new ConcurrentSet<>(); registerMetaSet = attr.setIfAbsent(newRegisterMetaSet); if (registerMetaSet == null) { registerMetaSet = newRegisterMetaSet; } } return registerMetaSet.add(meta); } // ?channel(??) private static boolean attachPublishCancelEventOnChannel(RegisterMeta meta, Channel channel) { Attribute<ConcurrentSet<RegisterMeta>> attr = channel.attr(S_PUBLISH_KEY); ConcurrentSet<RegisterMeta> registerMetaSet = attr.get(); if (registerMetaSet == null) { ConcurrentSet<RegisterMeta> newRegisterMetaSet = new ConcurrentSet<>(); registerMetaSet = attr.setIfAbsent(newRegisterMetaSet); if (registerMetaSet == null) { registerMetaSet = newRegisterMetaSet; } } return registerMetaSet.remove(meta); } // channel(?) private static boolean attachSubscribeEventOnChannel(ServiceMeta serviceMeta, Channel channel) { Attribute<ConcurrentSet<ServiceMeta>> attr = channel.attr(S_SUBSCRIBE_KEY); ConcurrentSet<ServiceMeta> serviceMetaSet = attr.get(); if (serviceMetaSet == null) { ConcurrentSet<ServiceMeta> newServiceMetaSet = new ConcurrentSet<>(); serviceMetaSet = attr.setIfAbsent(newServiceMetaSet); if (serviceMetaSet == null) { serviceMetaSet = newServiceMetaSet; } } return serviceMetaSet.add(serviceMeta); } // channel(??) private static boolean isChannelSubscribeOnServiceMeta(ServiceMeta serviceMeta, Channel channel) { ConcurrentSet<ServiceMeta> serviceMetaSet = channel.attr(S_SUBSCRIBE_KEY).get(); return serviceMetaSet != null && serviceMetaSet.contains(serviceMeta); } /** * ACK, ???? */ static class MessageNonAck { private final String id; private final ServiceMeta serviceMeta; private final Message msg; private final Channel channel; private final long version; private final long timestamp = SystemClock.millisClock().now(); public MessageNonAck(ServiceMeta serviceMeta, Message msg, Channel channel) { this.serviceMeta = serviceMeta; this.msg = msg; this.channel = channel; this.version = msg.getVersion(); id = key(msg.sequence(), channel); } } /** * ************************************************************************************************** * Protocol * ? * 2 1 1 8 4 * * * MAGIC Sign Status Invoke Id Body Length Body Content * * * * ?16 * = 2 // MAGIC = (short) 0xbabe * + 1 // ??, ?? * + 1 // * + 8 // ? id long * + 4 // ?body, int */ static class MessageDecoder extends ReplayingDecoder<MessageDecoder.State> { public MessageDecoder() { super(State.HEADER_MAGIC); } // ?? private final JProtocolHeader header = new JProtocolHeader(); @Override protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception { switch (state()) { case HEADER_MAGIC: checkMagic(in.readShort()); // MAGIC checkpoint(State.HEADER_SIGN); case HEADER_SIGN: header.sign(in.readByte()); // ?? checkpoint(State.HEADER_STATUS); case HEADER_STATUS: in.readByte(); // no-op checkpoint(State.HEADER_ID); case HEADER_ID: header.id(in.readLong()); // ?id checkpoint(State.HEADER_BODY_LENGTH); case HEADER_BODY_LENGTH: header.bodyLength(in.readInt()); // ? checkpoint(State.BODY); case BODY: switch (header.sign()) { case HEARTBEAT: break; case PUBLISH_SERVICE: case PUBLISH_CANCEL_SERVICE: case SUBSCRIBE_SERVICE: case OFFLINE_NOTICE: { byte[] bytes = new byte[header.bodyLength()]; in.readBytes(bytes); Message msg = serializerImpl().readObject(bytes, Message.class); msg.sign(header.sign()); out.add(msg); break; } case ACK: { byte[] bytes = new byte[header.bodyLength()]; in.readBytes(bytes); Acknowledge ack = serializerImpl().readObject(bytes, Acknowledge.class); out.add(ack); break; } default: throw ILLEGAL_SIGN; } checkpoint(State.HEADER_MAGIC); } } private static void checkMagic(short magic) throws Signal { if (MAGIC != magic) { throw ILLEGAL_MAGIC; } } enum State { HEADER_MAGIC, HEADER_SIGN, HEADER_STATUS, HEADER_ID, HEADER_BODY_LENGTH, BODY } } /** * ************************************************************************************************** * Protocol * ? * 2 1 1 8 4 * * * MAGIC Sign Status Invoke Id Body Length Body Content * * * * ?16 * = 2 // MAGIC = (short) 0xbabe * + 1 // ??, ?? * + 1 // * + 8 // ? id long * + 4 // ?body, int */ @ChannelHandler.Sharable static class MessageEncoder extends MessageToByteEncoder<Message> { @Override protected void encode(ChannelHandlerContext ctx, Message msg, ByteBuf out) throws Exception { byte[] bytes = serializerImpl().writeObject(msg); out.writeShort(MAGIC).writeByte(msg.sign()).writeByte(0).writeLong(0).writeInt(bytes.length) .writeBytes(bytes); } } @ChannelHandler.Sharable class MessageHandler extends ChannelInboundHandlerAdapter { @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { Channel channel = ctx.channel(); if (msg instanceof Message) { Message obj = (Message) msg; switch (obj.sign()) { case PUBLISH_SERVICE: case PUBLISH_CANCEL_SERVICE: RegisterMeta meta = (RegisterMeta) obj.data(); if (Strings.isNullOrEmpty(meta.getHost())) { SocketAddress address = channel.remoteAddress(); if (address instanceof InetSocketAddress) { meta.setHost(((InetSocketAddress) address).getAddress().getHostAddress()); } else { logger.warn("Could not get remote host: {}, info: {}", channel, meta); return; } } if (obj.sign() == PUBLISH_SERVICE) { handlePublish(meta, channel); } else if (obj.sign() == PUBLISH_CANCEL_SERVICE) { handlePublishCancel(meta, channel); } channel.writeAndFlush(new Acknowledge(obj.sequence())) // ?ACK .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); break; case SUBSCRIBE_SERVICE: handleSubscribe((ServiceMeta) obj.data(), channel); channel.writeAndFlush(new Acknowledge(obj.sequence())) // ?ACK .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); break; case OFFLINE_NOTICE: handleOfflineNotice((Address) obj.data()); break; } } else if (msg instanceof Acknowledge) { handleAcknowledge((Acknowledge) msg, channel); } else { logger.warn("Unexpected msg type received:{}.", msg.getClass()); ReferenceCountUtil.release(msg); } } @Override public void channelInactive(ChannelHandlerContext ctx) throws Exception { Channel channel = ctx.channel(); // ???? ConcurrentSet<RegisterMeta> registerMetaSet = channel.attr(S_PUBLISH_KEY).get(); if (registerMetaSet == null) { return; } Address address = null; for (RegisterMeta meta : registerMetaSet) { if (address == null) { address = meta.getAddress(); } handlePublishCancel(meta, channel); } // handleOfflineNotice(address); } @Override public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception { Channel ch = ctx.channel(); // ?: ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK // ?: ChannelOption.WRITE_BUFFER_LOW_WATER_MARK if (!ch.isWritable()) { // ?channel(OutboundBuffer)?WRITE_BUFFER_HIGH_WATER_MARK logger.warn( "{} is not writable, high water mask: {}, the number of flushed entries that are not written yet: {}.", ch, ch.config().getWriteBufferHighWaterMark(), ch.unsafe().outboundBuffer().size()); ch.config().setAutoRead(false); } else { // ??OutboundBuffer?WRITE_BUFFER_LOW_WATER_MARK logger.warn( "{} is writable(rehabilitate), low water mask: {}, the number of flushed entries that are not written yet: {}.", ch, ch.config().getWriteBufferLowWaterMark(), ch.unsafe().outboundBuffer().size()); ch.config().setAutoRead(true); } } @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { JChannel jChannel = NettyChannel.attachChannel(ctx.channel()); if (cause instanceof Signal) { IoSignals.handleSignal((Signal) cause, jChannel); } else { logger.error("An exception has been caught {}, on {}.", stackTrace(cause), jChannel); } } } private class AckTimeoutScanner implements Runnable { @SuppressWarnings("InfiniteLoopStatement") @Override public void run() { for (;;) { try { for (MessageNonAck m : messagesNonAck.values()) { if (SystemClock.millisClock().now() - m.timestamp > SECONDS.toMillis(10)) { // if (messagesNonAck.remove(m.id) == null) { continue; } if (registerInfoContext.getRegisterMeta(m.serviceMeta).getVersion() > m.version) { // ???? continue; } if (m.channel.isActive()) { MessageNonAck msgNonAck = new MessageNonAck(m.serviceMeta, m.msg, m.channel); messagesNonAck.put(msgNonAck.id, msgNonAck); m.channel.writeAndFlush(m.msg) .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); } } } Thread.sleep(300); } catch (Throwable t) { logger.error("An exception has been caught while scanning the timeout acknowledges {}.", t); } } } } { Thread t = new Thread(new AckTimeoutScanner(), "ack.timeout.scanner"); t.setDaemon(true); t.start(); } }