org.jupiter.registry.ConfigServer.java Source code

Java tutorial

Introduction

Here is the source code for org.jupiter.registry.ConfigServer.java

Source

/*
 * Copyright (c) 2015 The Jupiter Project
 *
 * Licensed under the Apache License, version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.jupiter.registry;

import io.netty.bootstrap.ServerBootstrap;
import io.netty.buffer.ByteBuf;
import io.netty.channel.*;
import io.netty.channel.group.ChannelGroup;
import io.netty.channel.group.ChannelMatcher;
import io.netty.channel.group.DefaultChannelGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.MessageToByteEncoder;
import io.netty.handler.codec.ReplayingDecoder;
import io.netty.util.Attribute;
import io.netty.util.AttributeKey;
import io.netty.util.ReferenceCountUtil;
import io.netty.util.concurrent.GlobalEventExecutor;
import org.jupiter.common.concurrent.ConcurrentSet;
import org.jupiter.common.util.*;
import org.jupiter.common.util.internal.logging.InternalLogger;
import org.jupiter.common.util.internal.logging.InternalLoggerFactory;
import org.jupiter.rpc.channel.JChannel;
import org.jupiter.transport.Acknowledge;
import org.jupiter.transport.JConfig;
import org.jupiter.transport.JOption;
import org.jupiter.transport.JProtocolHeader;
import org.jupiter.transport.exception.IoSignals;
import org.jupiter.transport.netty.NettyTcpAcceptor;
import org.jupiter.transport.netty.channel.NettyChannel;
import org.jupiter.transport.netty.handler.AcknowledgeEncoder;
import org.jupiter.transport.netty.handler.IdleStateChecker;
import org.jupiter.transport.netty.handler.acceptor.AcceptorIdleStateTrigger;

import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.util.List;
import java.util.concurrent.ConcurrentMap;

import static java.util.concurrent.TimeUnit.SECONDS;
import static org.jupiter.common.util.JConstants.READER_IDLE_TIME_SECONDS;
import static org.jupiter.common.util.StackTraceUtil.stackTrace;
import static org.jupiter.registry.RegisterMeta.Address;
import static org.jupiter.registry.RegisterMeta.ServiceMeta;
import static org.jupiter.serialization.SerializerHolder.serializerImpl;
import static org.jupiter.transport.JProtocolHeader.*;
import static org.jupiter.transport.exception.IoSignals.ILLEGAL_MAGIC;
import static org.jupiter.transport.exception.IoSignals.ILLEGAL_SIGN;

/**
 * The server of registration center.
 *
 * ??, ??.
 *
 * provider(client)provider??server, ??provider???,
 * server???.
 *
 * consumer(client)consumer?server, ??consumer??.
 *
 * jupiter
 * org.jupiter.registry
 *
 * @author jiachun.fjc
 */
public class ConfigServer extends NettyTcpAcceptor implements RegistryMonitor {

    private static final InternalLogger logger = InternalLoggerFactory.getInstance(ConfigServer.class);

    private static final AttributeKey<ConcurrentSet<ServiceMeta>> S_SUBSCRIBE_KEY = AttributeKey
            .valueOf("server.subscribed");
    private static final AttributeKey<ConcurrentSet<RegisterMeta>> S_PUBLISH_KEY = AttributeKey
            .valueOf("server.published");

    // ?
    private final RegisterInfoContext registerInfoContext = new RegisterInfoContext();
    // 
    private final ChannelGroup subscriberChannels = new DefaultChannelGroup("subscribers",
            GlobalEventExecutor.INSTANCE);
    // ack, ????
    private final ConcurrentMap<String, MessageNonAck> messagesNonAck = Maps.newConcurrentHashMap();

    // handlers
    private final AcceptorIdleStateTrigger idleStateTrigger = new AcceptorIdleStateTrigger();
    private final MessageHandler handler = new MessageHandler();
    private final MessageEncoder encoder = new MessageEncoder();
    private final AcknowledgeEncoder ackEncoder = new AcknowledgeEncoder();

    public ConfigServer(int port) {
        super(port, false);
    }

    public ConfigServer(SocketAddress address) {
        super(address, false);
    }

    public ConfigServer(int port, int nWorks) {
        super(port, nWorks, false);
    }

    public ConfigServer(SocketAddress address, int nWorks) {
        super(address, nWorks, false);
    }

    @Override
    protected void init() {
        super.init();

        // parent options
        JConfig parent = configGroup().parent();
        parent.setOption(JOption.SO_BACKLOG, 1024);
        parent.setOption(JOption.SO_REUSEADDR, true);

        // child options
        JConfig child = configGroup().child();
        child.setOption(JOption.SO_REUSEADDR, true);
    }

    @Override
    public ChannelFuture bind(SocketAddress localAddress) {
        ServerBootstrap boot = bootstrap();

        boot.channel(NioServerSocketChannel.class).childHandler(new ChannelInitializer<SocketChannel>() {

            @Override
            protected void initChannel(SocketChannel ch) throws Exception {
                ch.pipeline().addLast(new IdleStateChecker(timer, READER_IDLE_TIME_SECONDS, 0, 0), idleStateTrigger,
                        new MessageDecoder(), encoder, ackEncoder, handler);
            }
        });

        setOptions();

        return boot.bind(localAddress);
    }

    @Override
    public List<String> listPublisherHosts() {
        List<Address> fromList = registerInfoContext.listPublisherHosts();

        return Lists.transform(fromList, new Function<Address, String>() {

            @Override
            public String apply(Address input) {
                return input.getHost();
            }
        });
    }

    @Override
    public List<String> listSubscriberAddresses() {
        List<String> hosts = Lists.newArrayList();
        for (Channel ch : subscriberChannels) {
            SocketAddress address = ch.remoteAddress();
            if (address instanceof InetSocketAddress) {
                String host = ((InetSocketAddress) address).getAddress().getHostAddress();
                int port = ((InetSocketAddress) address).getPort();
                hosts.add(new Address(host, port).toString());
            }
        }
        return hosts;
    }

    @Override
    public List<String> listAddressesByService(String group, String version, String serviceProviderName) {
        ServiceMeta serviceMeta = new ServiceMeta(group, version, serviceProviderName);
        List<Address> fromList = registerInfoContext.listAddressesByService(serviceMeta);

        return Lists.transform(fromList, new Function<Address, String>() {

            @Override
            public String apply(Address input) {
                return input.toString();
            }
        });
    }

    @Override
    public List<String> listServicesByAddress(String host, int port) {
        Address address = new Address(host, port);
        List<ServiceMeta> fromList = registerInfoContext.listServicesByAddress(address);

        return Lists.transform(fromList, new Function<ServiceMeta, String>() {

            @Override
            public String apply(ServiceMeta input) {
                return input.toString();
            }
        });
    }

    // ?, ???
    private void handlePublish(RegisterMeta meta, Channel channel) {

        logger.info("Publish {} on channel{}.", meta, channel);

        attachPublishEventOnChannel(meta, channel);

        final ServiceMeta serviceMeta = meta.getServiceMeta();
        ConfigWithVersion<ConcurrentMap<Address, RegisterMeta>> config = registerInfoContext
                .getRegisterMeta(serviceMeta);

        synchronized (registerInfoContext.publishLock(config)) {
            // putIfAbsentconfig.newVersion()???, ?
            if (config.getConfig().putIfAbsent(meta.getAddress(), meta) == null) {
                registerInfoContext.getServiceMeta(meta.getAddress()).add(serviceMeta);

                final Message msg = new Message();
                msg.sign(PUBLISH_SERVICE);
                msg.setVersion(config.newVersion()); // ?+1
                List<RegisterMeta> registerMetaList = Lists.newArrayList(config.getConfig().values());
                // ????meta??
                msg.data(new Pair<>(serviceMeta, registerMetaList));

                subscriberChannels.writeAndFlush(msg, new ChannelMatcher() {

                    @Override
                    public boolean matches(Channel channel) {
                        boolean doSend = isChannelSubscribeOnServiceMeta(serviceMeta, channel);
                        if (doSend) {
                            MessageNonAck msgNonAck = new MessageNonAck(serviceMeta, msg, channel);
                            // ack??key(??handleAcknowledge), ???
                            messagesNonAck.put(msgNonAck.id, msgNonAck);
                        }
                        return doSend;
                    }
                });
            }
        }
    }

    // ?, ???
    private void handlePublishCancel(RegisterMeta meta, Channel channel) {

        logger.info("Cancel publish {} on channel{}.", meta, channel);

        attachPublishCancelEventOnChannel(meta, channel);

        final ServiceMeta serviceMeta = meta.getServiceMeta();
        ConfigWithVersion<ConcurrentMap<Address, RegisterMeta>> config = registerInfoContext
                .getRegisterMeta(serviceMeta);
        if (config.getConfig().isEmpty()) {
            return;
        }

        synchronized (registerInfoContext.publishLock(config)) {
            // putIfAbsentconfig.newVersion()???, ?
            Address address = meta.getAddress();
            RegisterMeta data = config.getConfig().remove(address);
            if (data != null) {
                registerInfoContext.getServiceMeta(address).remove(serviceMeta);

                final Message msg = new Message();
                msg.sign(PUBLISH_SERVICE);
                msg.setVersion(config.newVersion()); // ?+1
                List<RegisterMeta> registerMetaList = Lists.newArrayList(config.getConfig().values());
                // ????meta??
                msg.data(new Pair<>(serviceMeta, registerMetaList));

                subscriberChannels.writeAndFlush(msg, new ChannelMatcher() {

                    @Override
                    public boolean matches(Channel channel) {
                        boolean doSend = isChannelSubscribeOnServiceMeta(serviceMeta, channel);
                        if (doSend) {
                            MessageNonAck msgNonAck = new MessageNonAck(serviceMeta, msg, channel);
                            // ack??key(??handleAcknowledge), ???
                            messagesNonAck.put(msgNonAck.id, msgNonAck);
                        }
                        return doSend;
                    }
                });
            }
        }
    }

    // ?
    private void handleSubscribe(ServiceMeta serviceMeta, Channel channel) {

        logger.info("Subscribe {} on channel{}.", serviceMeta, channel);

        attachSubscribeEventOnChannel(serviceMeta, channel);

        subscriberChannels.add(channel);

        ConfigWithVersion<ConcurrentMap<Address, RegisterMeta>> config = registerInfoContext
                .getRegisterMeta(serviceMeta);
        if (config.getConfig().isEmpty()) {
            return;
        }

        final Message msg = new Message();
        msg.sign(PUBLISH_SERVICE);
        msg.setVersion(config.getVersion()); // ?
        List<RegisterMeta> registerMetaList = Lists.newArrayList(config.getConfig().values());
        // ????meta??
        msg.data(new Pair<>(serviceMeta, registerMetaList));

        MessageNonAck msgNonAck = new MessageNonAck(serviceMeta, msg, channel);
        // ack??key(??handleAcknowledge), ???
        messagesNonAck.put(msgNonAck.id, msgNonAck);
        channel.writeAndFlush(msg);
    }

    // ?ack
    private void handleAcknowledge(Acknowledge ack, Channel channel) {
        messagesNonAck.remove(key(ack.sequence(), channel));
    }

    // ?Provider
    private void handleOfflineNotice(Address address) {

        logger.info("OfflineNotice on {}.", address);

        Message msg = new Message();
        msg.sign(OFFLINE_NOTICE);
        msg.data(address);
        subscriberChannels.writeAndFlush(msg);
    }

    private static String key(long sequence, Channel channel) {
        return String.valueOf(sequence) + '-' + channel.id().asShortText();
    }

    // channel(??)
    private static boolean attachPublishEventOnChannel(RegisterMeta meta, Channel channel) {
        Attribute<ConcurrentSet<RegisterMeta>> attr = channel.attr(S_PUBLISH_KEY);
        ConcurrentSet<RegisterMeta> registerMetaSet = attr.get();
        if (registerMetaSet == null) {
            ConcurrentSet<RegisterMeta> newRegisterMetaSet = new ConcurrentSet<>();
            registerMetaSet = attr.setIfAbsent(newRegisterMetaSet);
            if (registerMetaSet == null) {
                registerMetaSet = newRegisterMetaSet;
            }
        }

        return registerMetaSet.add(meta);
    }

    // ?channel(??)
    private static boolean attachPublishCancelEventOnChannel(RegisterMeta meta, Channel channel) {
        Attribute<ConcurrentSet<RegisterMeta>> attr = channel.attr(S_PUBLISH_KEY);
        ConcurrentSet<RegisterMeta> registerMetaSet = attr.get();
        if (registerMetaSet == null) {
            ConcurrentSet<RegisterMeta> newRegisterMetaSet = new ConcurrentSet<>();
            registerMetaSet = attr.setIfAbsent(newRegisterMetaSet);
            if (registerMetaSet == null) {
                registerMetaSet = newRegisterMetaSet;
            }
        }

        return registerMetaSet.remove(meta);
    }

    // channel(?)
    private static boolean attachSubscribeEventOnChannel(ServiceMeta serviceMeta, Channel channel) {
        Attribute<ConcurrentSet<ServiceMeta>> attr = channel.attr(S_SUBSCRIBE_KEY);
        ConcurrentSet<ServiceMeta> serviceMetaSet = attr.get();
        if (serviceMetaSet == null) {
            ConcurrentSet<ServiceMeta> newServiceMetaSet = new ConcurrentSet<>();
            serviceMetaSet = attr.setIfAbsent(newServiceMetaSet);
            if (serviceMetaSet == null) {
                serviceMetaSet = newServiceMetaSet;
            }
        }

        return serviceMetaSet.add(serviceMeta);
    }

    // channel(??)
    private static boolean isChannelSubscribeOnServiceMeta(ServiceMeta serviceMeta, Channel channel) {
        ConcurrentSet<ServiceMeta> serviceMetaSet = channel.attr(S_SUBSCRIBE_KEY).get();

        return serviceMetaSet != null && serviceMetaSet.contains(serviceMeta);
    }

    /**
     * ACK, ????
     */
    static class MessageNonAck {
        private final String id;

        private final ServiceMeta serviceMeta;
        private final Message msg;
        private final Channel channel;
        private final long version;
        private final long timestamp = SystemClock.millisClock().now();

        public MessageNonAck(ServiceMeta serviceMeta, Message msg, Channel channel) {
            this.serviceMeta = serviceMeta;
            this.msg = msg;
            this.channel = channel;
            this.version = msg.getVersion();

            id = key(msg.sequence(), channel);
        }
    }

    /**
     * **************************************************************************************************
     *                                          Protocol
     *                                                  ?
     *       2      1       1        8           4      
     *                                                  
     *                                                  
     *    MAGIC   Sign    Status   Invoke Id   Body Length                   Body Content              
     *                                                  
     *                                                  
     *
     * ?16
     * = 2 // MAGIC = (short) 0xbabe
     * + 1 // ??, ??
     * + 1 // 
     * + 8 // ? id long 
     * + 4 // ?body, int
     */
    static class MessageDecoder extends ReplayingDecoder<MessageDecoder.State> {

        public MessageDecoder() {
            super(State.HEADER_MAGIC);
        }

        // ??
        private final JProtocolHeader header = new JProtocolHeader();

        @Override
        protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
            switch (state()) {
            case HEADER_MAGIC:
                checkMagic(in.readShort()); // MAGIC
                checkpoint(State.HEADER_SIGN);
            case HEADER_SIGN:
                header.sign(in.readByte()); // ??
                checkpoint(State.HEADER_STATUS);
            case HEADER_STATUS:
                in.readByte(); // no-op
                checkpoint(State.HEADER_ID);
            case HEADER_ID:
                header.id(in.readLong()); // ?id
                checkpoint(State.HEADER_BODY_LENGTH);
            case HEADER_BODY_LENGTH:
                header.bodyLength(in.readInt()); // ?
                checkpoint(State.BODY);
            case BODY:
                switch (header.sign()) {
                case HEARTBEAT:
                    break;
                case PUBLISH_SERVICE:
                case PUBLISH_CANCEL_SERVICE:
                case SUBSCRIBE_SERVICE:
                case OFFLINE_NOTICE: {
                    byte[] bytes = new byte[header.bodyLength()];
                    in.readBytes(bytes);

                    Message msg = serializerImpl().readObject(bytes, Message.class);
                    msg.sign(header.sign());
                    out.add(msg);

                    break;
                }
                case ACK: {
                    byte[] bytes = new byte[header.bodyLength()];
                    in.readBytes(bytes);

                    Acknowledge ack = serializerImpl().readObject(bytes, Acknowledge.class);
                    out.add(ack);

                    break;
                }
                default:
                    throw ILLEGAL_SIGN;
                }
                checkpoint(State.HEADER_MAGIC);
            }
        }

        private static void checkMagic(short magic) throws Signal {
            if (MAGIC != magic) {
                throw ILLEGAL_MAGIC;
            }
        }

        enum State {
            HEADER_MAGIC, HEADER_SIGN, HEADER_STATUS, HEADER_ID, HEADER_BODY_LENGTH, BODY
        }
    }

    /**
     * **************************************************************************************************
     *                                          Protocol
     *                                                  ?
     *       2      1       1        8           4      
     *                                                  
     *                                                  
     *    MAGIC   Sign    Status   Invoke Id   Body Length                   Body Content              
     *                                                  
     *                                                  
     *
     * ?16
     * = 2 // MAGIC = (short) 0xbabe
     * + 1 // ??, ??
     * + 1 // 
     * + 8 // ? id long 
     * + 4 // ?body, int
     */
    @ChannelHandler.Sharable
    static class MessageEncoder extends MessageToByteEncoder<Message> {

        @Override
        protected void encode(ChannelHandlerContext ctx, Message msg, ByteBuf out) throws Exception {
            byte[] bytes = serializerImpl().writeObject(msg);
            out.writeShort(MAGIC).writeByte(msg.sign()).writeByte(0).writeLong(0).writeInt(bytes.length)
                    .writeBytes(bytes);
        }
    }

    @ChannelHandler.Sharable
    class MessageHandler extends ChannelInboundHandlerAdapter {

        @Override
        public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
            Channel channel = ctx.channel();

            if (msg instanceof Message) {
                Message obj = (Message) msg;
                switch (obj.sign()) {
                case PUBLISH_SERVICE:
                case PUBLISH_CANCEL_SERVICE:
                    RegisterMeta meta = (RegisterMeta) obj.data();
                    if (Strings.isNullOrEmpty(meta.getHost())) {
                        SocketAddress address = channel.remoteAddress();
                        if (address instanceof InetSocketAddress) {
                            meta.setHost(((InetSocketAddress) address).getAddress().getHostAddress());
                        } else {
                            logger.warn("Could not get remote host: {}, info: {}", channel, meta);

                            return;
                        }
                    }

                    if (obj.sign() == PUBLISH_SERVICE) {
                        handlePublish(meta, channel);
                    } else if (obj.sign() == PUBLISH_CANCEL_SERVICE) {
                        handlePublishCancel(meta, channel);
                    }
                    channel.writeAndFlush(new Acknowledge(obj.sequence())) // ?ACK
                            .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE);

                    break;
                case SUBSCRIBE_SERVICE:
                    handleSubscribe((ServiceMeta) obj.data(), channel);
                    channel.writeAndFlush(new Acknowledge(obj.sequence())) // ?ACK
                            .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE);

                    break;
                case OFFLINE_NOTICE:
                    handleOfflineNotice((Address) obj.data());

                    break;
                }
            } else if (msg instanceof Acknowledge) {
                handleAcknowledge((Acknowledge) msg, channel);
            } else {
                logger.warn("Unexpected msg type received:{}.", msg.getClass());

                ReferenceCountUtil.release(msg);
            }
        }

        @Override
        public void channelInactive(ChannelHandlerContext ctx) throws Exception {
            Channel channel = ctx.channel();

            // ????
            ConcurrentSet<RegisterMeta> registerMetaSet = channel.attr(S_PUBLISH_KEY).get();

            if (registerMetaSet == null) {
                return;
            }

            Address address = null;
            for (RegisterMeta meta : registerMetaSet) {
                if (address == null) {
                    address = meta.getAddress();
                }
                handlePublishCancel(meta, channel);
            }

            // 
            handleOfflineNotice(address);
        }

        @Override
        public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception {
            Channel ch = ctx.channel();

            // ?: ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK
            // ?: ChannelOption.WRITE_BUFFER_LOW_WATER_MARK
            if (!ch.isWritable()) {
                // ?channel(OutboundBuffer)?WRITE_BUFFER_HIGH_WATER_MARK
                logger.warn(
                        "{} is not writable, high water mask: {}, the number of flushed entries that are not written yet: {}.",
                        ch, ch.config().getWriteBufferHighWaterMark(), ch.unsafe().outboundBuffer().size());

                ch.config().setAutoRead(false);
            } else {
                // ??OutboundBuffer?WRITE_BUFFER_LOW_WATER_MARK
                logger.warn(
                        "{} is writable(rehabilitate), low water mask: {}, the number of flushed entries that are not written yet: {}.",
                        ch, ch.config().getWriteBufferLowWaterMark(), ch.unsafe().outboundBuffer().size());

                ch.config().setAutoRead(true);
            }
        }

        @Override
        public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
            JChannel jChannel = NettyChannel.attachChannel(ctx.channel());
            if (cause instanceof Signal) {
                IoSignals.handleSignal((Signal) cause, jChannel);
            } else {
                logger.error("An exception has been caught {}, on {}.", stackTrace(cause), jChannel);
            }
        }
    }

    private class AckTimeoutScanner implements Runnable {

        @SuppressWarnings("InfiniteLoopStatement")
        @Override
        public void run() {
            for (;;) {
                try {
                    for (MessageNonAck m : messagesNonAck.values()) {
                        if (SystemClock.millisClock().now() - m.timestamp > SECONDS.toMillis(10)) {

                            // 
                            if (messagesNonAck.remove(m.id) == null) {
                                continue;
                            }

                            if (registerInfoContext.getRegisterMeta(m.serviceMeta).getVersion() > m.version) {
                                // ????
                                continue;
                            }

                            if (m.channel.isActive()) {
                                MessageNonAck msgNonAck = new MessageNonAck(m.serviceMeta, m.msg, m.channel);
                                messagesNonAck.put(msgNonAck.id, msgNonAck);
                                m.channel.writeAndFlush(m.msg)
                                        .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE);
                            }
                        }
                    }

                    Thread.sleep(300);
                } catch (Throwable t) {
                    logger.error("An exception has been caught while scanning the timeout acknowledges {}.", t);
                }
            }
        }
    }

    {
        Thread t = new Thread(new AckTimeoutScanner(), "ack.timeout.scanner");
        t.setDaemon(true);
        t.start();
    }
}