Java tutorial
/** * Copyright (C) 2010-2013 Alibaba Group Holding Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.alibaba.rocketmq.remoting.netty; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandlerContext; import java.util.HashMap; import java.util.Iterator; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.rocketmq.remoting.ChannelEventListener; import com.alibaba.rocketmq.remoting.InvokeCallback; import com.alibaba.rocketmq.remoting.RPCHook; import com.alibaba.rocketmq.remoting.common.Pair; import com.alibaba.rocketmq.remoting.common.RemotingHelper; import com.alibaba.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; import com.alibaba.rocketmq.remoting.common.ServiceThread; import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; import com.alibaba.rocketmq.remoting.exception.RemotingTooMuchRequestException; import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; import com.alibaba.rocketmq.remoting.protocol.RemotingSysResponseCode; /** * ServerClient * * @author shijia.wxr<vintage.wang@gmail.com> * @since 2013-7-13 */ public abstract class NettyRemotingAbstract { private static final Logger plog = LoggerFactory.getLogger(RemotingHelper.RemotingLogName); // ??OnewayNetty protected final Semaphore semaphoreOneway; // ??Netty protected final Semaphore semaphoreAsync; // protected final ConcurrentHashMap<Integer /* opaque */, ResponseFuture> responseTable = new ConcurrentHashMap<Integer, ResponseFuture>( 256); // ?? protected Pair<NettyRequestProcessor, ExecutorService> defaultRequestProcessor; // ?RPC? protected final HashMap<Integer/* request code */, Pair<NettyRequestProcessor, ExecutorService>> processorTable = new HashMap<Integer, Pair<NettyRequestProcessor, ExecutorService>>( 64); protected final NettyEventExecuter nettyEventExecuter = new NettyEventExecuter(); public abstract ChannelEventListener getChannelEventListener(); public abstract RPCHook getRPCHook(); public void putNettyEvent(final NettyEvent event) { this.nettyEventExecuter.putNettyEvent(event); } class NettyEventExecuter extends ServiceThread { private final LinkedBlockingQueue<NettyEvent> eventQueue = new LinkedBlockingQueue<NettyEvent>(); private final int MaxSize = 10000; public void putNettyEvent(final NettyEvent event) { if (this.eventQueue.size() <= MaxSize) { this.eventQueue.add(event); } else { plog.warn("event queue size[{}] enough, so drop this event {}", this.eventQueue.size(), event.toString()); } } @Override public void run() { plog.info(this.getServiceName() + " service started"); final ChannelEventListener listener = NettyRemotingAbstract.this.getChannelEventListener(); while (!this.isStoped()) { try { NettyEvent event = this.eventQueue.poll(3000, TimeUnit.MILLISECONDS); if (event != null && listener != null) { switch (event.getType()) { case IDLE: listener.onChannelIdle(event.getRemoteAddr(), event.getChannel()); break; case CLOSE: listener.onChannelClose(event.getRemoteAddr(), event.getChannel()); break; case CONNECT: listener.onChannelConnect(event.getRemoteAddr(), event.getChannel()); break; case EXCEPTION: listener.onChannelException(event.getRemoteAddr(), event.getChannel()); break; default: break; } } } catch (Exception e) { plog.warn(this.getServiceName() + " service has exception. ", e); } } plog.info(this.getServiceName() + " service end"); } @Override public String getServiceName() { return NettyEventExecuter.class.getSimpleName(); } } public NettyRemotingAbstract(final int permitsOneway, final int permitsAsync) { this.semaphoreOneway = new Semaphore(permitsOneway, true); this.semaphoreAsync = new Semaphore(permitsAsync, true); } public void processRequestCommand(final ChannelHandlerContext ctx, final RemotingCommand cmd) { final Pair<NettyRequestProcessor, ExecutorService> matched = this.processorTable.get(cmd.getCode()); final Pair<NettyRequestProcessor, ExecutorService> pair = null == matched ? this.defaultRequestProcessor : matched; if (pair != null) { Runnable run = new Runnable() { @Override public void run() { try { RPCHook rpcHook = NettyRemotingAbstract.this.getRPCHook(); if (rpcHook != null) { rpcHook.doBeforeRequest(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd); } final RemotingCommand response = pair.getObject1().processRequest(ctx, cmd); if (rpcHook != null) { rpcHook.doAfterResponse(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd, response); } if (!cmd.isOnewayRPC()) { if (response != null) { response.setOpaque(cmd.getOpaque()); response.markResponseType(); try { ctx.writeAndFlush(response); } catch (Throwable e) { plog.error("process request over, but response failed", e); plog.error(cmd.toString()); plog.error(response.toString()); } } else { // ?processRequest? } } } catch (Throwable e) { plog.error("process request exception", e); plog.error(cmd.toString()); if (!cmd.isOnewayRPC()) { final RemotingCommand response = RemotingCommand.createResponseCommand( RemotingSysResponseCode.SYSTEM_ERROR, // RemotingHelper.exceptionSimpleDesc(e)); response.setOpaque(cmd.getOpaque()); ctx.writeAndFlush(response); } } } }; try { // ?????? pair.getObject2().submit(run); } catch (RejectedExecutionException e) { // ?10s? if ((System.currentTimeMillis() % 10000) == 0) { plog.warn(RemotingHelper.parseChannelRemoteAddr(ctx.channel()) // + ", too many requests and system thread pool busy, RejectedExecutionException " // + pair.getObject2().toString() // + " request code: " + cmd.getCode()); } if (!cmd.isOnewayRPC()) { final RemotingCommand response = RemotingCommand.createResponseCommand( RemotingSysResponseCode.SYSTEM_BUSY, "too many requests and system thread pool busy, please try another server"); response.setOpaque(cmd.getOpaque()); ctx.writeAndFlush(response); } } } else { String error = " request type " + cmd.getCode() + " not supported"; final RemotingCommand response = RemotingCommand .createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error); response.setOpaque(cmd.getOpaque()); ctx.writeAndFlush(response); plog.error(RemotingHelper.parseChannelRemoteAddr(ctx.channel()) + error); } } public void processResponseCommand(ChannelHandlerContext ctx, RemotingCommand cmd) { final ResponseFuture responseFuture = responseTable.get(cmd.getOpaque()); if (responseFuture != null) { responseFuture.setResponseCommand(cmd); responseFuture.release(); responseTable.remove(cmd.getOpaque()); if (responseFuture.getInvokeCallback() != null) { boolean runInThisThread = false; ExecutorService executor = this.getCallbackExecutor(); if (executor != null) { try { executor.submit(new Runnable() { @Override public void run() { try { responseFuture.executeInvokeCallback(); } catch (Throwable e) { plog.warn("excute callback in executor exception, and callback throw", e); } } }); } catch (Exception e) { runInThisThread = true; plog.warn("excute callback in executor exception, maybe executor busy", e); } } else { runInThisThread = true; } if (runInThisThread) { try { responseFuture.executeInvokeCallback(); } catch (Throwable e) { plog.warn("executeInvokeCallback Exception", e); } } } else { responseFuture.putResponse(cmd); } } else { plog.warn("receive response, but not matched any request, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); plog.warn(cmd.toString()); } } public void processMessageReceived(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception { final RemotingCommand cmd = msg; if (cmd != null) { switch (cmd.getType()) { case REQUEST_COMMAND: processRequestCommand(ctx, cmd); break; case RESPONSE_COMMAND: processResponseCommand(ctx, cmd); break; default: break; } } } abstract public ExecutorService getCallbackExecutor(); public void scanResponseTable() { Iterator<Entry<Integer, ResponseFuture>> it = this.responseTable.entrySet().iterator(); while (it.hasNext()) { Entry<Integer, ResponseFuture> next = it.next(); ResponseFuture rep = next.getValue(); if ((rep.getBeginTimestamp() + rep.getTimeoutMillis() + 1000) <= System.currentTimeMillis()) { it.remove(); try { rep.executeInvokeCallback(); } catch (Throwable e) { plog.warn("scanResponseTable, operationComplete Exception", e); } finally { rep.release(); } plog.warn("remove timeout request, " + rep); } } } public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis) throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException { try { final ResponseFuture responseFuture = new ResponseFuture(request.getOpaque(), timeoutMillis, null, null); this.responseTable.put(request.getOpaque(), responseFuture); channel.writeAndFlush(request).addListener(new ChannelFutureListener() { @Override public void operationComplete(ChannelFuture f) throws Exception { if (f.isSuccess()) { responseFuture.setSendRequestOK(true); return; } else { responseFuture.setSendRequestOK(false); } responseTable.remove(request.getOpaque()); responseFuture.setCause(f.cause()); responseFuture.putResponse(null); plog.warn("send a request command to channel <" + channel.remoteAddress() + "> failed."); plog.warn(request.toString()); } }); RemotingCommand responseCommand = responseFuture.waitResponse(timeoutMillis); if (null == responseCommand) { if (responseFuture.isSendRequestOK()) { throw new RemotingTimeoutException(RemotingHelper.parseChannelRemoteAddr(channel), timeoutMillis, responseFuture.getCause()); } else { throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), responseFuture.getCause()); } } return responseCommand; } finally { this.responseTable.remove(request.getOpaque()); } } public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis, final InvokeCallback invokeCallback) throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { boolean acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); if (acquired) { final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync); final ResponseFuture responseFuture = new ResponseFuture(request.getOpaque(), timeoutMillis, invokeCallback, once); this.responseTable.put(request.getOpaque(), responseFuture); try { channel.writeAndFlush(request).addListener(new ChannelFutureListener() { @Override public void operationComplete(ChannelFuture f) throws Exception { if (f.isSuccess()) { responseFuture.setSendRequestOK(true); return; } else { responseFuture.setSendRequestOK(false); } responseFuture.putResponse(null); responseTable.remove(request.getOpaque()); try { responseFuture.executeInvokeCallback(); } catch (Throwable e) { plog.warn("excute callback in writeAndFlush addListener, and callback throw", e); } finally { responseFuture.release(); } plog.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel)); plog.warn(request.toString()); } }); } catch (Exception e) { responseFuture.release(); plog.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e); throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); } } else { if (timeoutMillis <= 0) { throw new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast"); } else { String info = String.format( "invokeAsyncImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d", // timeoutMillis, // this.semaphoreAsync.getQueueLength(), // this.semaphoreAsync.availablePermits()// ); plog.warn(info); plog.warn(request.toString()); throw new RemotingTimeoutException(info); } } } public void invokeOnewayImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis) throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { request.markOnewayRPC(); boolean acquired = this.semaphoreOneway.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); if (acquired) { final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreOneway); try { channel.writeAndFlush(request).addListener(new ChannelFutureListener() { @Override public void operationComplete(ChannelFuture f) throws Exception { once.release(); if (!f.isSuccess()) { plog.warn( "send a request command to channel <" + channel.remoteAddress() + "> failed."); plog.warn(request.toString()); } } }); } catch (Exception e) { once.release(); plog.warn("write send a request command to channel <" + channel.remoteAddress() + "> failed."); throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); } } else { if (timeoutMillis <= 0) { throw new RemotingTooMuchRequestException("invokeOnewayImpl invoke too fast"); } else { String info = String.format( "invokeOnewayImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d", // timeoutMillis, // this.semaphoreAsync.getQueueLength(), // this.semaphoreAsync.availablePermits()// ); plog.warn(info); plog.warn(request.toString()); throw new RemotingTimeoutException(info); } } } }