Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.wolfbe.configcenter.remoting.netty; import com.wolfbe.configcenter.remoting.ChannelEventListener; import com.wolfbe.configcenter.remoting.InvokeCallback; import com.wolfbe.configcenter.remoting.RPCHook; import com.wolfbe.configcenter.remoting.common.RemotingPair; import com.wolfbe.configcenter.remoting.common.RemotingHelper; import com.wolfbe.configcenter.remoting.common.SemaphoreReleaseOnlyOnce; import com.wolfbe.configcenter.remoting.common.ServiceThread; import com.wolfbe.configcenter.remoting.exception.RemotingSendRequestException; import com.wolfbe.configcenter.remoting.exception.RemotingTimeoutException; import com.wolfbe.configcenter.remoting.exception.RemotingTooMuchRequestException; import com.wolfbe.configcenter.remoting.protocol.RemotingCommand; import com.wolfbe.configcenter.remoting.protocol.RemotingSysResponseCode; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.SocketAddress; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map.Entry; import java.util.concurrent.*; /** * @author shijia.wxr */ public abstract class NettyRemotingAbstract { private static final Logger plog = LoggerFactory.getLogger(RemotingHelper.RemotingLogName); protected final Semaphore semaphoreOneway; protected final Semaphore semaphoreAsync; protected final ConcurrentHashMap<Integer /* opaque */, ResponseFuture> responseTable = new ConcurrentHashMap<Integer, ResponseFuture>( 256); protected final HashMap<Integer/* request code */, RemotingPair<NettyRequestProcessor, ExecutorService>> processorTable = new HashMap<Integer, RemotingPair<NettyRequestProcessor, ExecutorService>>( 64); protected final NettyEventExecuter nettyEventExecuter = new NettyEventExecuter(); protected RemotingPair<NettyRequestProcessor, ExecutorService> defaultRequestProcessor; public NettyRemotingAbstract(final int permitsOneway, final int permitsAsync) { this.semaphoreOneway = new Semaphore(permitsOneway, true); this.semaphoreAsync = new Semaphore(permitsAsync, true); } public abstract ChannelEventListener getChannelEventListener(); public void putNettyEvent(final NettyEvent event) { this.nettyEventExecuter.putNettyEvent(event); } public void processMessageReceived(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception { final RemotingCommand cmd = msg; if (cmd != null) { switch (cmd.getType()) { case REQUEST_COMMAND: processRequestCommand(ctx, cmd); break; case RESPONSE_COMMAND: processResponseCommand(ctx, cmd); break; default: break; } } } /** * ?? * @param ctx * @param cmd ??cmd */ public void processRequestCommand(final ChannelHandlerContext ctx, final RemotingCommand cmd) { final RemotingPair<NettyRequestProcessor, ExecutorService> matched = this.processorTable.get(cmd.getCode()); final RemotingPair<NettyRequestProcessor, ExecutorService> pair = null == matched ? this.defaultRequestProcessor : matched; final int opaque = cmd.getOpaque(); if (pair != null) { // Runnable // ?pairExecutorService Runnable run = new Runnable() { @Override public void run() { try { RPCHook rpcHook = NettyRemotingAbstract.this.getRPCHook(); if (rpcHook != null) { // rpcHook?? rpcHook.doBeforeRequest(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd); } // NettyRequestProcessor? final RemotingCommand response = pair.getObject1().processRequest(ctx, cmd); if (rpcHook != null) { // rpcHook?? rpcHook.doAfterResponse(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd, response); } if (!cmd.isOnewayRPC()) { if (response != null) { response.setOpaque(opaque); response.markResponseType(); try { ctx.writeAndFlush(response); } catch (Throwable e) { plog.error("process request over, but response failed", e); plog.error(cmd.toString()); plog.error(response.toString()); } } else { } } } catch (Throwable e) { if (!"com.aliyun.openservices.ons.api.impl.authority.exception.AuthenticationException" .equals(e.getClass().getCanonicalName())) { plog.error("process request exception", e); plog.error(cmd.toString()); } if (!cmd.isOnewayRPC()) { final RemotingCommand response = RemotingCommand.createResponseCommand( RemotingSysResponseCode.SYSTEM_ERROR, // RemotingHelper.exceptionSimpleDesc(e)); response.setOpaque(opaque); ctx.writeAndFlush(response); } } } }; if (pair.getObject1().rejectRequest()) { final RemotingCommand response = RemotingCommand.createResponseCommand( RemotingSysResponseCode.SYSTEM_BUSY, "[REJECTREQUEST]system busy, start flow control for a while"); response.setOpaque(opaque); ctx.writeAndFlush(response); return; } try { final RequestTask requestTask = new RequestTask(run, ctx.channel(), cmd); // ? pair.getObject2().submit(requestTask); } catch (RejectedExecutionException e) { if ((System.currentTimeMillis() % 10000) == 0) { plog.warn(RemotingHelper.parseChannelRemoteAddr(ctx.channel()) // + ", too many requests and system thread pool busy, RejectedExecutionException " // + pair.getObject2().toString() // + " request code: " + cmd.getCode()); } if (!cmd.isOnewayRPC()) { final RemotingCommand response = RemotingCommand.createResponseCommand( RemotingSysResponseCode.SYSTEM_BUSY, "[OVERLOAD]system busy, start flow control for a while"); response.setOpaque(opaque); ctx.writeAndFlush(response); } } } else { // ?defaultRequestProcessor // codeRequestProcessor?? // TODO ??cmd.isOnewayRPC() String error = " request type " + cmd.getCode() + " not supported"; final RemotingCommand response = RemotingCommand .createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error); response.setOpaque(opaque); ctx.writeAndFlush(response); plog.error(RemotingHelper.parseChannelRemoteAddr(ctx.channel()) + error); } } /** * ???????scanResponseTable? * scanResponseTable * @param ctx * @param cmd ??cmd */ public void processResponseCommand(ChannelHandlerContext ctx, RemotingCommand cmd) { final int opaque = cmd.getOpaque(); final ResponseFuture responseFuture = responseTable.get(opaque); if (responseFuture != null) { responseFuture.setResponseCommand(cmd); responseFuture.release(); responseTable.remove(opaque); if (responseFuture.getInvokeCallback() != null) { boolean runInThisThread = false; ExecutorService executor = this.getCallbackExecutor(); if (executor != null) { try { executor.submit(new Runnable() { @Override public void run() { try { // responseFuture.executeInvokeCallback(); } catch (Throwable e) { plog.warn("execute callback in executor exception, and callback throw", e); } } }); } catch (Exception e) { runInThisThread = true; plog.warn("execute callback in executor exception, maybe executor busy", e); } } else { runInThisThread = true; } if (runInThisThread) { try { responseFuture.executeInvokeCallback(); } catch (Throwable e) { plog.warn("executeInvokeCallback Exception", e); } } } else { responseFuture.putResponse(cmd); } } else { plog.warn("receive response, but not matched any request, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); plog.warn(cmd.toString()); } } public abstract RPCHook getRPCHook(); abstract public ExecutorService getCallbackExecutor(); /** * ??? */ public void scanResponseTable() { final List<ResponseFuture> rfList = new LinkedList<ResponseFuture>(); Iterator<Entry<Integer, ResponseFuture>> it = this.responseTable.entrySet().iterator(); while (it.hasNext()) { Entry<Integer, ResponseFuture> next = it.next(); ResponseFuture rep = next.getValue(); if ((rep.getBeginTimestamp() + rep.getTimeoutMillis() + 1000) <= System.currentTimeMillis()) { rep.release(); it.remove(); rfList.add(rep); plog.warn("remove timeout request, " + rep); } } for (ResponseFuture rf : rfList) { try { // rf.executeInvokeCallback(); } catch (Throwable e) { plog.warn("scanResponseTable, operationComplete Exception", e); } } } /** * ????? * @param channel * @param request * @param timeoutMillis * @return * @throws InterruptedException * @throws RemotingSendRequestException * @throws RemotingTimeoutException */ public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis) throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException { final int opaque = request.getOpaque(); try { final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis, null, null); this.responseTable.put(opaque, responseFuture); final SocketAddress addr = channel.remoteAddress(); channel.writeAndFlush(request).addListener(new ChannelFutureListener() { @Override public void operationComplete(ChannelFuture f) throws Exception { if (f.isSuccess()) { responseFuture.setSendRequestOK(true); return; } else { responseFuture.setSendRequestOK(false); } responseTable.remove(opaque); responseFuture.setCause(f.cause()); responseFuture.putResponse(null); plog.warn("send a request command to channel <" + addr + "> failed."); } }); // ?responseFuture.putResponse RemotingCommand responseCommand = responseFuture.waitResponse(timeoutMillis); if (null == responseCommand) { if (responseFuture.isSendRequestOK()) { throw new RemotingTimeoutException(RemotingHelper.parseSocketAddressAddr(addr), timeoutMillis, responseFuture.getCause()); } else { throw new RemotingSendRequestException(RemotingHelper.parseSocketAddressAddr(addr), responseFuture.getCause()); } } return responseCommand; } finally { this.responseTable.remove(opaque); } } /** * ???? * @param channel * @param request * @param timeoutMillis * @param invokeCallback * @throws InterruptedException * @throws RemotingTooMuchRequestException * @throws RemotingTimeoutException * @throws RemotingSendRequestException */ public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis, final InvokeCallback invokeCallback) throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { final int opaque = request.getOpaque(); boolean acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); if (acquired) { final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync); final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis, invokeCallback, once); this.responseTable.put(opaque, responseFuture); try { channel.writeAndFlush(request).addListener(new ChannelFutureListener() { @Override public void operationComplete(ChannelFuture f) throws Exception { if (f.isSuccess()) { responseFuture.setSendRequestOK(true); return; } else { responseFuture.setSendRequestOK(false); } responseFuture.putResponse(null); responseTable.remove(opaque); try { responseFuture.executeInvokeCallback(); } catch (Throwable e) { plog.warn("excute callback in writeAndFlush addListener, and callback throw", e); } finally { responseFuture.release(); } plog.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel)); } }); } catch (Exception e) { responseFuture.release(); plog.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e); throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); } } else { String info = String.format( "invokeAsyncImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d", // timeoutMillis, // this.semaphoreAsync.getQueueLength(), // this.semaphoreAsync.availablePermits()// ); plog.warn(info); throw new RemotingTooMuchRequestException(info); } } /** * ????? * @param channel * @param request * @param timeoutMillis * @throws InterruptedException * @throws RemotingTooMuchRequestException * @throws RemotingTimeoutException * @throws RemotingSendRequestException */ public void invokeOnewayImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis) throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { request.markOnewayRPC(); boolean acquired = this.semaphoreOneway.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); if (acquired) { final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreOneway); try { channel.writeAndFlush(request).addListener(new ChannelFutureListener() { @Override public void operationComplete(ChannelFuture f) throws Exception { once.release(); if (!f.isSuccess()) { plog.warn( "send a request command to channel <" + channel.remoteAddress() + "> failed."); } } }); } catch (Exception e) { once.release(); plog.warn("write send a request command to channel <" + channel.remoteAddress() + "> failed."); throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); } } else { if (timeoutMillis <= 0) { throw new RemotingTooMuchRequestException("invokeOnewayImpl invoke too fast"); } else { String info = String.format( "invokeOnewayImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d", // timeoutMillis, // this.semaphoreOneway.getQueueLength(), // this.semaphoreOneway.availablePermits()// ); plog.warn(info); throw new RemotingTimeoutException(info); } } } /** * ?Channel??? */ class NettyEventExecuter extends ServiceThread { private final LinkedBlockingQueue<NettyEvent> eventQueue = new LinkedBlockingQueue<NettyEvent>(); private final int maxSize = 10000; public void putNettyEvent(final NettyEvent event) { if (this.eventQueue.size() <= maxSize) { this.eventQueue.add(event); } else { plog.warn("event queue size[{}] enough, so drop this event {}", this.eventQueue.size(), event.toString()); } } @Override public void run() { plog.info(this.getServiceName() + " service started"); final ChannelEventListener listener = NettyRemotingAbstract.this.getChannelEventListener(); while (!this.isStoped()) { try { NettyEvent event = this.eventQueue.poll(3000, TimeUnit.MILLISECONDS); if (event != null && listener != null) { switch (event.getType()) { case IDLE: listener.onChannelIdle(event.getRemoteAddr(), event.getChannel()); break; case CLOSE: listener.onChannelClose(event.getRemoteAddr(), event.getChannel()); break; case CONNECT: listener.onChannelConnect(event.getRemoteAddr(), event.getChannel()); break; case EXCEPTION: listener.onChannelException(event.getRemoteAddr(), event.getChannel()); break; default: break; } } } catch (Exception e) { plog.warn(this.getServiceName() + " service has exception. ", e); } } plog.info(this.getServiceName() + " service end"); } @Override public String getServiceName() { return NettyEventExecuter.class.getSimpleName(); } } }