Java tutorial
/*********************************************************************************************************************** * Copyright (C) 2010-2014 by the Stratosphere project (http://stratosphere.eu) * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. **********************************************************************************************************************/ package eu.stratosphere.runtime.io.network.netty; import eu.stratosphere.runtime.io.network.ChannelManager; import eu.stratosphere.runtime.io.network.Envelope; import eu.stratosphere.runtime.io.network.RemoteReceiver; import io.netty.bootstrap.Bootstrap; import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.PooledByteBufAllocator; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelOption; import io.netty.channel.FixedRecvByteBufAllocator; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.util.concurrent.Future; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import java.io.IOException; import java.net.InetAddress; import java.util.Date; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; public class NettyConnectionManager { private static final Log LOG = LogFactory.getLog(NettyConnectionManager.class); private static final int DEBUG_PRINT_QUEUED_ENVELOPES_EVERY_MS = 10000; private final ChannelManager channelManager; private final ServerBootstrap in; private final Bootstrap out; private final ConcurrentMap<RemoteReceiver, Object> outConnections; public NettyConnectionManager(ChannelManager channelManager, InetAddress bindAddress, int bindPort, int bufferSize, int numInThreads, int numOutThreads, int lowWaterMark, int highWaterMark) { this.outConnections = new ConcurrentHashMap<RemoteReceiver, Object>(); this.channelManager = channelManager; // -------------------------------------------------------------------- int defaultNumThreads = Math.max(Runtime.getRuntime().availableProcessors() / 4, 1); numInThreads = (numInThreads == -1) ? defaultNumThreads : numInThreads; numOutThreads = (numOutThreads == -1) ? defaultNumThreads : numOutThreads; LOG.info(String.format("Starting with %d incoming and %d outgoing connection threads.", numInThreads, numOutThreads)); lowWaterMark = (lowWaterMark == -1) ? bufferSize / 2 : lowWaterMark; highWaterMark = (highWaterMark == -1) ? bufferSize : highWaterMark; LOG.info(String.format("Setting low water mark to %d and high water mark to %d bytes.", lowWaterMark, highWaterMark)); // -------------------------------------------------------------------- // server bootstrap (incoming connections) // -------------------------------------------------------------------- this.in = new ServerBootstrap(); this.in.group(new NioEventLoopGroup(numInThreads)).channel(NioServerSocketChannel.class) .localAddress(bindAddress, bindPort).childHandler(new ChannelInitializer<SocketChannel>() { @Override public void initChannel(SocketChannel channel) throws Exception { channel.pipeline() .addLast(new InboundEnvelopeDecoder(NettyConnectionManager.this.channelManager)) .addLast(new InboundEnvelopeDispatcherHandler( NettyConnectionManager.this.channelManager)); } }).option(ChannelOption.RCVBUF_ALLOCATOR, new FixedRecvByteBufAllocator(bufferSize)) .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT); // -------------------------------------------------------------------- // client bootstrap (outgoing connections) // -------------------------------------------------------------------- this.out = new Bootstrap(); this.out.group(new NioEventLoopGroup(numOutThreads)).channel(NioSocketChannel.class) .handler(new ChannelInitializer<SocketChannel>() { @Override public void initChannel(SocketChannel channel) throws Exception { channel.pipeline().addLast(new OutboundEnvelopeEncoder()); } }).option(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, lowWaterMark) .option(ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK, highWaterMark) .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) .option(ChannelOption.TCP_NODELAY, false).option(ChannelOption.SO_KEEPALIVE, true); try { this.in.bind().sync(); } catch (InterruptedException e) { throw new RuntimeException("Could not bind server socket for incoming connections."); } if (LOG.isDebugEnabled()) { new Thread(new Runnable() { @Override public void run() { Date date = new Date(); while (true) { try { Thread.sleep(DEBUG_PRINT_QUEUED_ENVELOPES_EVERY_MS); date.setTime(System.currentTimeMillis()); System.out.println(date); System.out.println(getNonZeroNumQueuedEnvelopes()); } catch (InterruptedException e) { e.printStackTrace(); } } } }).start(); } } public void shutdown() { Future<?> inShutdownFuture = this.in.group().shutdownGracefully(); Future<?> outShutdownFuture = this.out.group().shutdownGracefully(); try { inShutdownFuture.sync(); outShutdownFuture.sync(); } catch (InterruptedException e) { throw new RuntimeException("Could not properly shutdown connections."); } } public void enqueue(Envelope envelope, RemoteReceiver receiver) throws IOException { // Get the channel. The channel may be // 1) a channel that already exists (usual case) -> just send the data // 2) a channel that is in buildup (sometimes) -> attach to the future and wait for the actual channel // 3) not yet existing -> establish the channel final Object entry = this.outConnections.get(receiver); final OutboundConnectionQueue channel; if (entry != null) { // existing channel or channel in buildup if (entry instanceof OutboundConnectionQueue) { channel = (OutboundConnectionQueue) entry; } else { ChannelInBuildup future = (ChannelInBuildup) entry; channel = future.waitForChannel(); } } else { // No channel yet. Create one, but watch out for a race. // We create a "buildup future" and atomically add it to the map. // Only the thread that really added it establishes the channel. // The others need to wait on that original establisher's future. ChannelInBuildup inBuildup = new ChannelInBuildup(this.out, receiver); Object old = this.outConnections.putIfAbsent(receiver, inBuildup); if (old == null) { this.out.connect(receiver.getConnectionAddress()).addListener(inBuildup); channel = inBuildup.waitForChannel(); Object previous = this.outConnections.put(receiver, channel); if (inBuildup != previous) { throw new IOException("Race condition during channel build up."); } } else if (old instanceof ChannelInBuildup) { channel = ((ChannelInBuildup) old).waitForChannel(); } else { channel = (OutboundConnectionQueue) old; } } channel.enqueue(envelope); } private String getNonZeroNumQueuedEnvelopes() { StringBuilder str = new StringBuilder(); str.append(String.format("==== %d outgoing connections ===\n", this.outConnections.size())); for (Map.Entry<RemoteReceiver, Object> entry : this.outConnections.entrySet()) { RemoteReceiver receiver = entry.getKey(); Object value = entry.getValue(); if (value instanceof OutboundConnectionQueue) { OutboundConnectionQueue queue = (OutboundConnectionQueue) value; if (queue.getNumQueuedEnvelopes() > 0) { str.append(String.format("%s> Number of queued envelopes for %s with channel %s: %d\n", Thread.currentThread().getId(), receiver, queue.getChannel(), queue.getNumQueuedEnvelopes())); } } else if (value instanceof ChannelInBuildup) { str.append(String.format("%s> Connection to %s is still in buildup\n", Thread.currentThread().getId(), receiver)); } } return str.toString(); } // ------------------------------------------------------------------------ private static final class ChannelInBuildup implements ChannelFutureListener { private Object lock = new Object(); private volatile OutboundConnectionQueue channel; private volatile Throwable error; private int numRetries = 2; private final Bootstrap out; private final RemoteReceiver receiver; private ChannelInBuildup(Bootstrap out, RemoteReceiver receiver) { this.out = out; this.receiver = receiver; } private void handInChannel(OutboundConnectionQueue c) { synchronized (this.lock) { this.channel = c; this.lock.notifyAll(); } } private void notifyOfError(Throwable error) { synchronized (this.lock) { this.error = error; this.lock.notifyAll(); } } private OutboundConnectionQueue waitForChannel() throws IOException { synchronized (this.lock) { while (this.error == null && this.channel == null) { try { this.lock.wait(2000); } catch (InterruptedException e) { throw new RuntimeException("Channel buildup interrupted."); } } } if (this.error != null) { throw new IOException("Connecting the channel failed: " + error.getMessage(), error); } return this.channel; } @Override public void operationComplete(ChannelFuture future) throws Exception { if (future.isSuccess()) { if (LOG.isDebugEnabled()) { LOG.debug(String.format("Channel %s connected", future.channel())); } handInChannel(new OutboundConnectionQueue(future.channel())); } else if (this.numRetries > 0) { LOG.debug(String.format("Connection request did not succeed, retrying (%d attempts left)", this.numRetries)); this.out.connect(this.receiver.getConnectionAddress()).addListener(this); this.numRetries--; } else { if (future.getClass() != null) { notifyOfError(future.cause()); } else { notifyOfError(new Exception("Connection could not be established.")); } } } } }