Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.taobao.adfs.distributed.rpc; import java.net.Socket; import java.net.InetSocketAddress; import java.net.SocketTimeoutException; import java.net.UnknownHostException; import java.net.ConnectException; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.FilterInputStream; import java.io.InputStream; import java.util.Hashtable; import java.util.Iterator; import java.util.Map.Entry; import java.util.Random; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import javax.net.SocketFactory; import org.apache.commons.logging.*; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.ReflectionUtils; /** * A client for an IPC service. IPC calls take a single {@link Writable} as a parameter, and return a {@link Writable} * as their value. A service runs on a port and is defined by a parameter class and a value class. * * @see Server */ public class Client { public static final Log LOG = LogFactory.getLog(Client.class); static private Random random = new Random(); private Hashtable<ConnectionId, Connection> connections = new Hashtable<ConnectionId, Connection>(); private Class<? extends Writable> valueClass; // class of call values private int counter; // counter for call ids private AtomicBoolean running = new AtomicBoolean(true); // if client runs final private Configuration conf; final private int maxIdleTime; // connections will be culled if it was idle for // maxIdleTime msecs final private int maxRetriesForIOException; // the max. no. of retries for socket connections final private int maxRetriesForSocketTimeoutException; // the max. no. of retries for socket connections private boolean tcpNoDelay; // if T then disable Nagle's Algorithm private int pingInterval; // how often sends ping to the server in msecs private SocketFactory socketFactory; // how to create sockets private int refCount = 1; private int connectionNumberPerAddress = 1; final private static String PING_INTERVAL_NAME = "ipc.ping.interval"; final static int DEFAULT_PING_INTERVAL = 60000; // 1 min final static int PING_CALL_ID = -1; /** * set the ping interval value in configuration * * @param conf * Configuration * @param pingInterval * the ping interval */ final public static void setPingInterval(Configuration conf, int pingInterval) { conf.setInt(PING_INTERVAL_NAME, pingInterval); } /** * Get the ping interval from configuration; If not set in the configuration, return the default value. * * @param conf * Configuration * @return the ping interval */ final static int getPingInterval(Configuration conf) { return conf.getInt(PING_INTERVAL_NAME, DEFAULT_PING_INTERVAL); } /** * Increment this client's reference count * */ synchronized void incCount() { refCount++; } /** * Decrement this client's reference count * */ synchronized void decCount() { refCount--; } /** * Return if this client has no reference * * @return true if this client has no reference; false otherwise */ synchronized boolean isZeroReference() { return refCount == 0; } /** A call waiting for a value. */ private class Call { int id; // call id byte[] resultBytes; Writable param; // parameter Writable value; // value, null if error IOException error; // exception, null if value boolean done; // true when call is done protected Call(Writable param) { this.param = param; synchronized (Client.this) { this.id = counter++; } } /** * Indicate when the call is complete and the value or error are available. Notifies by default. */ protected synchronized void callComplete() { this.done = true; notify(); // notify caller } /** * Set the exception when there is an error. Notify the caller the call is done. * * @param error * exception thrown by the call; either local or remote */ public synchronized void setException(IOException error) { this.error = error; callComplete(); } public synchronized void setResultBytes(byte[] resultBytes) { this.resultBytes = resultBytes; callComplete(); } public void processResult() { if (error != null) return; if (resultBytes == null) return; try { DataInputStream in = new DataInputStream(new ByteArrayInputStream(resultBytes)); boolean isError = in.readBoolean(); // read if error if (isError) error = new RemoteException(WritableUtils.readString(in), WritableUtils.readString(in)); else (value = ReflectionUtils.newInstance(valueClass, conf)).readFields(in); // read value resultBytes = null; in.close(); } catch (Throwable t) { error = new IOException("fail to process result for call.id=" + id + ", call.param=" + param, t); } } } /** * Thread that reads responses and notifies callers. Each connection owns a socket connected to a remote address. * Calls are multiplexed through this socket: responses may be delivered out of order. */ private class Connection extends Thread { private ConnectionId remoteId; private Socket socket = null; // connected socket private DataInputStream in; private DataOutputStream out; // currently active calls private Hashtable<Integer, Call> calls = new Hashtable<Integer, Call>(); private AtomicLong lastActivity = new AtomicLong();// last I/O activity time private AtomicBoolean shouldCloseConnection = new AtomicBoolean(); // indicate if the connection is closed private IOException closeException; // close reason public Connection(ConnectionId remoteId) throws IOException { if (remoteId.getAddress().isUnresolved()) { throw new UnknownHostException("unknown host: " + remoteId.getAddress().getHostName()); } this.remoteId = remoteId; this.setName("IPC Client (" + socketFactory.hashCode() + ") connection to " + remoteId.getAddress().toString() + "[" + remoteId.getOffset() + "/" + connectionNumberPerAddress + "]"); this.setDaemon(true); } /** Update lastActivity with the current time. */ private void touch() { lastActivity.set(System.currentTimeMillis()); } /** * Add a call to this connection's call queue and notify a listener; synchronized. Returns false if called during * shutdown. * * @param call * to add * @return true if the call was added. */ private synchronized boolean addCall(Call call) { if (shouldCloseConnection.get()) return false; calls.put(call.id, call); notify(); return true; } /** * This class sends a ping to the remote side when timeout on reading. If no failure is detected, it retries until * at least a byte is read. */ private class PingInputStream extends FilterInputStream { /* constructor */ protected PingInputStream(InputStream in) { super(in); } /* * Process timeout exception if the connection is not going to be closed, send a ping. otherwise, throw the * timeout exception. */ private void handleTimeout(SocketTimeoutException e) throws IOException { if (shouldCloseConnection.get() || !running.get()) { throw e; } else { sendPing(); } } /** * Read a byte from the stream. Send a ping if timeout on read. Retries if no failure is detected until a byte is * read. * * @throws IOException * for any IO problem other than socket timeout */ public int read() throws IOException { do { try { return super.read(); } catch (SocketTimeoutException e) { handleTimeout(e); } } while (true); } /** * Read bytes into a buffer starting from offset <code>off</code> Send a ping if timeout on read. Retries if no * failure is detected until a byte is read. * * @return the total number of bytes read; -1 if the connection is closed. */ public int read(byte[] buf, int off, int len) throws IOException { do { try { return super.read(buf, off, len); } catch (SocketTimeoutException e) { handleTimeout(e); } } while (true); } } /** * Connect to the server and set up the I/O streams. It then sends a header to the server and starts the connection * thread that waits for responses. */ private synchronized void setupIOstreams() { if (socket != null || shouldCloseConnection.get()) { return; } short ioFailures = 0; short timeoutFailures = 0; try { if (LOG.isDebugEnabled()) { LOG.debug("Connecting to " + remoteId.getAddress()); } while (true) { try { this.socket = socketFactory.createSocket(); this.socket.setTcpNoDelay(tcpNoDelay); // connection time out is 20s this.socket.connect(remoteId.getAddress(), 20000); this.socket.setSoTimeout(pingInterval); break; } catch (SocketTimeoutException toe) { /* * The max number of retries is 45, which amounts to 20s*45 = 15 minutes retries. */ handleConnectionFailure(timeoutFailures++, maxRetriesForSocketTimeoutException, toe); } catch (IOException ie) { handleConnectionFailure(ioFailures++, maxRetriesForIOException, ie); } } this.in = new DataInputStream( new BufferedInputStream(new PingInputStream(NetUtils.getInputStream(socket)))); this.out = new DataOutputStream(new BufferedOutputStream(NetUtils.getOutputStream(socket))); writeHeader(); // update last activity time touch(); // start the receiver thread after the socket connection has been set up start(); } catch (IOException e) { markClosed(e); close(); } } /* * Handle connection failures * * If the current number of retries is equal to the max number of retries, stop retrying and throw the exception; * Otherwise backoff 1 second and try connecting again. * * This Method is only called from inside setupIOstreams(), which is synchronized. Hence the sleep is synchronized; * the locks will be retained. * * @param curRetries current number of retries * * @param maxRetries max number of retries allowed * * @param ioe failure reason * * @throws IOException if max number of retries is reached */ private void handleConnectionFailure(int curRetries, int maxRetries, IOException ioe) throws IOException { // close the current connection try { socket.close(); } catch (IOException e) { LOG.warn("Not able to close a socket", e); } // set socket to null so that the next call to setupIOstreams // can start the process of connect all over again. socket = null; // throw the exception if the maximum number of retries is reached if (curRetries >= maxRetries) { throw ioe; } // otherwise back off and retry try { Thread.sleep(1000); } catch (InterruptedException ignored) { } LOG.info("Retrying connect to server: " + remoteId.getAddress() + ". Already tried " + curRetries + " time(s)."); } /* * Write the header for each connection Out is not synchronized because only the first thread does this. */ private void writeHeader() throws IOException { out.write(Server.HEADER.array()); out.write(Server.CURRENT_VERSION); // When there are more fields we can have ConnectionHeader Writable. DataOutputBuffer buf = new DataOutputBuffer(); int bufLen = buf.getLength(); out.writeInt(bufLen); out.write(buf.getData(), 0, bufLen); } /* * wait till someone signals us to start reading RPC response or it is idle too long, it is marked as to be closed, * or the client is marked as not running. * * Return true if it is time to read a response; false otherwise. */ private synchronized boolean waitForWork() { if (calls.isEmpty() && !shouldCloseConnection.get() && running.get()) { long timeout = maxIdleTime - (System.currentTimeMillis() - lastActivity.get()); if (timeout > 0) { try { wait(timeout); } catch (InterruptedException e) { } } } if (!calls.isEmpty() && !shouldCloseConnection.get() && running.get()) { return true; } else if (shouldCloseConnection.get()) { return false; } else if (calls.isEmpty()) { // idle connection closed or stopped markClosed(null); return false; } else { // get stopped but there are still pending requests markClosed((IOException) new IOException().initCause(new InterruptedException())); return false; } } @SuppressWarnings("unused") public InetSocketAddress getRemoteAddress() { return remoteId.getAddress(); } /* * Send a ping to the server if the time elapsed since last I/O activity is equal to or greater than the ping * interval */ private synchronized void sendPing() throws IOException { long curTime = System.currentTimeMillis(); if (curTime - lastActivity.get() >= pingInterval) { lastActivity.set(curTime); synchronized (out) { out.writeInt(PING_CALL_ID); out.flush(); } } } public void run() { if (LOG.isDebugEnabled()) LOG.debug(getName() + ": starting, having connections " + connections.size()); while (waitForWork()) {// wait here for work - read or close connection receiveResponse(); } close(); if (LOG.isDebugEnabled()) LOG.debug(getName() + ": stopped, remaining connections " + connections.size()); } /** * Initiates a call by sending the parameter to the remote server. Note: this is not called from the Connection * thread, but by other threads. */ public void sendParam(Call call) { if (shouldCloseConnection.get()) { return; } DataOutputBuffer d = null; try { // for serializing the data to be written d = new DataOutputBuffer(); d.writeInt(call.id); call.param.write(d); byte[] data = d.getData(); int dataLength = d.getLength(); if (LOG.isDebugEnabled()) LOG.debug(getName() + " sending #" + call.id); synchronized (this.out) { out.writeInt(dataLength); // first put the data length out.write(data, 0, dataLength);// write the data out.flush(); } if (LOG.isDebugEnabled()) LOG.debug(getName() + " has sent #" + call.id); } catch (IOException e) { markClosed(e); } finally { // the buffer is just an in-memory buffer, but it is still polite to // close early IOUtils.closeStream(d); } } /* * Receive a response. Because only one receiver, so no synchronization on in. */ private void receiveResponse() { if (shouldCloseConnection.get()) { return; } touch(); try { int id = in.readInt(); // try to read an id if (LOG.isDebugEnabled()) LOG.debug(getName() + " got value #" + id); // read length int resultLength = in.readInt(); byte[] resultBytes = new byte[resultLength]; in.readFully(resultBytes);// note: DO NOT user in.read(resultBytes); // from patch HADOOP-6498 Call call = calls.get(id); call.setResultBytes(resultBytes); // from patch HADOOP-6498 calls.remove(id); } catch (IOException e) { markClosed(e); } } private synchronized void markClosed(IOException e) { if (shouldCloseConnection.compareAndSet(false, true)) { closeException = e; notifyAll(); } } /** Close the connection. */ private synchronized void close() { if (!shouldCloseConnection.get()) { LOG.error("The connection is not in the closed state"); return; } // release the resources // first thing to do;take the connection out of the connection list synchronized (connections) { if (connections.get(remoteId) == this) { connections.remove(remoteId); } } // close the streams and therefore the socket IOUtils.closeStream(out); IOUtils.closeStream(in); // clean up all calls if (closeException == null) { if (!calls.isEmpty()) { LOG.warn("A connection is closed for no cause and calls are not empty"); // clean up calls anyway closeException = new IOException("Unexpected closed connection"); cleanupCalls(); } } else { // log the info if (LOG.isDebugEnabled()) { LOG.debug("closing ipc connection to " + remoteId.address + ": " + closeException.getMessage(), closeException); } // cleanup calls cleanupCalls(); } if (LOG.isDebugEnabled()) LOG.debug(getName() + ": closed"); } /* Cleanup all calls and mark them as done */ private void cleanupCalls() { Iterator<Entry<Integer, Call>> itor = calls.entrySet().iterator(); while (itor.hasNext()) { Call c = itor.next().getValue(); c.setException(closeException); // local exception itor.remove(); } } } /** Call implementation used for parallel calls. */ private class ParallelCall extends Call { private ParallelResults results; private int index; public ParallelCall(Writable param, ParallelResults results, int index) { super(param); this.results = results; this.index = index; } /** Deliver result to result collector. */ protected void callComplete() { results.callComplete(this); } } /** Result collector for parallel calls. */ private static class ParallelResults { private Writable[] values; private int size; private int count; public ParallelResults(int size) { this.values = new Writable[size]; this.size = size; } /** Collect a result. */ public synchronized void callComplete(ParallelCall call) { values[call.index] = call.value; // store the value count++; // count it if (count == size) // if all values are in notify(); // then notify waiting caller } } /** * Construct an IPC client whose values are of the given {@link Writable} class. */ public Client(Class<? extends Writable> valueClass, Configuration conf, SocketFactory factory) { this.valueClass = valueClass; this.maxIdleTime = conf.getInt("ipc.client.connection.maxidletime", 3600000); // 3600s this.maxRetriesForIOException = conf.getInt("ipc.client.connect.max.retries.ioexception", 0); this.maxRetriesForSocketTimeoutException = conf .getInt("ipc.client.connect.max.retries.sockettimeoutexception", 0); this.tcpNoDelay = conf.getBoolean("ipc.client.tcpnodelay", true); this.pingInterval = getPingInterval(conf); if (LOG.isDebugEnabled()) { LOG.debug("The ping interval is" + this.pingInterval + "ms."); } this.conf = conf; this.socketFactory = factory; this.connectionNumberPerAddress = conf.getInt("ipc.client.connection.number.per.address", 1); if (LOG.isDebugEnabled()) { LOG.debug("ipc.client.connection.number.per.address=" + connectionNumberPerAddress); } } /** * Construct an IPC client with the default SocketFactory * * @param valueClass * @param conf */ public Client(Class<? extends Writable> valueClass, Configuration conf) { this(valueClass, conf, NetUtils.getDefaultSocketFactory(conf)); } /** * Return the socket factory of this client * * @return this client's socket factory */ SocketFactory getSocketFactory() { return socketFactory; } /** * Stop all threads related to this client. No further calls may be made using this client. */ public void stop() { if (LOG.isDebugEnabled()) { LOG.debug("Stopping client"); } if (!running.compareAndSet(true, false)) { return; } // wake up all connections synchronized (connections) { for (Connection conn : connections.values()) { conn.interrupt(); } } // wait until all connections are closed while (!connections.isEmpty()) { try { Thread.sleep(100); } catch (InterruptedException e) { } } } /** * Make a call, passing <code>param</code>, to the IPC server running at <code>address</code>, returning the value. * Throws exceptions if there are network problems or if the remote code threw an exception. */ public Writable call(Writable param, InetSocketAddress addr) throws InterruptedException, IOException { try { return callInternal(param, addr); } catch (IOException e) { if (e instanceof RemoteException) throw e; // retry when failed for connection problem: e.g. remote server has restart // reason is that RPC has cached the old incorrect connection return callInternal(param, addr); } } private Writable callInternal(Writable param, InetSocketAddress addr) throws InterruptedException, IOException { Call call = new Call(param); Connection connection = getConnection(addr, call, getConnectionOffset()); connection.sendParam(call); // send the parameter synchronized (call) { while (!call.done) { try { call.wait(); // wait for the result } catch (InterruptedException ignored) { } } call.processResult(); if (call.error != null) { if (call.error instanceof RemoteException) { call.error.fillInStackTrace(); throw call.error; } else { // local exception throw wrapException(addr, call.error); } } else { return call.value; } } } /** * Take an IOException and the address we were trying to connect to and return an IOException with the input exception * as the cause. The new exception provides the stack trace of the place where the exception is thrown and some extra * diagnostics information. If the exception is ConnectException or SocketTimeoutException, return a new one of the * same type; Otherwise return an IOException. * * @param addr * target address * @param exception * the relevant exception * @return an exception to throw */ private IOException wrapException(InetSocketAddress addr, IOException exception) { if (exception instanceof ConnectException) { // connection refused; include the host:port in the error return (ConnectException) new ConnectException( "Call to " + addr + " failed on connection exception: " + exception).initCause(exception); } else if (exception instanceof SocketTimeoutException) { return (SocketTimeoutException) new SocketTimeoutException( "Call to " + addr + " failed on socket timeout exception: " + exception).initCause(exception); } else { return (IOException) new IOException("Call to " + addr + " failed on local exception: " + exception) .initCause(exception); } } /** * Makes a set of calls in parallel. Each parameter is sent to the corresponding address. When all values are * available, or have timed out or errored, the collected results are returned in an array. The array contains nulls * for calls that timed out or errored. */ public Writable[] call(Writable[] params, InetSocketAddress[] addresses) throws IOException { if (addresses.length == 0) return new Writable[0]; ParallelResults results = new ParallelResults(params.length); synchronized (results) { for (int i = 0; i < params.length; i++) { ParallelCall call = new ParallelCall(params[i], results, i); try { Connection connection = getConnection(addresses[i], call, getConnectionOffset()); connection.sendParam(call); // send each parameter } catch (IOException e) { // log errors LOG.info("Calling " + addresses[i] + " caught: " + e.getMessage(), e); results.size--; // wait for one fewer result } } while (results.count != results.size) { try { results.wait(); // wait for all results } catch (InterruptedException e) { } } return results.values; } } int getConnectionOffset() { return random.nextInt(connectionNumberPerAddress); } /** * Get a connection from the pool, or create a new one and add it to the pool. Connections to a given host/port are * reused. */ private Connection getConnection(InetSocketAddress addr, Call call, int offset) throws IOException { if (!running.get()) { // the client is stopped throw new IOException("The client is stopped"); } Connection connection; /* * we could avoid this allocation for each RPC by having a connectionsId object and with set() method. We need to * manage the refs for keys in HashMap properly. For now its ok. */ ConnectionId remoteId = new ConnectionId(addr, offset); do { synchronized (connections) { connection = connections.get(remoteId); if (connection == null) { connection = new Connection(remoteId); connections.put(remoteId, connection); } } } while (!connection.addCall(call)); // we don't invoke the method below inside "synchronized (connections)" // block above. The reason for that is if the server happens to be slow, // it will take longer to establish a connection and that will slow the // entire system down. connection.setupIOstreams(); return connection; } /** * This class holds the address and the user ticket. The client connections to servers are uniquely identified by * <remoteAddress, ticket> */ private static class ConnectionId { InetSocketAddress address; Integer offset = 0; ConnectionId(InetSocketAddress address, int offset) { this.address = address; this.offset = offset; } InetSocketAddress getAddress() { return address; } int getOffset() { return offset; } @Override public boolean equals(Object obj) { if (obj instanceof ConnectionId) { ConnectionId id = (ConnectionId) obj; return address.equals(id.address) && offset == id.offset; } return false; } @Override public int hashCode() { return address.hashCode() ^ offset.hashCode(); } } }