org.apache.hadoop.io.retry.RetryPolicies.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.io.retry.RetryPolicies.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.io.retry;

import java.io.EOFException;
import java.io.IOException;
import java.net.ConnectException;
import java.net.NoRouteToHostException;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.ipc.NotALeaderException;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.net.ConnectTimeoutException;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;

import com.google.common.annotations.VisibleForTesting;

/**
 * <p>
 * A collection of useful implementations of {@link RetryPolicy}.
 * </p>
 */
public class RetryPolicies {

    public static final Log LOG = LogFactory.getLog(RetryPolicies.class);

    /**
     * <p>
     * Try once, and fail by re-throwing the exception.
     * This corresponds to having no retry mechanism in place.
     * </p>
     */
    public static final RetryPolicy TRY_ONCE_THEN_FAIL = new TryOnceThenFail();

    /**
     * <p>
     * Keep trying forever.
     * </p>
     */
    public static final RetryPolicy RETRY_FOREVER = new RetryForever();

    /**
     * <p>
     * Keep trying forever with a fixed time between attempts.
     * </p>
     */
    public static final RetryPolicy retryForeverWithFixedSleep(long sleepTime, TimeUnit timeUnit) {
        return new RetryUpToMaximumCountWithFixedSleep(Integer.MAX_VALUE, sleepTime, timeUnit);
    }

    /**
     * <p>
     * Keep trying a limited number of times, waiting a fixed time between attempts,
     * and then fail by re-throwing the exception.
     * </p>
     */
    public static final RetryPolicy retryUpToMaximumCountWithFixedSleep(int maxRetries, long sleepTime,
            TimeUnit timeUnit) {
        return new RetryUpToMaximumCountWithFixedSleep(maxRetries, sleepTime, timeUnit);
    }

    /**
     * <p>
     * Keep trying for a maximum time, waiting a fixed time between attempts,
     * and then fail by re-throwing the exception.
     * </p>
     */
    public static final RetryPolicy retryUpToMaximumTimeWithFixedSleep(long maxTime, long sleepTime,
            TimeUnit timeUnit) {
        return new RetryUpToMaximumTimeWithFixedSleep(maxTime, sleepTime, timeUnit);
    }

    /**
     * <p>
     * Keep trying a limited number of times, waiting a growing amount of time between attempts,
     * and then fail by re-throwing the exception.
     * The time between attempts is <code>sleepTime</code> mutliplied by the number of tries so far.
     * </p>
     */
    public static final RetryPolicy retryUpToMaximumCountWithProportionalSleep(int maxRetries, long sleepTime,
            TimeUnit timeUnit) {
        return new RetryUpToMaximumCountWithProportionalSleep(maxRetries, sleepTime, timeUnit);
    }

    /**
     * <p>
     * Keep trying a limited number of times, waiting a growing amount of time between attempts,
     * and then fail by re-throwing the exception.
     * The time between attempts is <code>sleepTime</code> mutliplied by a random
     * number in the range of [0, 2 to the number of retries)
     * </p>
     */
    public static final RetryPolicy exponentialBackoffRetry(int maxRetries, long sleepTime, TimeUnit timeUnit) {
        return new ExponentialBackoffRetry(maxRetries, sleepTime, timeUnit);
    }

    /**
     * <p>
     * Set a default policy with some explicit handlers for specific exceptions.
     * </p>
     */
    public static final RetryPolicy retryByException(RetryPolicy defaultPolicy,
            Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
        return new ExceptionDependentRetry(defaultPolicy, exceptionToPolicyMap);
    }

    /**
     * <p>
     * A retry policy for RemoteException
     * Set a default policy with some explicit handlers for specific exceptions.
     * </p>
     */
    public static final RetryPolicy retryByRemoteException(RetryPolicy defaultPolicy,
            Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
        return new RemoteExceptionDependentRetry(defaultPolicy, exceptionToPolicyMap);
    }

    /**
     * A retry policy for exceptions other than RemoteException.
     */
    public static final RetryPolicy retryOtherThanRemoteException(RetryPolicy defaultPolicy,
            Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
        return new OtherThanRemoteExceptionDependentRetry(defaultPolicy, exceptionToPolicyMap);
    }

    public static final RetryPolicy failoverOnNetworkException(int maxFailovers) {
        return failoverOnNetworkException(TRY_ONCE_THEN_FAIL, maxFailovers);
    }

    public static final RetryPolicy failoverOnNetworkException(RetryPolicy fallbackPolicy, int maxFailovers) {
        return failoverOnNetworkException(fallbackPolicy, maxFailovers, 0, 0);
    }

    public static final RetryPolicy failoverOnNetworkException(RetryPolicy fallbackPolicy, int maxFailovers,
            long delayMillis, long maxDelayBase) {
        return new FailoverOnNetworkExceptionRetry(fallbackPolicy, maxFailovers, delayMillis, maxDelayBase);
    }

    public static final RetryPolicy failoverOnNetworkException(RetryPolicy fallbackPolicy, int maxFailovers,
            int maxRetries, long delayMillis, long maxDelayBase) {
        return new FailoverOnNetworkExceptionRetry(fallbackPolicy, maxFailovers, maxRetries, delayMillis,
                maxDelayBase, null);
    }

    public static final RetryPolicy failoverOnNetworkException(RetryPolicy fallbackPolicy, int maxFailovers,
            int maxRetries, long delayMillis, long maxDelayBase,
            Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
        return new FailoverOnNetworkExceptionRetry(fallbackPolicy, maxFailovers, maxRetries, delayMillis,
                maxDelayBase, exceptionToPolicyMap);
    }

    public static final RetryPolicy failoverOnLeaderChange(RetryPolicy fallbackPolicy, int maxFailovers,
            int maxRetries, long delayMillis, long maxDelayBase) {
        return new FailoverOnLeaderChange(fallbackPolicy, maxFailovers, maxRetries, delayMillis, maxDelayBase,
                null);
    }

    public static final RetryPolicy failoverOnLeaderChange(RetryPolicy fallbackPolicy, int maxFailovers,
            int maxRetries, long delayMillis, long maxDelayBase,
            Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
        return new FailoverOnLeaderChange(fallbackPolicy, maxFailovers, maxRetries, delayMillis, maxDelayBase,
                exceptionToPolicyMap);
    }

    static class TryOnceThenFail implements RetryPolicy {
        @Override
        public RetryAction shouldRetry(Exception e, int retries, int failovers, boolean isIdempotentOrAtMostOnce)
                throws Exception {
            return new RetryAction(RetryAction.RetryDecision.FAIL, 0, "try once " + "and fail.");
        }

        @Override
        public boolean equals(Object obj) {
            if (obj == this) {
                return true;
            } else {
                return obj != null && obj.getClass() == this.getClass();
            }
        }

        @Override
        public int hashCode() {
            return this.getClass().hashCode();
        }
    }

    static class RetryForever implements RetryPolicy {
        @Override
        public RetryAction shouldRetry(Exception e, int retries, int failovers, boolean isIdempotentOrAtMostOnce)
                throws Exception {
            return RetryAction.RETRY;
        }
    }

    /**
     * Retry up to maxRetries.
     * The actual sleep time of the n-th retry is f(n, sleepTime),
     * where f is a function provided by the subclass implementation.
     *
     * The object of the subclasses should be immutable;
     * otherwise, the subclass must override hashCode(), equals(..) and toString().
     */
    static abstract class RetryLimited implements RetryPolicy {
        final int maxRetries;
        final long sleepTime;
        final TimeUnit timeUnit;

        private String myString;

        RetryLimited(int maxRetries, long sleepTime, TimeUnit timeUnit) {
            if (maxRetries < 0) {
                throw new IllegalArgumentException("maxRetries = " + maxRetries + " < 0");
            }
            if (sleepTime < 0) {
                throw new IllegalArgumentException("sleepTime = " + sleepTime + " < 0");
            }

            this.maxRetries = maxRetries;
            this.sleepTime = sleepTime;
            this.timeUnit = timeUnit;
        }

        @Override
        public RetryAction shouldRetry(Exception e, int retries, int failovers, boolean isIdempotentOrAtMostOnce)
                throws Exception {
            if (retries >= maxRetries) {
                return new RetryAction(RetryAction.RetryDecision.FAIL, 0, getReason());
            }
            return new RetryAction(RetryAction.RetryDecision.RETRY, timeUnit.toMillis(calculateSleepTime(retries)),
                    getReason());
        }

        protected String getReason() {
            return constructReasonString(maxRetries);
        }

        @VisibleForTesting
        public static String constructReasonString(int retries) {
            return "retries get failed due to exceeded maximum allowed retries " + "number: " + retries;
        }

        protected abstract long calculateSleepTime(int retries);

        @Override
        public int hashCode() {
            return toString().hashCode();
        }

        @Override
        public boolean equals(final Object that) {
            if (this == that) {
                return true;
            } else if (that == null || this.getClass() != that.getClass()) {
                return false;
            }
            return this.toString().equals(that.toString());
        }

        @Override
        public String toString() {
            if (myString == null) {
                myString = getClass().getSimpleName() + "(maxRetries=" + maxRetries + ", sleepTime=" + sleepTime
                        + " " + timeUnit + ")";
            }
            return myString;
        }
    }

    static class RetryUpToMaximumCountWithFixedSleep extends RetryLimited {
        public RetryUpToMaximumCountWithFixedSleep(int maxRetries, long sleepTime, TimeUnit timeUnit) {
            super(maxRetries, sleepTime, timeUnit);
        }

        @Override
        protected long calculateSleepTime(int retries) {
            return sleepTime;
        }
    }

    static class RetryUpToMaximumTimeWithFixedSleep extends RetryUpToMaximumCountWithFixedSleep {
        private long maxTime = 0;
        private TimeUnit timeUnit;

        public RetryUpToMaximumTimeWithFixedSleep(long maxTime, long sleepTime, TimeUnit timeUnit) {
            super((int) (maxTime / sleepTime), sleepTime, timeUnit);
            this.maxTime = maxTime;
            this.timeUnit = timeUnit;
        }

        @Override
        protected String getReason() {
            return constructReasonString(this.maxTime, this.timeUnit);
        }

        @VisibleForTesting
        public static String constructReasonString(long maxTime, TimeUnit timeUnit) {
            return "retries get failed due to exceeded maximum allowed time (" + "in " + timeUnit.toString() + "): "
                    + maxTime;
        }
    }

    static class RetryUpToMaximumCountWithProportionalSleep extends RetryLimited {
        public RetryUpToMaximumCountWithProportionalSleep(int maxRetries, long sleepTime, TimeUnit timeUnit) {
            super(maxRetries, sleepTime, timeUnit);
        }

        @Override
        protected long calculateSleepTime(int retries) {
            return sleepTime * (retries + 1);
        }
    }

    /**
     * Given pairs of number of retries and sleep time (n0, t0), (n1, t1), ...,
     * the first n0 retries sleep t0 milliseconds on average,
     * the following n1 retries sleep t1 milliseconds on average, and so on.
     * 
     * For all the sleep, the actual sleep time is randomly uniform distributed
     * in the close interval [0.5t, 1.5t], where t is the sleep time specified.
     *
     * The objects of this class are immutable.
     */
    public static class MultipleLinearRandomRetry implements RetryPolicy {
        /** Pairs of numRetries and sleepSeconds */
        public static class Pair {
            final int numRetries;
            final int sleepMillis;

            public Pair(final int numRetries, final int sleepMillis) {
                if (numRetries < 0) {
                    throw new IllegalArgumentException("numRetries = " + numRetries + " < 0");
                }
                if (sleepMillis < 0) {
                    throw new IllegalArgumentException("sleepMillis = " + sleepMillis + " < 0");
                }

                this.numRetries = numRetries;
                this.sleepMillis = sleepMillis;
            }

            @Override
            public String toString() {
                return numRetries + "x" + sleepMillis + "ms";
            }
        }

        private final List<Pair> pairs;
        private String myString;

        public MultipleLinearRandomRetry(List<Pair> pairs) {
            if (pairs == null || pairs.isEmpty()) {
                throw new IllegalArgumentException("pairs must be neither null nor empty.");
            }
            this.pairs = Collections.unmodifiableList(pairs);
        }

        @Override
        public RetryAction shouldRetry(Exception e, int curRetry, int failovers, boolean isIdempotentOrAtMostOnce)
                throws Exception {
            final Pair p = searchPair(curRetry);
            if (p == null) {
                //no more retries.
                return new RetryAction(RetryAction.RetryDecision.FAIL, 0,
                        "Retry " + "all pairs in MultipleLinearRandomRetry: " + pairs);
            }

            //calculate sleep time and return.
            // ensure 0.5 <= ratio <=1.5
            final double ratio = ThreadLocalRandom.current().nextDouble() + 0.5;
            final long sleepTime = Math.round(p.sleepMillis * ratio);
            return new RetryAction(RetryAction.RetryDecision.RETRY, sleepTime);
        }

        /**
         * Given the current number of retry, search the corresponding pair.
         * @return the corresponding pair,
         *   or null if the current number of retry > maximum number of retry. 
         */
        private Pair searchPair(int curRetry) {
            int i = 0;
            for (; i < pairs.size() && curRetry > pairs.get(i).numRetries; i++) {
                curRetry -= pairs.get(i).numRetries;
            }
            return i == pairs.size() ? null : pairs.get(i);
        }

        @Override
        public int hashCode() {
            return toString().hashCode();
        }

        @Override
        public boolean equals(final Object that) {
            if (this == that) {
                return true;
            } else if (that == null || this.getClass() != that.getClass()) {
                return false;
            }
            return this.toString().equals(that.toString());
        }

        @Override
        public String toString() {
            if (myString == null) {
                myString = getClass().getSimpleName() + pairs;
            }
            return myString;
        }

        /**
         * Parse the given string as a MultipleLinearRandomRetry object.
         * The format of the string is "t_1, n_1, t_2, n_2, ...",
         * where t_i and n_i are the i-th pair of sleep time and number of retries.
         * Note that the white spaces in the string are ignored.
         *
         * @return the parsed object, or null if the parsing fails.
         */
        public static MultipleLinearRandomRetry parseCommaSeparatedString(String s) {
            final String[] elements = s.split(",");
            if (elements.length == 0) {
                LOG.warn("Illegal value: there is no element in \"" + s + "\".");
                return null;
            }
            if (elements.length % 2 != 0) {
                LOG.warn("Illegal value: the number of elements in \"" + s + "\" is " + elements.length
                        + " but an even number of elements is expected.");
                return null;
            }

            final List<RetryPolicies.MultipleLinearRandomRetry.Pair> pairs = new ArrayList<RetryPolicies.MultipleLinearRandomRetry.Pair>();

            for (int i = 0; i < elements.length;) {
                //parse the i-th sleep-time
                final int sleep = parsePositiveInt(elements, i++, s);
                if (sleep == -1) {
                    return null; //parse fails
                }

                //parse the i-th number-of-retries
                final int retries = parsePositiveInt(elements, i++, s);
                if (retries == -1) {
                    return null; //parse fails
                }

                pairs.add(new RetryPolicies.MultipleLinearRandomRetry.Pair(retries, sleep));
            }
            return new RetryPolicies.MultipleLinearRandomRetry(pairs);
        }

        /**
         * Parse the i-th element as an integer.
         * @return -1 if the parsing fails or the parsed value <= 0;
         *   otherwise, return the parsed value.
         */
        private static int parsePositiveInt(final String[] elements, final int i, final String originalString) {
            final String s = elements[i].trim();
            final int n;
            try {
                n = Integer.parseInt(s);
            } catch (NumberFormatException nfe) {
                LOG.warn("Failed to parse \"" + s + "\", which is the index " + i + " element in \""
                        + originalString + "\"", nfe);
                return -1;
            }

            if (n <= 0) {
                LOG.warn("The value " + n + " <= 0: it is parsed from the string \"" + s + "\" which is the index "
                        + i + " element in \"" + originalString + "\"");
                return -1;
            }
            return n;
        }
    }

    static class ExceptionDependentRetry implements RetryPolicy {

        RetryPolicy defaultPolicy;
        Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap;

        public ExceptionDependentRetry(RetryPolicy defaultPolicy,
                Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
            this.defaultPolicy = defaultPolicy;
            this.exceptionToPolicyMap = exceptionToPolicyMap;
        }

        @Override
        public RetryAction shouldRetry(Exception e, int retries, int failovers, boolean isIdempotentOrAtMostOnce)
                throws Exception {
            RetryPolicy policy = exceptionToPolicyMap.get(e.getClass());
            if (policy == null) {
                policy = defaultPolicy;
            }
            return policy.shouldRetry(e, retries, failovers, isIdempotentOrAtMostOnce);
        }

    }

    static class RemoteExceptionDependentRetry implements RetryPolicy {

        RetryPolicy defaultPolicy;
        Map<String, RetryPolicy> exceptionNameToPolicyMap;

        public RemoteExceptionDependentRetry(RetryPolicy defaultPolicy,
                Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
            this.defaultPolicy = defaultPolicy;
            this.exceptionNameToPolicyMap = new HashMap<String, RetryPolicy>();
            for (Entry<Class<? extends Exception>, RetryPolicy> e : exceptionToPolicyMap.entrySet()) {
                exceptionNameToPolicyMap.put(e.getKey().getName(), e.getValue());
            }
        }

        @Override
        public RetryAction shouldRetry(Exception e, int retries, int failovers, boolean isIdempotentOrAtMostOnce)
                throws Exception {
            RetryPolicy policy = null;
            if (e instanceof RemoteException) {
                policy = exceptionNameToPolicyMap.get(((RemoteException) e).getClassName());
            }
            if (policy == null) {
                policy = defaultPolicy;
            }
            return policy.shouldRetry(e, retries, failovers, isIdempotentOrAtMostOnce);
        }
    }

    static class OtherThanRemoteExceptionDependentRetry implements RetryPolicy {

        private RetryPolicy defaultPolicy;
        private Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap;

        public OtherThanRemoteExceptionDependentRetry(RetryPolicy defaultPolicy,
                Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
            this.defaultPolicy = defaultPolicy;
            this.exceptionToPolicyMap = exceptionToPolicyMap;
        }

        @Override
        public RetryAction shouldRetry(Exception e, int retries, int failovers, boolean isIdempotentOrAtMostOnce)
                throws Exception {
            RetryPolicy policy = null;
            // ignore Remote Exception
            if (e instanceof RemoteException) {
                // do nothing
            } else {
                policy = exceptionToPolicyMap.get(e.getClass());
            }
            if (policy == null) {
                policy = defaultPolicy;
            }
            return policy.shouldRetry(e, retries, failovers, isIdempotentOrAtMostOnce);
        }
    }

    static class ExponentialBackoffRetry extends RetryLimited {

        public ExponentialBackoffRetry(int maxRetries, long sleepTime, TimeUnit timeUnit) {
            super(maxRetries, sleepTime, timeUnit);

            if (maxRetries < 0) {
                throw new IllegalArgumentException("maxRetries = " + maxRetries + " < 0");
            } else if (maxRetries >= Long.SIZE - 1) {
                //calculateSleepTime may overflow. 
                throw new IllegalArgumentException("maxRetries = " + maxRetries + " >= " + (Long.SIZE - 1));
            }
        }

        @Override
        protected long calculateSleepTime(int retries) {
            return calculateExponentialTime(sleepTime, retries + 1);
        }

    }

    /**
     * Fail over and retry in the case of:
     *   Remote StandbyException (server is up, but is not the active server)
     *   Immediate socket exceptions (e.g. no route to host, econnrefused)
     *   Socket exceptions after initial connection when operation is idempotent
     * 
     * The first failover is immediate, while all subsequent failovers wait an
     * exponentially-increasing random amount of time.
     * 
     * Fail immediately in the case of:
     *   Socket exceptions after initial connection when operation is not idempotent
     * 
     * Fall back on underlying retry policy otherwise.
     */
    static class FailoverOnNetworkExceptionRetry implements RetryPolicy {

        protected RetryPolicy fallbackPolicy;
        protected int maxFailovers;
        protected int maxRetries;
        protected long delayMillis;
        protected long maxDelayBase;
        protected Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap;

        public FailoverOnNetworkExceptionRetry(RetryPolicy fallbackPolicy, int maxFailovers) {
            this(fallbackPolicy, maxFailovers, 0, 0, 0, null);
        }

        public FailoverOnNetworkExceptionRetry(RetryPolicy fallbackPolicy, int maxFailovers, long delayMillis,
                long maxDelayBase) {
            this(fallbackPolicy, maxFailovers, 0, delayMillis, maxDelayBase, null);
        }

        public FailoverOnNetworkExceptionRetry(RetryPolicy fallbackPolicy, int maxFailovers, int maxRetries,
                long delayMillis, long maxDelayBase,
                Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
            this.fallbackPolicy = fallbackPolicy;
            this.maxFailovers = maxFailovers;
            this.maxRetries = maxRetries;
            this.delayMillis = delayMillis;
            this.maxDelayBase = maxDelayBase;
            if (exceptionToPolicyMap != null) {
                this.exceptionToPolicyMap = exceptionToPolicyMap;
            } else {
                this.exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();
            }
        }

        /**
         * @return 0 if this is our first failover/retry (i.e., retry immediately),
         *         sleep exponentially otherwise
         */
        protected long getFailoverOrRetrySleepTime(int times) {
            return times == 0 ? 0 : calculateExponentialTime(delayMillis, times, maxDelayBase);
        }

        @Override
        public RetryAction shouldRetry(Exception e, int retries, int failovers, boolean isIdempotentOrAtMostOnce)
                throws Exception {

            RetryAction action = checkRetryCount(e, retries, failovers);
            if (action != null) {
                return action;
            }

            for (Class wrappedException : exceptionToPolicyMap.keySet()) {
                if (wrappsException(e, wrappedException)) {
                    RetryPolicy policy = exceptionToPolicyMap.get(wrappedException);
                    if (policy != null) {
                        return policy.shouldRetry(e, retries, failovers, isIdempotentOrAtMostOnce);
                    }
                }
            }

            if (e instanceof ConnectException || e instanceof EOFException || e instanceof NoRouteToHostException
                    || e instanceof UnknownHostException || e instanceof StandbyException
                    || e instanceof ConnectTimeoutException || isWrappedStandbyException(e)) {
                return new RetryAction(RetryAction.RetryDecision.FAILOVER_AND_RETRY,
                        getFailoverOrRetrySleepTime(failovers));
            } else if (e instanceof RetriableException || getWrappedRetriableException(e) != null) {
                // RetriableException or RetriableException wrapped 
                return new RetryAction(RetryAction.RetryDecision.RETRY, getFailoverOrRetrySleepTime(retries));
            } else if (e instanceof InvalidToken) {
                return new RetryAction(RetryAction.RetryDecision.FAIL, 0, "Invalid or Cancelled Token");
            } else if (e instanceof SocketException
                    || (e instanceof IOException && !(e instanceof RemoteException))) {
                if (isIdempotentOrAtMostOnce) {
                    return RetryAction.FAILOVER_AND_RETRY;
                } else {
                    return new RetryAction(RetryAction.RetryDecision.FAIL, 0,
                            "the invoked method is not idempotent, and unable to determine "
                                    + "whether it was invoked");
                }
            } else {
                return fallbackPolicy.shouldRetry(e, retries, failovers, isIdempotentOrAtMostOnce);
            }
        }

        protected RetryAction checkRetryCount(Exception e, int retries, int failovers) throws Exception {
            if (failovers >= maxFailovers) {
                return new RetryAction(RetryAction.RetryDecision.FAIL, 0,
                        "failovers (" + failovers + ") exceeded maximum allowed (" + maxFailovers + ")");
            }
            if (retries - failovers > maxRetries) {
                return new RetryAction(RetryAction.RetryDecision.FAIL, 0,
                        "retries (" + retries + ") exceeded maximum allowed (" + maxRetries + ")");
            }

            return null;
        }
    }

    /**
     * Fail over and retry in the case of:
     *   Leader dies (Network failure)
     *   Leader node changes.
     *
     *   If an operation that is supposted to be executed on the leader node
     *   is passed to a non-leader node then the non-leader node will thow
     *   NotALeaderException. In case of NotALeaderException the client should
     *   failover to the new leader namenode
     */
    static class FailoverOnLeaderChange extends FailoverOnNetworkExceptionRetry {

        public FailoverOnLeaderChange(RetryPolicy fallbackPolicy, int maxFailovers, int maxRetries,
                long delayMillis, long maxDelayBase,
                Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
            super(fallbackPolicy, maxFailovers, maxRetries, delayMillis, maxDelayBase, exceptionToPolicyMap);
        }

        @Override
        public RetryAction shouldRetry(Exception e, int retries, int failovers, boolean isIdempotentOrAtMostOnce)
                throws Exception {

            RetryAction action = checkRetryCount(e, retries, failovers);
            if (action != null) {
                return action;
            }

            if (isWrappedNotALeaderException(e)) {
                return new RetryAction(RetryAction.RetryDecision.FAILOVER_AND_RETRY,
                        getFailoverOrRetrySleepTime(failovers));
            }

            return super.shouldRetry(e, retries, failovers, isIdempotentOrAtMostOnce);
        }
    }

    /**
     * Return a value which is <code>time</code> increasing exponentially as a
     * function of <code>retries</code>, +/- 0%-50% of that value, chosen
     * randomly.
     * 
     * @param time the base amount of time to work with
     * @param retries the number of retries that have so occurred so far
     * @param cap value at which to cap the base sleep time
     * @return an amount of time to sleep
     */
    public static long calculateExponentialTime(long time, int retries, long cap) {
        long baseTime = Math.min(time * (1L << retries), cap);
        return (long) (baseTime * (ThreadLocalRandom.current().nextDouble() + 0.5));
    }

    private static long calculateExponentialTime(long time, int retries) {
        return calculateExponentialTime(time, retries, Long.MAX_VALUE);
    }

    private static boolean isWrappedStandbyException(Exception e) {
        if (!(e instanceof RemoteException)) {
            return false;
        }
        Exception unwrapped = ((RemoteException) e).unwrapRemoteException(StandbyException.class);
        return unwrapped instanceof StandbyException;
    }

    static RetriableException getWrappedRetriableException(Exception e) {
        if (!(e instanceof RemoteException)) {
            return null;
        }
        Exception unwrapped = ((RemoteException) e).unwrapRemoteException(RetriableException.class);
        return unwrapped instanceof RetriableException ? (RetriableException) unwrapped : null;
    }

    private static boolean isWrappedNotALeaderException(Exception e) {
        if (!(e instanceof RemoteException)) {
            return false;
        }
        Exception unwrapped = ((RemoteException) e).unwrapRemoteException(NotALeaderException.class);
        return unwrapped instanceof NotALeaderException;
    }

    static boolean wrappsException(Exception wrapper, Class wrappedException) {
        if (!(wrapper instanceof RemoteException)) {
            return false;
        }
        return wrappedException.getName().equals(((RemoteException) wrapper).getClassName());
    }

}