org.apache.hadoop.hbase.client.HBaseAdmin.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hbase.client.HBaseAdmin.java

Source

/**
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.client;

import java.io.Closeable;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Pattern;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.RegionException;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotEnabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.UnknownRegionException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.catalog.CatalogTracker;
import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.exceptions.MergeRegionException;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
import org.apache.hadoop.hbase.ipc.MasterCoprocessorRpcChannel;
import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.RequestConverter;
import org.apache.hadoop.hbase.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateNamespaceRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteColumnRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteNamespaceRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteSnapshotRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetClusterStatusRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnapshotsRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetNamespaceDescriptorRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListNamespaceDescriptorsRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableDescriptorsByNamespaceRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableNamesByNamespaceRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.TruncateTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.TruncateTableResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneResponse;
import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
import org.apache.hadoop.hbase.util.Addressing;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.util.StringUtils;
import org.apache.zookeeper.KeeperException;

import com.google.protobuf.ByteString;
import com.google.protobuf.ServiceException;

/**
 * Provides an interface to manage HBase database table metadata + general
 * administrative functions.  Use HBaseAdmin to create, drop, list, enable and
 * disable tables. Use it also to add and drop table column families.
 *
 * <p>See {@link HTable} to add, update, and delete data from an individual table.
 * <p>Currently HBaseAdmin instances are not expected to be long-lived.  For
 * example, an HBaseAdmin instance will not ride over a Master restart.
 */
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class HBaseAdmin implements Admin {
    private static final Log LOG = LogFactory.getLog(HBaseAdmin.class);

    // We use the implementation class rather then the interface because we
    //  need the package protected functions to get the connection to master
    private ClusterConnection connection;

    private volatile Configuration conf;
    private final long pause;
    private final int numRetries;
    // Some operations can take a long time such as disable of big table.
    // numRetries is for 'normal' stuff... Multiply by this factor when
    // want to wait a long time.
    private final int retryLongerMultiplier;
    private boolean aborted;
    private boolean cleanupConnectionOnClose = false; // close the connection in close()
    private boolean closed = false;
    private int operationTimeout;

    private RpcRetryingCallerFactory rpcCallerFactory;

    /**
     * Constructor.
     * See {@link #HBaseAdmin(HConnection connection)}
     *
     * @param c Configuration object. Copied internally.
     */
    public HBaseAdmin(Configuration c) throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
        // Will not leak connections, as the new implementation of the constructor
        // does not throw exceptions anymore.
        this(ConnectionManager.getConnectionInternal(new Configuration(c)));
        this.cleanupConnectionOnClose = true;
    }

    public int getOperationTimeout() {
        return operationTimeout;
    }

    /**
     * Constructor for externally managed HConnections.
     * The connection to master will be created when required by admin functions.
     *
     * @param connection The HConnection instance to use
     * @throws MasterNotRunningException, ZooKeeperConnectionException are not
     *  thrown anymore but kept into the interface for backward api compatibility
     * @deprecated Do not use this internal ctor.
     */
    @Deprecated
    public HBaseAdmin(HConnection connection) throws MasterNotRunningException, ZooKeeperConnectionException {
        this((ClusterConnection) connection);
    }

    HBaseAdmin(ClusterConnection connection) {
        this.conf = connection.getConfiguration();
        this.connection = connection;

        this.pause = this.conf.getLong(HConstants.HBASE_CLIENT_PAUSE, HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
        this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
                HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
        this.retryLongerMultiplier = this.conf.getInt("hbase.client.retries.longer.multiplier", 10);
        this.operationTimeout = this.conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
                HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);

        this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf);
    }

    /**
     * @return A new CatalogTracker instance; call {@link #cleanupCatalogTracker(CatalogTracker)}
     * to cleanup the returned catalog tracker.
     * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
     * @throws IOException
     * @see #cleanupCatalogTracker(CatalogTracker)
     */
    private synchronized CatalogTracker getCatalogTracker() throws ZooKeeperConnectionException, IOException {
        CatalogTracker ct = null;
        try {
            ct = new CatalogTracker(this.conf);
            ct.start();
        } catch (InterruptedException e) {
            // Let it out as an IOE for now until we redo all so tolerate IEs
            throw (InterruptedIOException) new InterruptedIOException("Interrupted").initCause(e);
        }
        return ct;
    }

    private void cleanupCatalogTracker(final CatalogTracker ct) {
        ct.stop();
    }

    @Override
    public void abort(String why, Throwable e) {
        // Currently does nothing but throw the passed message and exception
        this.aborted = true;
        throw new RuntimeException(why, e);
    }

    @Override
    public boolean isAborted() {
        return this.aborted;
    }

    /** @return HConnection used by this object. */
    public HConnection getConnection() {
        return connection;
    }

    /** @return - true if the master server is running. Throws an exception
     *  otherwise.
     * @throws ZooKeeperConnectionException
     * @throws MasterNotRunningException
     */
    public boolean isMasterRunning() throws MasterNotRunningException, ZooKeeperConnectionException {
        return connection.isMasterRunning();
    }

    /**
     * @param tableName Table to check.
     * @return True if table exists already.
     * @throws IOException
     */
    public boolean tableExists(final TableName tableName) throws IOException {
        boolean b = false;
        CatalogTracker ct = getCatalogTracker();
        try {
            b = MetaReader.tableExists(ct, tableName);
        } finally {
            cleanupCatalogTracker(ct);
        }
        return b;
    }

    public boolean tableExists(final byte[] tableName) throws IOException {
        return tableExists(TableName.valueOf(tableName));
    }

    public boolean tableExists(final String tableName) throws IOException {
        return tableExists(TableName.valueOf(tableName));
    }

    /**
     * List all the userspace tables.  In other words, scan the hbase:meta table.
     *
     * If we wanted this to be really fast, we could implement a special
     * catalog table that just contains table names and their descriptors.
     * Right now, it only exists as part of the hbase:meta table's region info.
     *
     * @return - returns an array of HTableDescriptors
     * @throws IOException if a remote or network exception occurs
     */
    public HTableDescriptor[] listTables() throws IOException {
        return this.connection.listTables();
    }

    /**
     * List all the userspace tables matching the given pattern.
     *
     * @param pattern The compiled regular expression to match against
     * @return - returns an array of HTableDescriptors
     * @throws IOException if a remote or network exception occurs
     * @see #listTables()
     */
    public HTableDescriptor[] listTables(Pattern pattern) throws IOException {
        List<HTableDescriptor> matched = new LinkedList<HTableDescriptor>();
        HTableDescriptor[] tables = listTables();
        for (HTableDescriptor table : tables) {
            if (pattern.matcher(table.getTableName().getNameAsString()).matches()) {
                matched.add(table);
            }
        }
        return matched.toArray(new HTableDescriptor[matched.size()]);
    }

    /**
     * List all the userspace tables matching the given regular expression.
     *
     * @param regex The regular expression to match against
     * @return - returns an array of HTableDescriptors
     * @throws IOException if a remote or network exception occurs
     * @see #listTables(java.util.regex.Pattern)
     */
    public HTableDescriptor[] listTables(String regex) throws IOException {
        return listTables(Pattern.compile(regex));
    }

    /**
     * List all of the names of userspace tables.
     * @return String[] table names
     * @throws IOException if a remote or network exception occurs
     */
    @Deprecated
    public String[] getTableNames() throws IOException {
        return this.connection.getTableNames();
    }

    /**
     * List all of the names of userspace tables matching the given regular expression.
     * @param pattern The regular expression to match against
     * @return String[] table names
     * @throws IOException if a remote or network exception occurs
     */
    @Deprecated
    public String[] getTableNames(Pattern pattern) throws IOException {
        List<String> matched = new ArrayList<String>();
        for (String name : this.connection.getTableNames()) {
            if (pattern.matcher(name).matches()) {
                matched.add(name);
            }
        }
        return matched.toArray(new String[matched.size()]);
    }

    /**
     * List all of the names of userspace tables matching the given regular expression.
     * @param regex The regular expression to match against
     * @return String[] table names
     * @throws IOException if a remote or network exception occurs
     */
    @Deprecated
    public String[] getTableNames(String regex) throws IOException {
        return getTableNames(Pattern.compile(regex));
    }

    /**
     * List all of the names of userspace tables.
     * @return TableName[] table names
     * @throws IOException if a remote or network exception occurs
     */
    public TableName[] listTableNames() throws IOException {
        return this.connection.listTableNames();
    }

    /**
     * Method for getting the tableDescriptor
     * @param tableName as a byte []
     * @return the tableDescriptor
     * @throws TableNotFoundException
     * @throws IOException if a remote or network exception occurs
     */
    public HTableDescriptor getTableDescriptor(final TableName tableName)
            throws TableNotFoundException, IOException {
        return this.connection.getHTableDescriptor(tableName);
    }

    public HTableDescriptor getTableDescriptor(final byte[] tableName) throws TableNotFoundException, IOException {
        return getTableDescriptor(TableName.valueOf(tableName));
    }

    private long getPauseTime(int tries) {
        int triesCount = tries;
        if (triesCount >= HConstants.RETRY_BACKOFF.length) {
            triesCount = HConstants.RETRY_BACKOFF.length - 1;
        }
        return this.pause * HConstants.RETRY_BACKOFF[triesCount];
    }

    /**
     * Creates a new table.
     * Synchronous operation.
     *
     * @param desc table descriptor for table
     *
     * @throws IllegalArgumentException if the table name is reserved
     * @throws MasterNotRunningException if master is not running
     * @throws TableExistsException if table already exists (If concurrent
     * threads, the table may have been created between test-for-existence
     * and attempt-at-creation).
     * @throws IOException if a remote or network exception occurs
     */
    public void createTable(HTableDescriptor desc) throws IOException {
        createTable(desc, null);
    }

    /**
     * Creates a new table with the specified number of regions.  The start key
     * specified will become the end key of the first region of the table, and
     * the end key specified will become the start key of the last region of the
     * table (the first region has a null start key and the last region has a
     * null end key).
     *
     * BigInteger math will be used to divide the key range specified into
     * enough segments to make the required number of total regions.
     *
     * Synchronous operation.
     *
     * @param desc table descriptor for table
     * @param startKey beginning of key range
     * @param endKey end of key range
     * @param numRegions the total number of regions to create
     *
     * @throws IllegalArgumentException if the table name is reserved
     * @throws MasterNotRunningException if master is not running
     * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent
     * threads, the table may have been created between test-for-existence
     * and attempt-at-creation).
     * @throws IOException
     */
    public void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions)
            throws IOException {
        if (numRegions < 3) {
            throw new IllegalArgumentException("Must create at least three regions");
        } else if (Bytes.compareTo(startKey, endKey) >= 0) {
            throw new IllegalArgumentException("Start key must be smaller than end key");
        }
        if (numRegions == 3) {
            createTable(desc, new byte[][] { startKey, endKey });
            return;
        }
        byte[][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
        if (splitKeys == null || splitKeys.length != numRegions - 1) {
            throw new IllegalArgumentException("Unable to split key range into enough regions");
        }
        createTable(desc, splitKeys);
    }

    /**
     * Creates a new table with an initial set of empty regions defined by the
     * specified split keys.  The total number of regions created will be the
     * number of split keys plus one. Synchronous operation.
     * Note : Avoid passing empty split key.
     *
     * @param desc table descriptor for table
     * @param splitKeys array of split keys for the initial regions of the table
     *
     * @throws IllegalArgumentException if the table name is reserved, if the split keys
     * are repeated and if the split key has empty byte array.
     * @throws MasterNotRunningException if master is not running
     * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent
     * threads, the table may have been created between test-for-existence
     * and attempt-at-creation).
     * @throws IOException
     */
    public void createTable(final HTableDescriptor desc, byte[][] splitKeys) throws IOException {
        try {
            createTableAsync(desc, splitKeys);
        } catch (SocketTimeoutException ste) {
            LOG.warn("Creating " + desc.getTableName() + " took too long", ste);
        }
        int numRegs = splitKeys == null ? 1 : splitKeys.length + 1;
        int prevRegCount = 0;
        boolean doneWithMetaScan = false;
        for (int tries = 0; tries < this.numRetries * this.retryLongerMultiplier; ++tries) {
            if (!doneWithMetaScan) {
                // Wait for new table to come on-line
                final AtomicInteger actualRegCount = new AtomicInteger(0);
                MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
                    @Override
                    public boolean processRow(Result rowResult) throws IOException {
                        HRegionInfo info = HRegionInfo.getHRegionInfo(rowResult);
                        if (info == null) {
                            LOG.warn("No serialized HRegionInfo in " + rowResult);
                            return true;
                        }
                        if (!info.getTable().equals(desc.getTableName())) {
                            return false;
                        }
                        ServerName serverName = HRegionInfo.getServerName(rowResult);
                        // Make sure that regions are assigned to server
                        if (!(info.isOffline() || info.isSplit()) && serverName != null
                                && serverName.getHostAndPort() != null) {
                            actualRegCount.incrementAndGet();
                        }
                        return true;
                    }
                };
                MetaScanner.metaScan(conf, connection, visitor, desc.getTableName());
                if (actualRegCount.get() < numRegs) {
                    if (tries == this.numRetries * this.retryLongerMultiplier - 1) {
                        throw new RegionOfflineException("Only " + actualRegCount.get() + " of " + numRegs
                                + " regions are online; retries exhausted.");
                    }
                    try { // Sleep
                        Thread.sleep(getPauseTime(tries));
                    } catch (InterruptedException e) {
                        throw new InterruptedIOException("Interrupted when opening" + " regions; "
                                + actualRegCount.get() + " of " + numRegs + " regions processed so far");
                    }
                    if (actualRegCount.get() > prevRegCount) { // Making progress
                        prevRegCount = actualRegCount.get();
                        tries = -1;
                    }
                } else {
                    doneWithMetaScan = true;
                    tries = -1;
                }
            } else if (isTableEnabled(desc.getTableName())) {
                return;
            } else {
                try { // Sleep
                    Thread.sleep(getPauseTime(tries));
                } catch (InterruptedException e) {
                    throw new InterruptedIOException(
                            "Interrupted when waiting" + " for table to be enabled; meta scan was done");
                }
            }
        }
        throw new TableNotEnabledException(
                "Retries exhausted while still waiting for table: " + desc.getTableName() + " to be enabled");
    }

    /**
     * Creates a new table but does not block and wait for it to come online.
     * Asynchronous operation.  To check if the table exists, use
     * {@link #isTableAvailable} -- it is not safe to create an HTable
     * instance to this table before it is available.
     * Note : Avoid passing empty split key.
     * @param desc table descriptor for table
     *
     * @throws IllegalArgumentException Bad table name, if the split keys
     * are repeated and if the split key has empty byte array.
     * @throws MasterNotRunningException if master is not running
     * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent
     * threads, the table may have been created between test-for-existence
     * and attempt-at-creation).
     * @throws IOException
     */
    public void createTableAsync(final HTableDescriptor desc, final byte[][] splitKeys) throws IOException {
        if (desc.getTableName() == null) {
            throw new IllegalArgumentException("TableName cannot be null");
        }
        if (splitKeys != null && splitKeys.length > 0) {
            Arrays.sort(splitKeys, Bytes.BYTES_COMPARATOR);
            // Verify there are no duplicate split keys
            byte[] lastKey = null;
            for (byte[] splitKey : splitKeys) {
                if (Bytes.compareTo(splitKey, HConstants.EMPTY_BYTE_ARRAY) == 0) {
                    throw new IllegalArgumentException("Empty split key must not be passed in the split keys.");
                }
                if (lastKey != null && Bytes.equals(splitKey, lastKey)) {
                    throw new IllegalArgumentException("All split keys must be unique, " + "found duplicate: "
                            + Bytes.toStringBinary(splitKey) + ", " + Bytes.toStringBinary(lastKey));
                }
                lastKey = splitKey;
            }
        }

        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws ServiceException {
                CreateTableRequest request = RequestConverter.buildCreateTableRequest(desc, splitKeys);
                master.createTable(null, request);
                return null;
            }
        });
    }

    public void deleteTable(final String tableName) throws IOException {
        deleteTable(TableName.valueOf(tableName));
    }

    public void deleteTable(final byte[] tableName) throws IOException {
        deleteTable(TableName.valueOf(tableName));
    }

    /**
     * Deletes a table.
     * Synchronous operation.
     *
     * @param tableName name of table to delete
     * @throws IOException if a remote or network exception occurs
     */
    public void deleteTable(final TableName tableName) throws IOException {
        boolean tableExists = true;

        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws ServiceException {
                DeleteTableRequest req = RequestConverter.buildDeleteTableRequest(tableName);
                master.deleteTable(null, req);
                return null;
            }
        });

        int failures = 0;
        // Wait until all regions deleted
        for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
            try {
                HRegionLocation firstMetaServer = getFirstMetaServerForTable(tableName);
                Scan scan = MetaReader.getScanForTableName(tableName);
                scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
                ScanRequest request = RequestConverter
                        .buildScanRequest(firstMetaServer.getRegionInfo().getRegionName(), scan, 1, true);
                Result[] values = null;
                // Get a batch at a time.
                ClientService.BlockingInterface server = connection.getClient(firstMetaServer.getServerName());
                PayloadCarryingRpcController controller = new PayloadCarryingRpcController();
                try {
                    controller.setPriority(tableName);
                    ScanResponse response = server.scan(controller, request);
                    values = ResponseConverter.getResults(controller.cellScanner(), response);
                } catch (ServiceException se) {
                    throw ProtobufUtil.getRemoteException(se);
                }

                // let us wait until hbase:meta table is updated and
                // HMaster removes the table from its HTableDescriptors
                if (values == null || values.length == 0) {
                    tableExists = false;
                    GetTableDescriptorsResponse htds;
                    MasterKeepAliveConnection master = connection.getKeepAliveMasterService();
                    try {
                        GetTableDescriptorsRequest req = RequestConverter
                                .buildGetTableDescriptorsRequest(tableName);
                        htds = master.getTableDescriptors(null, req);
                    } catch (ServiceException se) {
                        throw ProtobufUtil.getRemoteException(se);
                    } finally {
                        master.close();
                    }
                    tableExists = !htds.getTableSchemaList().isEmpty();
                    if (!tableExists) {
                        break;
                    }
                }
            } catch (IOException ex) {
                failures++;
                if (failures == numRetries - 1) { // no more tries left
                    if (ex instanceof RemoteException) {
                        throw ((RemoteException) ex).unwrapRemoteException();
                    } else {
                        throw ex;
                    }
                }
            }
            try {
                Thread.sleep(getPauseTime(tries));
            } catch (InterruptedException e) {
                throw new InterruptedIOException("Interrupted when waiting" + " for table to be deleted");
            }
        }

        if (tableExists) {
            throw new IOException("Retries exhausted, it took too long to wait" + " for the table " + tableName
                    + " to be deleted.");
        }
        // Delete cached information to prevent clients from using old locations
        this.connection.clearRegionCache(tableName);
        LOG.info("Deleted " + tableName);
    }

    /**
     * Deletes tables matching the passed in pattern and wait on completion.
     *
     * Warning: Use this method carefully, there is no prompting and the effect is
     * immediate. Consider using {@link #listTables(java.lang.String)} and
     * {@link #deleteTable(byte[])}
     *
     * @param regex The regular expression to match table names against
     * @return Table descriptors for tables that couldn't be deleted
     * @throws IOException
     * @see #deleteTables(java.util.regex.Pattern)
     * @see #deleteTable(java.lang.String)
     */
    public HTableDescriptor[] deleteTables(String regex) throws IOException {
        return deleteTables(Pattern.compile(regex));
    }

    /**
     * Delete tables matching the passed in pattern and wait on completion.
     *
     * Warning: Use this method carefully, there is no prompting and the effect is
     * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
     * {@link #deleteTable(byte[])}
     *
     * @param pattern The pattern to match table names against
     * @return Table descriptors for tables that couldn't be deleted
     * @throws IOException
     */
    public HTableDescriptor[] deleteTables(Pattern pattern) throws IOException {
        List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
        for (HTableDescriptor table : listTables(pattern)) {
            try {
                deleteTable(table.getTableName());
            } catch (IOException ex) {
                LOG.info("Failed to delete table " + table.getTableName(), ex);
                failed.add(table);
            }
        }
        return failed.toArray(new HTableDescriptor[failed.size()]);
    }

    /**
     * Truncate a table.
     * Synchronous operation.
     *
     * @param tableName name of table to truncate
     * @param preserveSplits True if the splits should be preserved
     * @throws IOException if a remote or network exception occurs
     */
    public void truncateTable(final TableName tableName, final boolean preserveSplits) throws IOException {
        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws ServiceException {
                TruncateTableRequest req = RequestConverter.buildTruncateTableRequest(tableName, preserveSplits);
                master.truncateTable(null, req);
                return null;
            }
        });
    }

    /**
     * Enable a table.  May timeout.  Use {@link #enableTableAsync(byte[])}
     * and {@link #isTableEnabled(byte[])} instead.
     * The table has to be in disabled state for it to be enabled.
     * @param tableName name of the table
     * @throws IOException if a remote or network exception occurs
     * There could be couple types of IOException
     * TableNotFoundException means the table doesn't exist.
     * TableNotDisabledException means the table isn't in disabled state.
     * @see #isTableEnabled(byte[])
     * @see #disableTable(byte[])
     * @see #enableTableAsync(byte[])
     */
    public void enableTable(final TableName tableName) throws IOException {
        enableTableAsync(tableName);

        // Wait until all regions are enabled
        waitUntilTableIsEnabled(tableName);

        LOG.info("Enabled table " + tableName);
    }

    public void enableTable(final byte[] tableName) throws IOException {
        enableTable(TableName.valueOf(tableName));
    }

    public void enableTable(final String tableName) throws IOException {
        enableTable(TableName.valueOf(tableName));
    }

    /**
     * Wait for the table to be enabled and available
     * If enabling the table exceeds the retry period, an exception is thrown.
     * @param tableName name of the table
     * @throws IOException if a remote or network exception occurs or
     *    table is not enabled after the retries period.
     */
    private void waitUntilTableIsEnabled(final TableName tableName) throws IOException {
        boolean enabled = false;
        long start = EnvironmentEdgeManager.currentTimeMillis();
        for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
            try {
                enabled = isTableEnabled(tableName);
            } catch (TableNotFoundException tnfe) {
                // wait for table to be created
                enabled = false;
            }
            enabled = enabled && isTableAvailable(tableName);
            if (enabled) {
                break;
            }
            long sleep = getPauseTime(tries);
            if (LOG.isDebugEnabled()) {
                LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " + "enabled in " + tableName);
            }
            try {
                Thread.sleep(sleep);
            } catch (InterruptedException e) {
                // Do this conversion rather than let it out because do not want to
                // change the method signature.
                throw (InterruptedIOException) new InterruptedIOException("Interrupted").initCause(e);
            }
        }
        if (!enabled) {
            long msec = EnvironmentEdgeManager.currentTimeMillis() - start;
            throw new IOException("Table '" + tableName + "' not yet enabled, after " + msec + "ms.");
        }
    }

    /**
     * Brings a table on-line (enables it).  Method returns immediately though
     * enable of table may take some time to complete, especially if the table
     * is large (All regions are opened as part of enabling process).  Check
     * {@link #isTableEnabled(byte[])} to learn when table is fully online.  If
     * table is taking too long to online, check server logs.
     * @param tableName
     * @throws IOException
     * @since 0.90.0
     */
    public void enableTableAsync(final TableName tableName) throws IOException {
        TableName.isLegalFullyQualifiedTableName(tableName.getName());
        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws ServiceException {
                LOG.info("Started enable of " + tableName);
                EnableTableRequest req = RequestConverter.buildEnableTableRequest(tableName);
                master.enableTable(null, req);
                return null;
            }
        });
    }

    public void enableTableAsync(final byte[] tableName) throws IOException {
        enableTable(TableName.valueOf(tableName));
    }

    public void enableTableAsync(final String tableName) throws IOException {
        enableTableAsync(TableName.valueOf(tableName));
    }

    /**
     * Enable tables matching the passed in pattern and wait on completion.
     *
     * Warning: Use this method carefully, there is no prompting and the effect is
     * immediate. Consider using {@link #listTables(java.lang.String)} and
     * {@link #enableTable(byte[])}
     *
     * @param regex The regular expression to match table names against
     * @throws IOException
     * @see #enableTables(java.util.regex.Pattern)
     * @see #enableTable(java.lang.String)
     */
    public HTableDescriptor[] enableTables(String regex) throws IOException {
        return enableTables(Pattern.compile(regex));
    }

    /**
     * Enable tables matching the passed in pattern and wait on completion.
     *
     * Warning: Use this method carefully, there is no prompting and the effect is
     * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
     * {@link #enableTable(byte[])}
     *
     * @param pattern The pattern to match table names against
     * @throws IOException
     */
    public HTableDescriptor[] enableTables(Pattern pattern) throws IOException {
        List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
        for (HTableDescriptor table : listTables(pattern)) {
            if (isTableDisabled(table.getTableName())) {
                try {
                    enableTable(table.getTableName());
                } catch (IOException ex) {
                    LOG.info("Failed to enable table " + table.getTableName(), ex);
                    failed.add(table);
                }
            }
        }
        return failed.toArray(new HTableDescriptor[failed.size()]);
    }

    /**
     * Starts the disable of a table.  If it is being served, the master
     * will tell the servers to stop serving it.  This method returns immediately.
     * The disable of a table can take some time if the table is large (all
     * regions are closed as part of table disable operation).
     * Call {@link #isTableDisabled(byte[])} to check for when disable completes.
     * If table is taking too long to online, check server logs.
     * @param tableName name of table
     * @throws IOException if a remote or network exception occurs
     * @see #isTableDisabled(byte[])
     * @see #isTableEnabled(byte[])
     * @since 0.90.0
     */
    public void disableTableAsync(final TableName tableName) throws IOException {
        TableName.isLegalFullyQualifiedTableName(tableName.getName());
        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws ServiceException {
                LOG.info("Started disable of " + tableName);
                DisableTableRequest req = RequestConverter.buildDisableTableRequest(tableName);
                master.disableTable(null, req);
                return null;
            }
        });
    }

    public void disableTableAsync(final byte[] tableName) throws IOException {
        disableTableAsync(TableName.valueOf(tableName));
    }

    public void disableTableAsync(final String tableName) throws IOException {
        disableTableAsync(TableName.valueOf(tableName));
    }

    /**
     * Disable table and wait on completion.  May timeout eventually.  Use
     * {@link #disableTableAsync(byte[])} and {@link #isTableDisabled(String)}
     * instead.
     * The table has to be in enabled state for it to be disabled.
     * @param tableName
     * @throws IOException
     * There could be couple types of IOException
     * TableNotFoundException means the table doesn't exist.
     * TableNotEnabledException means the table isn't in enabled state.
     */
    public void disableTable(final TableName tableName) throws IOException {
        disableTableAsync(tableName);
        // Wait until table is disabled
        boolean disabled = false;
        for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
            disabled = isTableDisabled(tableName);
            if (disabled) {
                break;
            }
            long sleep = getPauseTime(tries);
            if (LOG.isDebugEnabled()) {
                LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " + "disabled in " + tableName);
            }
            try {
                Thread.sleep(sleep);
            } catch (InterruptedException e) {
                // Do this conversion rather than let it out because do not want to
                // change the method signature.
                throw (InterruptedIOException) new InterruptedIOException("Interrupted").initCause(e);
            }
        }
        if (!disabled) {
            throw new RegionException("Retries exhausted, it took too long to wait" + " for the table " + tableName
                    + " to be disabled.");
        }
        LOG.info("Disabled " + tableName);
    }

    public void disableTable(final byte[] tableName) throws IOException {
        disableTable(TableName.valueOf(tableName));
    }

    public void disableTable(final String tableName) throws IOException {
        disableTable(TableName.valueOf(tableName));
    }

    /**
     * Disable tables matching the passed in pattern and wait on completion.
     *
     * Warning: Use this method carefully, there is no prompting and the effect is
     * immediate. Consider using {@link #listTables(java.lang.String)} and
     * {@link #disableTable(byte[])}
     *
     * @param regex The regular expression to match table names against
     * @return Table descriptors for tables that couldn't be disabled
     * @throws IOException
     * @see #disableTables(java.util.regex.Pattern)
     * @see #disableTable(java.lang.String)
     */
    public HTableDescriptor[] disableTables(String regex) throws IOException {
        return disableTables(Pattern.compile(regex));
    }

    /**
     * Disable tables matching the passed in pattern and wait on completion.
     *
     * Warning: Use this method carefully, there is no prompting and the effect is
     * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
     * {@link #disableTable(byte[])}
     *
     * @param pattern The pattern to match table names against
     * @return Table descriptors for tables that couldn't be disabled
     * @throws IOException
     */
    public HTableDescriptor[] disableTables(Pattern pattern) throws IOException {
        List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
        for (HTableDescriptor table : listTables(pattern)) {
            if (isTableEnabled(table.getTableName())) {
                try {
                    disableTable(table.getTableName());
                } catch (IOException ex) {
                    LOG.info("Failed to disable table " + table.getTableName(), ex);
                    failed.add(table);
                }
            }
        }
        return failed.toArray(new HTableDescriptor[failed.size()]);
    }

    /*
     * Checks whether table exists. If not, throws TableNotFoundException
     * @param tableName
     */
    private void checkTableExistence(TableName tableName) throws IOException {
        if (!tableExists(tableName)) {
            throw new TableNotFoundException(tableName);
        }
    }

    /**
     * @param tableName name of table to check
     * @return true if table is on-line
     * @throws IOException if a remote or network exception occurs
     */
    public boolean isTableEnabled(TableName tableName) throws IOException {
        checkTableExistence(tableName);
        return connection.isTableEnabled(tableName);
    }

    public boolean isTableEnabled(byte[] tableName) throws IOException {
        return isTableEnabled(TableName.valueOf(tableName));
    }

    public boolean isTableEnabled(String tableName) throws IOException {
        return isTableEnabled(TableName.valueOf(tableName));
    }

    /**
     * @param tableName name of table to check
     * @return true if table is off-line
     * @throws IOException if a remote or network exception occurs
     */
    public boolean isTableDisabled(TableName tableName) throws IOException {
        checkTableExistence(tableName);
        return connection.isTableDisabled(tableName);
    }

    public boolean isTableDisabled(byte[] tableName) throws IOException {
        return isTableDisabled(TableName.valueOf(tableName));
    }

    public boolean isTableDisabled(String tableName) throws IOException {
        return isTableDisabled(TableName.valueOf(tableName));
    }

    /**
     * @param tableName name of table to check
     * @return true if all regions of the table are available
     * @throws IOException if a remote or network exception occurs
     */
    public boolean isTableAvailable(TableName tableName) throws IOException {
        return connection.isTableAvailable(tableName);
    }

    public boolean isTableAvailable(byte[] tableName) throws IOException {
        return isTableAvailable(TableName.valueOf(tableName));
    }

    public boolean isTableAvailable(String tableName) throws IOException {
        return isTableAvailable(TableName.valueOf(tableName));
    }

    /**
     * Use this api to check if the table has been created with the specified number of
     * splitkeys which was used while creating the given table.
     * Note : If this api is used after a table's region gets splitted, the api may return
     * false.
     * @param tableName
     *          name of table to check
     * @param splitKeys
     *          keys to check if the table has been created with all split keys
     * @throws IOException
     *           if a remote or network excpetion occurs
     */
    public boolean isTableAvailable(TableName tableName, byte[][] splitKeys) throws IOException {
        return connection.isTableAvailable(tableName, splitKeys);
    }

    public boolean isTableAvailable(byte[] tableName, byte[][] splitKeys) throws IOException {
        return isTableAvailable(TableName.valueOf(tableName), splitKeys);
    }

    public boolean isTableAvailable(String tableName, byte[][] splitKeys) throws IOException {
        return isTableAvailable(TableName.valueOf(tableName), splitKeys);
    }

    /**
     * Get the status of alter command - indicates how many regions have received
     * the updated schema Asynchronous operation.
     *
     * @param tableName TableName instance
     * @return Pair indicating the number of regions updated Pair.getFirst() is the
     *         regions that are yet to be updated Pair.getSecond() is the total number
     *         of regions of the table
     * @throws IOException
     *           if a remote or network exception occurs
     */
    public Pair<Integer, Integer> getAlterStatus(final TableName tableName) throws IOException {
        return executeCallable(new MasterCallable<Pair<Integer, Integer>>(getConnection()) {
            @Override
            public Pair<Integer, Integer> call(int callTimeout) throws ServiceException {
                GetSchemaAlterStatusRequest req = RequestConverter.buildGetSchemaAlterStatusRequest(tableName);
                GetSchemaAlterStatusResponse ret = master.getSchemaAlterStatus(null, req);
                Pair<Integer, Integer> pair = new Pair<Integer, Integer>(
                        Integer.valueOf(ret.getYetToUpdateRegions()), Integer.valueOf(ret.getTotalRegions()));
                return pair;
            }
        });
    }

    /**
     * Get the status of alter command - indicates how many regions have received
     * the updated schema Asynchronous operation.
     *
     * @param tableName
     *          name of the table to get the status of
     * @return Pair indicating the number of regions updated Pair.getFirst() is the
     *         regions that are yet to be updated Pair.getSecond() is the total number
     *         of regions of the table
     * @throws IOException
     *           if a remote or network exception occurs
     */
    public Pair<Integer, Integer> getAlterStatus(final byte[] tableName) throws IOException {
        return getAlterStatus(TableName.valueOf(tableName));
    }

    /**
     * Add a column to an existing table.
     * Asynchronous operation.
     *
     * @param tableName name of the table to add column to
     * @param column column descriptor of column to be added
     * @throws IOException if a remote or network exception occurs
     */
    public void addColumn(final byte[] tableName, HColumnDescriptor column) throws IOException {
        addColumn(TableName.valueOf(tableName), column);
    }

    /**
     * Add a column to an existing table.
     * Asynchronous operation.
     *
     * @param tableName name of the table to add column to
     * @param column column descriptor of column to be added
     * @throws IOException if a remote or network exception occurs
     */
    public void addColumn(final String tableName, HColumnDescriptor column) throws IOException {
        addColumn(TableName.valueOf(tableName), column);
    }

    /**
     * Add a column to an existing table.
     * Asynchronous operation.
     *
     * @param tableName name of the table to add column to
     * @param column column descriptor of column to be added
     * @throws IOException if a remote or network exception occurs
     */
    public void addColumn(final TableName tableName, final HColumnDescriptor column) throws IOException {
        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws ServiceException {
                AddColumnRequest req = RequestConverter.buildAddColumnRequest(tableName, column);
                master.addColumn(null, req);
                return null;
            }
        });
    }

    /**
     * Delete a column from a table.
     * Asynchronous operation.
     *
     * @param tableName name of table
     * @param columnName name of column to be deleted
     * @throws IOException if a remote or network exception occurs
     */
    public void deleteColumn(final byte[] tableName, final String columnName) throws IOException {
        deleteColumn(TableName.valueOf(tableName), Bytes.toBytes(columnName));
    }

    /**
     * Delete a column from a table.
     * Asynchronous operation.
     *
     * @param tableName name of table
     * @param columnName name of column to be deleted
     * @throws IOException if a remote or network exception occurs
     */
    public void deleteColumn(final String tableName, final String columnName) throws IOException {
        deleteColumn(TableName.valueOf(tableName), Bytes.toBytes(columnName));
    }

    /**
     * Delete a column from a table.
     * Asynchronous operation.
     *
     * @param tableName name of table
     * @param columnName name of column to be deleted
     * @throws IOException if a remote or network exception occurs
     */
    public void deleteColumn(final TableName tableName, final byte[] columnName) throws IOException {
        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws ServiceException {
                DeleteColumnRequest req = RequestConverter.buildDeleteColumnRequest(tableName, columnName);
                master.deleteColumn(null, req);
                return null;
            }
        });
    }

    /**
     * Modify an existing column family on a table.
     * Asynchronous operation.
     *
     * @param tableName name of table
     * @param descriptor new column descriptor to use
     * @throws IOException if a remote or network exception occurs
     */
    public void modifyColumn(final String tableName, HColumnDescriptor descriptor) throws IOException {
        modifyColumn(TableName.valueOf(tableName), descriptor);
    }

    /**
     * Modify an existing column family on a table.
     * Asynchronous operation.
     *
     * @param tableName name of table
     * @param descriptor new column descriptor to use
     * @throws IOException if a remote or network exception occurs
     */
    public void modifyColumn(final byte[] tableName, HColumnDescriptor descriptor) throws IOException {
        modifyColumn(TableName.valueOf(tableName), descriptor);
    }

    /**
     * Modify an existing column family on a table.
     * Asynchronous operation.
     *
     * @param tableName name of table
     * @param descriptor new column descriptor to use
     * @throws IOException if a remote or network exception occurs
     */
    public void modifyColumn(final TableName tableName, final HColumnDescriptor descriptor) throws IOException {
        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws ServiceException {
                ModifyColumnRequest req = RequestConverter.buildModifyColumnRequest(tableName, descriptor);
                master.modifyColumn(null, req);
                return null;
            }
        });
    }

    /**
     * Close a region. For expert-admins.  Runs close on the regionserver.  The
     * master will not be informed of the close.
     * @param regionname region name to close
     * @param serverName If supplied, we'll use this location rather than
     * the one currently in <code>hbase:meta</code>
     * @throws IOException if a remote or network exception occurs
     */
    public void closeRegion(final String regionname, final String serverName) throws IOException {
        closeRegion(Bytes.toBytes(regionname), serverName);
    }

    /**
     * Close a region.  For expert-admins  Runs close on the regionserver.  The
     * master will not be informed of the close.
     * @param regionname region name to close
     * @param serverName The servername of the regionserver.  If passed null we
     * will use servername found in the hbase:meta table. A server name
     * is made of host, port and startcode.  Here is an example:
     * <code> host187.example.com,60020,1289493121758</code>
     * @throws IOException if a remote or network exception occurs
     */
    public void closeRegion(final byte[] regionname, final String serverName) throws IOException {
        CatalogTracker ct = getCatalogTracker();
        try {
            if (serverName != null) {
                Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, regionname);
                if (pair == null || pair.getFirst() == null) {
                    throw new UnknownRegionException(Bytes.toStringBinary(regionname));
                } else {
                    closeRegion(ServerName.valueOf(serverName), pair.getFirst());
                }
            } else {
                Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, regionname);
                if (pair == null) {
                    throw new UnknownRegionException(Bytes.toStringBinary(regionname));
                } else if (pair.getSecond() == null) {
                    throw new NoServerForRegionException(Bytes.toStringBinary(regionname));
                } else {
                    closeRegion(pair.getSecond(), pair.getFirst());
                }
            }
        } finally {
            cleanupCatalogTracker(ct);
        }
    }

    /**
     * For expert-admins. Runs close on the regionserver. Closes a region based on
     * the encoded region name. The region server name is mandatory. If the
     * servername is provided then based on the online regions in the specified
     * regionserver the specified region will be closed. The master will not be
     * informed of the close. Note that the regionname is the encoded regionname.
     *
     * @param encodedRegionName
     *          The encoded region name; i.e. the hash that makes up the region
     *          name suffix: e.g. if regionname is
     *          <code>TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396.</code>
     *          , then the encoded region name is:
     *          <code>527db22f95c8a9e0116f0cc13c680396</code>.
     * @param serverName
     *          The servername of the regionserver. A server name is made of host,
     *          port and startcode. This is mandatory. Here is an example:
     *          <code> host187.example.com,60020,1289493121758</code>
     * @return true if the region was closed, false if not.
     * @throws IOException
     *           if a remote or network exception occurs
     */
    public boolean closeRegionWithEncodedRegionName(final String encodedRegionName, final String serverName)
            throws IOException {
        if (null == serverName || ("").equals(serverName.trim())) {
            throw new IllegalArgumentException("The servername cannot be null or empty.");
        }
        ServerName sn = ServerName.valueOf(serverName);
        AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
        // Close the region without updating zk state.
        CloseRegionRequest request = RequestConverter.buildCloseRegionRequest(sn, encodedRegionName, false);
        try {
            CloseRegionResponse response = admin.closeRegion(null, request);
            boolean isRegionClosed = response.getClosed();
            if (false == isRegionClosed) {
                LOG.error("Not able to close the region " + encodedRegionName + ".");
            }
            return isRegionClosed;
        } catch (ServiceException se) {
            throw ProtobufUtil.getRemoteException(se);
        }
    }

    /**
     * Close a region.  For expert-admins  Runs close on the regionserver.  The
     * master will not be informed of the close.
     * @param sn
     * @param hri
     * @throws IOException
     */
    public void closeRegion(final ServerName sn, final HRegionInfo hri) throws IOException {
        AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
        // Close the region without updating zk state.
        ProtobufUtil.closeRegion(admin, sn, hri.getRegionName(), false);
    }

    /**
     * Get all the online regions on a region server.
     */
    public List<HRegionInfo> getOnlineRegions(final ServerName sn) throws IOException {
        AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
        return ProtobufUtil.getOnlineRegions(admin);
    }

    /**
     * Flush a table or an individual region.
     * Synchronous operation.
     *
     * @param tableNameOrRegionName table or region to flush
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException
     */
    public void flush(final String tableNameOrRegionName) throws IOException, InterruptedException {
        flush(Bytes.toBytes(tableNameOrRegionName));
    }

    /**
     * Flush a table or an individual region.
     * Synchronous operation.
     *
     * @param tableNameOrRegionName table or region to flush
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException
     */
    public void flush(final byte[] tableNameOrRegionName) throws IOException, InterruptedException {
        CatalogTracker ct = getCatalogTracker();
        try {
            Pair<HRegionInfo, ServerName> regionServerPair = getRegion(tableNameOrRegionName, ct);
            if (regionServerPair != null) {
                if (regionServerPair.getSecond() == null) {
                    throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
                } else {
                    flush(regionServerPair.getSecond(), regionServerPair.getFirst());
                }
            } else {
                final TableName tableName = checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
                if (isTableDisabled(tableName)) {
                    LOG.info("Table is disabled: " + tableName.getNameAsString());
                    return;
                }
                execProcedure("flush-table-proc", tableName.getNameAsString(), new HashMap<String, String>());
            }
        } finally {
            cleanupCatalogTracker(ct);
        }
    }

    private void flush(final ServerName sn, final HRegionInfo hri) throws IOException {
        AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
        FlushRegionRequest request = RequestConverter.buildFlushRegionRequest(hri.getRegionName());
        try {
            admin.flushRegion(null, request);
        } catch (ServiceException se) {
            throw ProtobufUtil.getRemoteException(se);
        }
    }

    /**
     * Compact a table or an individual region.
     * Asynchronous operation.
     *
     * @param tableNameOrRegionName table or region to compact
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException
     */
    public void compact(final String tableNameOrRegionName) throws IOException, InterruptedException {
        compact(Bytes.toBytes(tableNameOrRegionName));
    }

    /**
     * Compact a table or an individual region.
     * Asynchronous operation.
     *
     * @param tableNameOrRegionName table or region to compact
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException
     */
    public void compact(final byte[] tableNameOrRegionName) throws IOException, InterruptedException {
        compact(tableNameOrRegionName, null, false);
    }

    /**
     * Compact a column family within a table or region.
     * Asynchronous operation.
     *
     * @param tableOrRegionName table or region to compact
     * @param columnFamily column family within a table or region
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException
     */
    public void compact(String tableOrRegionName, String columnFamily) throws IOException, InterruptedException {
        compact(Bytes.toBytes(tableOrRegionName), Bytes.toBytes(columnFamily));
    }

    /**
     * Compact a column family within a table or region.
     * Asynchronous operation.
     *
     * @param tableNameOrRegionName table or region to compact
     * @param columnFamily column family within a table or region
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException
     */
    public void compact(final byte[] tableNameOrRegionName, final byte[] columnFamily)
            throws IOException, InterruptedException {
        compact(tableNameOrRegionName, columnFamily, false);
    }

    /**
     * Major compact a table or an individual region.
     * Asynchronous operation.
     *
     * @param tableNameOrRegionName table or region to major compact
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException
     */
    public void majorCompact(final String tableNameOrRegionName) throws IOException, InterruptedException {
        majorCompact(Bytes.toBytes(tableNameOrRegionName));
    }

    /**
     * Major compact a table or an individual region.
     * Asynchronous operation.
     *
     * @param tableNameOrRegionName table or region to major compact
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException
     */
    public void majorCompact(final byte[] tableNameOrRegionName) throws IOException, InterruptedException {
        compact(tableNameOrRegionName, null, true);
    }

    /**
     * Major compact a column family within a table or region.
     * Asynchronous operation.
     *
     * @param tableNameOrRegionName table or region to major compact
     * @param columnFamily column family within a table or region
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException
     */
    public void majorCompact(final String tableNameOrRegionName, final String columnFamily)
            throws IOException, InterruptedException {
        majorCompact(Bytes.toBytes(tableNameOrRegionName), Bytes.toBytes(columnFamily));
    }

    /**
     * Major compact a column family within a table or region.
     * Asynchronous operation.
     *
     * @param tableNameOrRegionName table or region to major compact
     * @param columnFamily column family within a table or region
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException
     */
    public void majorCompact(final byte[] tableNameOrRegionName, final byte[] columnFamily)
            throws IOException, InterruptedException {
        compact(tableNameOrRegionName, columnFamily, true);
    }

    /**
     * Compact a table or an individual region.
     * Asynchronous operation.
     *
     * @param tableNameOrRegionName table or region to compact
     * @param columnFamily column family within a table or region
     * @param major True if we are to do a major compaction.
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException
     */
    private void compact(final byte[] tableNameOrRegionName, final byte[] columnFamily, final boolean major)
            throws IOException, InterruptedException {
        CatalogTracker ct = getCatalogTracker();
        try {
            Pair<HRegionInfo, ServerName> regionServerPair = getRegion(tableNameOrRegionName, ct);
            if (regionServerPair != null) {
                if (regionServerPair.getSecond() == null) {
                    throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
                } else {
                    compact(regionServerPair.getSecond(), regionServerPair.getFirst(), major, columnFamily);
                }
            } else {
                final TableName tableName = checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
                List<Pair<HRegionInfo, ServerName>> pairs = MetaReader.getTableRegionsAndLocations(ct, tableName);
                for (Pair<HRegionInfo, ServerName> pair : pairs) {
                    if (pair.getFirst().isOffline())
                        continue;
                    if (pair.getSecond() == null)
                        continue;
                    try {
                        compact(pair.getSecond(), pair.getFirst(), major, columnFamily);
                    } catch (NotServingRegionException e) {
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Trying to" + (major ? " major" : "") + " compact " + pair.getFirst() + ": "
                                    + StringUtils.stringifyException(e));
                        }
                    }
                }
            }
        } finally {
            cleanupCatalogTracker(ct);
        }
    }

    private void compact(final ServerName sn, final HRegionInfo hri, final boolean major, final byte[] family)
            throws IOException {
        AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
        CompactRegionRequest request = RequestConverter.buildCompactRegionRequest(hri.getRegionName(), major,
                family);
        try {
            admin.compactRegion(null, request);
        } catch (ServiceException se) {
            throw ProtobufUtil.getRemoteException(se);
        }
    }

    /**
     * Move the region <code>r</code> to <code>dest</code>.
     * @param encodedRegionName The encoded region name; i.e. the hash that makes
     * up the region name suffix: e.g. if regionname is
     * <code>TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396.</code>,
     * then the encoded region name is: <code>527db22f95c8a9e0116f0cc13c680396</code>.
     * @param destServerName The servername of the destination regionserver.  If
     * passed the empty byte array we'll assign to a random server.  A server name
     * is made of host, port and startcode.  Here is an example:
     * <code> host187.example.com,60020,1289493121758</code>
     * @throws UnknownRegionException Thrown if we can't find a region named
     * <code>encodedRegionName</code>
     * @throws ZooKeeperConnectionException
     * @throws MasterNotRunningException
     */
    public void move(final byte[] encodedRegionName, final byte[] destServerName)
            throws HBaseIOException, MasterNotRunningException, ZooKeeperConnectionException {
        MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
        try {
            MoveRegionRequest request = RequestConverter.buildMoveRegionRequest(encodedRegionName, destServerName);
            stub.moveRegion(null, request);
        } catch (ServiceException se) {
            IOException ioe = ProtobufUtil.getRemoteException(se);
            if (ioe instanceof HBaseIOException) {
                throw (HBaseIOException) ioe;
            }
            LOG.error("Unexpected exception: " + se + " from calling HMaster.moveRegion");
        } catch (DeserializationException de) {
            LOG.error("Could not parse destination server name: " + de);
        } finally {
            stub.close();
        }
    }

    /**
     * @param regionName
     *          Region name to assign.
     * @throws MasterNotRunningException
     * @throws ZooKeeperConnectionException
     * @throws IOException
     */
    public void assign(final byte[] regionName)
            throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
        final byte[] toBeAssigned = getRegionName(regionName);
        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws ServiceException {
                AssignRegionRequest request = RequestConverter.buildAssignRegionRequest(toBeAssigned);
                master.assignRegion(null, request);
                return null;
            }
        });
    }

    /**
     * Unassign a region from current hosting regionserver.  Region will then be
     * assigned to a regionserver chosen at random.  Region could be reassigned
     * back to the same server.  Use {@link #move(byte[], byte[])} if you want
     * to control the region movement.
     * @param regionName Region to unassign. Will clear any existing RegionPlan
     * if one found.
     * @param force If true, force unassign (Will remove region from
     * regions-in-transition too if present. If results in double assignment
     * use hbck -fix to resolve. To be used by experts).
     * @throws MasterNotRunningException
     * @throws ZooKeeperConnectionException
     * @throws IOException
     */
    public void unassign(final byte[] regionName, final boolean force)
            throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
        final byte[] toBeUnassigned = getRegionName(regionName);
        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws ServiceException {
                UnassignRegionRequest request = RequestConverter.buildUnassignRegionRequest(toBeUnassigned, force);
                master.unassignRegion(null, request);
                return null;
            }
        });
    }

    /**
     * Offline specified region from master's in-memory state. It will not attempt to reassign the
     * region as in unassign. This API can be used when a region not served by any region server and
     * still online as per Master's in memory state. If this API is incorrectly used on active region
     * then master will loose track of that region.
     *
     * This is a special method that should be used by experts or hbck.
     *
     * @param regionName
     *          Region to offline.
     * @throws IOException
     */
    public void offline(final byte[] regionName) throws IOException {
        MasterKeepAliveConnection master = connection.getKeepAliveMasterService();
        try {
            master.offlineRegion(null, RequestConverter.buildOfflineRegionRequest(regionName));
        } catch (ServiceException se) {
            throw ProtobufUtil.getRemoteException(se);
        } finally {
            master.close();
        }
    }

    /**
     * Turn the load balancer on or off.
     * @param on If true, enable balancer. If false, disable balancer.
     * @param synchronous If true, it waits until current balance() call, if outstanding, to return.
     * @return Previous balancer value
     */
    public boolean setBalancerRunning(final boolean on, final boolean synchronous)
            throws MasterNotRunningException, ZooKeeperConnectionException {
        MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
        try {
            SetBalancerRunningRequest req = RequestConverter.buildSetBalancerRunningRequest(on, synchronous);
            return stub.setBalancerRunning(null, req).getPrevBalanceValue();
        } catch (ServiceException se) {
            IOException ioe = ProtobufUtil.getRemoteException(se);
            if (ioe instanceof MasterNotRunningException) {
                throw (MasterNotRunningException) ioe;
            }
            if (ioe instanceof ZooKeeperConnectionException) {
                throw (ZooKeeperConnectionException) ioe;
            }

            // Throwing MasterNotRunningException even though not really valid in order to not
            // break interface by adding additional exception type.
            throw new MasterNotRunningException("Unexpected exception when calling balanceSwitch", se);
        } finally {
            stub.close();
        }
    }

    /**
     * Invoke the balancer.  Will run the balancer and if regions to move, it will
     * go ahead and do the reassignments.  Can NOT run for various reasons.  Check
     * logs.
     * @return True if balancer ran, false otherwise.
     */
    public boolean balancer() throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException {
        MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
        try {
            return stub.balance(null, RequestConverter.buildBalanceRequest()).getBalancerRan();
        } finally {
            stub.close();
        }
    }

    /**
     * Enable/Disable the catalog janitor
     * @param enable if true enables the catalog janitor
     * @return the previous state
     * @throws ServiceException
     * @throws MasterNotRunningException
     */
    public boolean enableCatalogJanitor(boolean enable) throws ServiceException, MasterNotRunningException {
        MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
        try {
            return stub.enableCatalogJanitor(null, RequestConverter.buildEnableCatalogJanitorRequest(enable))
                    .getPrevValue();
        } finally {
            stub.close();
        }
    }

    /**
     * Ask for a scan of the catalog table
     * @return the number of entries cleaned
     * @throws ServiceException
     * @throws MasterNotRunningException
     */
    public int runCatalogScan() throws ServiceException, MasterNotRunningException {
        MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
        try {
            return stub.runCatalogScan(null, RequestConverter.buildCatalogScanRequest()).getScanResult();
        } finally {
            stub.close();
        }
    }

    /**
     * Query on the catalog janitor state (Enabled/Disabled?)
     * @throws ServiceException
     * @throws org.apache.hadoop.hbase.MasterNotRunningException
     */
    public boolean isCatalogJanitorEnabled() throws ServiceException, MasterNotRunningException {
        MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
        try {
            return stub.isCatalogJanitorEnabled(null, RequestConverter.buildIsCatalogJanitorEnabledRequest())
                    .getValue();
        } finally {
            stub.close();
        }
    }

    /**
     * Merge two regions. Asynchronous operation.
     * @param encodedNameOfRegionA encoded name of region a
     * @param encodedNameOfRegionB encoded name of region b
     * @param forcible true if do a compulsory merge, otherwise we will only merge
     *          two adjacent regions
     * @throws IOException
     */
    public void mergeRegions(final byte[] encodedNameOfRegionA, final byte[] encodedNameOfRegionB,
            final boolean forcible) throws IOException {
        MasterKeepAliveConnection master = connection.getKeepAliveMasterService();
        try {
            DispatchMergingRegionsRequest request = RequestConverter
                    .buildDispatchMergingRegionsRequest(encodedNameOfRegionA, encodedNameOfRegionB, forcible);
            master.dispatchMergingRegions(null, request);
        } catch (ServiceException se) {
            IOException ioe = ProtobufUtil.getRemoteException(se);
            if (ioe instanceof UnknownRegionException) {
                throw (UnknownRegionException) ioe;
            }
            if (ioe instanceof MergeRegionException) {
                throw (MergeRegionException) ioe;
            }
            LOG.error("Unexpected exception: " + se + " from calling HMaster.dispatchMergingRegions");
        } catch (DeserializationException de) {
            LOG.error("Could not parse destination server name: " + de);
        } finally {
            master.close();
        }
    }

    /**
     * Split a table or an individual region.
     * Asynchronous operation.
     *
     * @param tableNameOrRegionName table or region to split
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException
     */
    public void split(final String tableNameOrRegionName) throws IOException, InterruptedException {
        split(Bytes.toBytes(tableNameOrRegionName));
    }

    /**
     * Split a table or an individual region.  Implicitly finds an optimal split
     * point.  Asynchronous operation.
     *
     * @param tableNameOrRegionName table to region to split
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException
     */
    public void split(final byte[] tableNameOrRegionName) throws IOException, InterruptedException {
        split(tableNameOrRegionName, null);
    }

    public void split(final String tableNameOrRegionName, final String splitPoint)
            throws IOException, InterruptedException {
        split(Bytes.toBytes(tableNameOrRegionName), Bytes.toBytes(splitPoint));
    }

    /**
     * Split a table or an individual region.
     * Asynchronous operation.
     *
     * @param tableNameOrRegionName table to region to split
     * @param splitPoint the explicit position to split on
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException interrupt exception occurred
     */
    public void split(final byte[] tableNameOrRegionName, final byte[] splitPoint)
            throws IOException, InterruptedException {
        CatalogTracker ct = getCatalogTracker();
        try {
            Pair<HRegionInfo, ServerName> regionServerPair = getRegion(tableNameOrRegionName, ct);
            if (regionServerPair != null) {
                if (regionServerPair.getSecond() == null) {
                    throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
                } else {
                    split(regionServerPair.getSecond(), regionServerPair.getFirst(), splitPoint);
                }
            } else {
                final TableName tableName = checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
                List<Pair<HRegionInfo, ServerName>> pairs = MetaReader.getTableRegionsAndLocations(ct, tableName);
                for (Pair<HRegionInfo, ServerName> pair : pairs) {
                    // May not be a server for a particular row
                    if (pair.getSecond() == null)
                        continue;
                    HRegionInfo r = pair.getFirst();
                    // check for parents
                    if (r.isSplitParent())
                        continue;
                    // if a split point given, only split that particular region
                    if (splitPoint != null && !r.containsRow(splitPoint))
                        continue;
                    // call out to region server to do split now
                    split(pair.getSecond(), pair.getFirst(), splitPoint);
                }
            }
        } finally {
            cleanupCatalogTracker(ct);
        }
    }

    private void split(final ServerName sn, final HRegionInfo hri, byte[] splitPoint) throws IOException {
        if (hri.getStartKey() != null && splitPoint != null
                && Bytes.compareTo(hri.getStartKey(), splitPoint) == 0) {
            throw new IOException("should not give a splitkey which equals to startkey!");
        }
        AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
        ProtobufUtil.split(admin, hri, splitPoint);
    }

    /**
     * Modify an existing table, more IRB friendly version.
     * Asynchronous operation.  This means that it may be a while before your
     * schema change is updated across all of the table.
     *
     * @param tableName name of table.
     * @param htd modified description of the table
     * @throws IOException if a remote or network exception occurs
     */
    public void modifyTable(final TableName tableName, final HTableDescriptor htd) throws IOException {
        if (!tableName.equals(htd.getTableName())) {
            throw new IllegalArgumentException("the specified table name '" + tableName
                    + "' doesn't match with the HTD one: " + htd.getTableName());
        }

        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws ServiceException {
                ModifyTableRequest request = RequestConverter.buildModifyTableRequest(tableName, htd);
                master.modifyTable(null, request);
                return null;
            }
        });
    }

    public void modifyTable(final byte[] tableName, final HTableDescriptor htd) throws IOException {
        modifyTable(TableName.valueOf(tableName), htd);
    }

    public void modifyTable(final String tableName, final HTableDescriptor htd) throws IOException {
        modifyTable(TableName.valueOf(tableName), htd);
    }

    /**
     * @param tableNameOrRegionName Name of a table or name of a region.
     * @param ct A {@link CatalogTracker} instance (caller of this method usually has one).
     * @return a pair of HRegionInfo and ServerName if <code>tableNameOrRegionName</code> is
     *  a verified region name (we call {@link  MetaReader#getRegion( CatalogTracker, byte[])}
     *  else null.
     * Throw an exception if <code>tableNameOrRegionName</code> is null.
     * @throws IOException
     */
    Pair<HRegionInfo, ServerName> getRegion(final byte[] tableNameOrRegionName, final CatalogTracker ct)
            throws IOException {
        if (tableNameOrRegionName == null) {
            throw new IllegalArgumentException("Pass a table name or region name");
        }
        Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, tableNameOrRegionName);
        if (pair == null) {
            final AtomicReference<Pair<HRegionInfo, ServerName>> result = new AtomicReference<Pair<HRegionInfo, ServerName>>(
                    null);
            final String encodedName = Bytes.toString(tableNameOrRegionName);
            MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
                @Override
                public boolean processRow(Result data) throws IOException {
                    HRegionInfo info = HRegionInfo.getHRegionInfo(data);
                    if (info == null) {
                        LOG.warn("No serialized HRegionInfo in " + data);
                        return true;
                    }
                    if (!encodedName.equals(info.getEncodedName()))
                        return true;
                    ServerName sn = HRegionInfo.getServerName(data);
                    result.set(new Pair<HRegionInfo, ServerName>(info, sn));
                    return false; // found the region, stop
                }
            };

            MetaScanner.metaScan(conf, connection, visitor, null);
            pair = result.get();
        }
        return pair;
    }

    /**
     * If the input is a region name, it is returned as is. If it's an
     * encoded region name, the corresponding region is found from meta
     * and its region name is returned. If we can't find any region in
     * meta matching the input as either region name or encoded region
     * name, the input is returned as is. We don't throw unknown
     * region exception.
     */
    private byte[] getRegionName(final byte[] regionNameOrEncodedRegionName) throws IOException {
        if (Bytes.equals(regionNameOrEncodedRegionName, HRegionInfo.FIRST_META_REGIONINFO.getRegionName()) || Bytes
                .equals(regionNameOrEncodedRegionName, HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes())) {
            return HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
        }
        CatalogTracker ct = getCatalogTracker();
        byte[] tmp = regionNameOrEncodedRegionName;
        try {
            Pair<HRegionInfo, ServerName> regionServerPair = getRegion(regionNameOrEncodedRegionName, ct);
            if (regionServerPair != null && regionServerPair.getFirst() != null) {
                tmp = regionServerPair.getFirst().getRegionName();
            }
        } finally {
            cleanupCatalogTracker(ct);
        }
        return tmp;
    }

    /**
     * Check if table exists or not
     * @param tableName Name of a table.
     * @param ct A {@link CatalogTracker} instance (caller of this method usually has one).
     * @return tableName instance
     * @throws IOException if a remote or network exception occurs.
     * @throws TableNotFoundException if table does not exist.
     */
    //TODO rename this method
    private TableName checkTableExists(final TableName tableName, CatalogTracker ct) throws IOException {
        if (!MetaReader.tableExists(ct, tableName)) {
            throw new TableNotFoundException(tableName);
        }
        return tableName;
    }

    /**
     * Shuts down the HBase cluster
     * @throws IOException if a remote or network exception occurs
     */
    public synchronized void shutdown() throws IOException {
        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws ServiceException {
                master.shutdown(null, ShutdownRequest.newBuilder().build());
                return null;
            }
        });
    }

    /**
     * Shuts down the current HBase master only.
     * Does not shutdown the cluster.
     * @see #shutdown()
     * @throws IOException if a remote or network exception occurs
     */
    public synchronized void stopMaster() throws IOException {
        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws ServiceException {
                master.stopMaster(null, StopMasterRequest.newBuilder().build());
                return null;
            }
        });
    }

    /**
     * Stop the designated regionserver
     * @param hostnamePort Hostname and port delimited by a <code>:</code> as in
     * <code>example.org:1234</code>
     * @throws IOException if a remote or network exception occurs
     */
    public synchronized void stopRegionServer(final String hostnamePort) throws IOException {
        String hostname = Addressing.parseHostname(hostnamePort);
        int port = Addressing.parsePort(hostnamePort);
        AdminService.BlockingInterface admin = this.connection.getAdmin(ServerName.valueOf(hostname, port, 0));
        StopServerRequest request = RequestConverter
                .buildStopServerRequest("Called by admin client " + this.connection.toString());
        try {
            admin.stopServer(null, request);
        } catch (ServiceException se) {
            throw ProtobufUtil.getRemoteException(se);
        }
    }

    /**
     * @return cluster status
     * @throws IOException if a remote or network exception occurs
     */
    public ClusterStatus getClusterStatus() throws IOException {
        return executeCallable(new MasterCallable<ClusterStatus>(getConnection()) {
            @Override
            public ClusterStatus call(int callTimeout) throws ServiceException {
                GetClusterStatusRequest req = RequestConverter.buildGetClusterStatusRequest();
                return ClusterStatus.convert(master.getClusterStatus(null, req).getClusterStatus());
            }
        });
    }

    private HRegionLocation getFirstMetaServerForTable(final TableName tableName) throws IOException {
        return connection.locateRegion(TableName.META_TABLE_NAME,
                HRegionInfo.createRegionName(tableName, null, HConstants.NINES, false));
    }

    /**
     * @return Configuration used by the instance.
     */
    public Configuration getConfiguration() {
        return this.conf;
    }

    /**
     * Create a new namespace
     * @param descriptor descriptor which describes the new namespace
     * @throws IOException
     */
    public void createNamespace(final NamespaceDescriptor descriptor) throws IOException {
        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws Exception {
                master.createNamespace(null, CreateNamespaceRequest.newBuilder()
                        .setNamespaceDescriptor(ProtobufUtil.toProtoNamespaceDescriptor(descriptor)).build());
                return null;
            }
        });
    }

    /**
     * Modify an existing namespace
     * @param descriptor descriptor which describes the new namespace
     * @throws IOException
     */
    public void modifyNamespace(final NamespaceDescriptor descriptor) throws IOException {
        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws Exception {
                master.modifyNamespace(null, ModifyNamespaceRequest.newBuilder()
                        .setNamespaceDescriptor(ProtobufUtil.toProtoNamespaceDescriptor(descriptor)).build());
                return null;
            }
        });
    }

    /**
     * Delete an existing namespace. Only empty namespaces (no tables) can be removed.
     * @param name namespace name
     * @throws IOException
     */
    public void deleteNamespace(final String name) throws IOException {
        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws Exception {
                master.deleteNamespace(null, DeleteNamespaceRequest.newBuilder().setNamespaceName(name).build());
                return null;
            }
        });
    }

    /**
     * Get a namespace descriptor by name
     * @param name name of namespace descriptor
     * @return A descriptor
     * @throws IOException
     */
    public NamespaceDescriptor getNamespaceDescriptor(final String name) throws IOException {
        return executeCallable(new MasterCallable<NamespaceDescriptor>(getConnection()) {
            @Override
            public NamespaceDescriptor call(int callTimeout) throws Exception {
                return ProtobufUtil.toNamespaceDescriptor(master
                        .getNamespaceDescriptor(null,
                                GetNamespaceDescriptorRequest.newBuilder().setNamespaceName(name).build())
                        .getNamespaceDescriptor());
            }
        });
    }

    /**
     * List available namespace descriptors
     * @return List of descriptors
     * @throws IOException
     */
    public NamespaceDescriptor[] listNamespaceDescriptors() throws IOException {
        return executeCallable(new MasterCallable<NamespaceDescriptor[]>(getConnection()) {
            @Override
            public NamespaceDescriptor[] call(int callTimeout) throws Exception {
                List<HBaseProtos.NamespaceDescriptor> list = master
                        .listNamespaceDescriptors(null, ListNamespaceDescriptorsRequest.newBuilder().build())
                        .getNamespaceDescriptorList();
                NamespaceDescriptor[] res = new NamespaceDescriptor[list.size()];
                for (int i = 0; i < list.size(); i++) {
                    res[i] = ProtobufUtil.toNamespaceDescriptor(list.get(i));
                }
                return res;
            }
        });
    }

    /**
     * Get list of table descriptors by namespace
     * @param name namespace name
     * @return A descriptor
     * @throws IOException
     */
    public HTableDescriptor[] listTableDescriptorsByNamespace(final String name) throws IOException {
        return executeCallable(new MasterCallable<HTableDescriptor[]>(getConnection()) {
            @Override
            public HTableDescriptor[] call(int callTimeout) throws Exception {
                List<TableSchema> list = master
                        .listTableDescriptorsByNamespace(null,
                                ListTableDescriptorsByNamespaceRequest.newBuilder().setNamespaceName(name).build())
                        .getTableSchemaList();
                HTableDescriptor[] res = new HTableDescriptor[list.size()];
                for (int i = 0; i < list.size(); i++) {

                    res[i] = HTableDescriptor.convert(list.get(i));
                }
                return res;
            }
        });
    }

    /**
     * Get list of table names by namespace
     * @param name namespace name
     * @return The list of table names in the namespace
     * @throws IOException
     */
    public TableName[] listTableNamesByNamespace(final String name) throws IOException {
        return executeCallable(new MasterCallable<TableName[]>(getConnection()) {
            @Override
            public TableName[] call(int callTimeout) throws Exception {
                List<HBaseProtos.TableName> tableNames = master
                        .listTableNamesByNamespace(null,
                                ListTableNamesByNamespaceRequest.newBuilder().setNamespaceName(name).build())
                        .getTableNameList();
                TableName[] result = new TableName[tableNames.size()];
                for (int i = 0; i < tableNames.size(); i++) {
                    result[i] = ProtobufUtil.toTableName(tableNames.get(i));
                }
                return result;
            }
        });
    }

    /**
     * Check to see if HBase is running. Throw an exception if not.
     * We consider that HBase is running if ZooKeeper and Master are running.
     *
     * @param conf system configuration
     * @throws MasterNotRunningException if the master is not running
     * @throws ZooKeeperConnectionException if unable to connect to zookeeper
     */
    public static void checkHBaseAvailable(Configuration conf)
            throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException, IOException {
        Configuration copyOfConf = HBaseConfiguration.create(conf);

        // We set it to make it fail as soon as possible if HBase is not available
        copyOfConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
        copyOfConf.setInt("zookeeper.recovery.retry", 0);

        ConnectionManager.HConnectionImplementation connection = (ConnectionManager.HConnectionImplementation) HConnectionManager
                .getConnection(copyOfConf);

        try {
            // Check ZK first.
            // If the connection exists, we may have a connection to ZK that does
            //  not work anymore
            ZooKeeperKeepAliveConnection zkw = null;
            try {
                zkw = connection.getKeepAliveZooKeeperWatcher();
                zkw.getRecoverableZooKeeper().getZooKeeper().exists(zkw.baseZNode, false);

            } catch (IOException e) {
                throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
            } catch (InterruptedException e) {
                throw (InterruptedIOException) new InterruptedIOException("Can't connect to ZooKeeper")
                        .initCause(e);
            } catch (KeeperException e) {
                throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
            } finally {
                if (zkw != null) {
                    zkw.close();
                }
            }

            // Check Master
            connection.isMasterRunning();

        } finally {
            connection.close();
        }
    }

    /**
     * get the regions of a given table.
     *
     * @param tableName the name of the table
     * @return Ordered list of {@link HRegionInfo}.
     * @throws IOException
     */
    public List<HRegionInfo> getTableRegions(final TableName tableName) throws IOException {
        CatalogTracker ct = getCatalogTracker();
        List<HRegionInfo> Regions = null;
        try {
            Regions = MetaReader.getTableRegions(ct, tableName, true);
        } finally {
            cleanupCatalogTracker(ct);
        }
        return Regions;
    }

    public List<HRegionInfo> getTableRegions(final byte[] tableName) throws IOException {
        return getTableRegions(TableName.valueOf(tableName));
    }

    @Override
    public synchronized void close() throws IOException {
        if (cleanupConnectionOnClose && this.connection != null && !this.closed) {
            this.connection.close();
            this.closed = true;
        }
    }

    /**
     * Get tableDescriptors
     * @param tableNames List of table names
     * @return HTD[] the tableDescriptor
     * @throws IOException if a remote or network exception occurs
     */
    public HTableDescriptor[] getTableDescriptorsByTableName(List<TableName> tableNames) throws IOException {
        return this.connection.getHTableDescriptorsByTableName(tableNames);
    }

    /**
     * Get tableDescriptors
     * @param names List of table names
     * @return HTD[] the tableDescriptor
     * @throws IOException if a remote or network exception occurs
     */
    public HTableDescriptor[] getTableDescriptors(List<String> names) throws IOException {
        List<TableName> tableNames = new ArrayList<TableName>(names.size());
        for (String name : names) {
            tableNames.add(TableName.valueOf(name));
        }
        return getTableDescriptorsByTableName(tableNames);
    }

    /**
     * Roll the log writer. That is, start writing log messages to a new file.
     *
     * @param serverName
     *          The servername of the regionserver. A server name is made of host,
     *          port and startcode. This is mandatory. Here is an example:
     *          <code> host187.example.com,60020,1289493121758</code>
     * @return If lots of logs, flush the returned regions so next time through
     * we can clean logs. Returns null if nothing to flush.  Names are actual
     * region names as returned by {@link HRegionInfo#getEncodedName()}
     * @throws IOException if a remote or network exception occurs
     * @throws FailedLogCloseException
     */
    public synchronized byte[][] rollHLogWriter(String serverName) throws IOException, FailedLogCloseException {
        ServerName sn = ServerName.valueOf(serverName);
        AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
        RollWALWriterRequest request = RequestConverter.buildRollWALWriterRequest();
        try {
            RollWALWriterResponse response = admin.rollWALWriter(null, request);
            int regionCount = response.getRegionToFlushCount();
            byte[][] regionsToFlush = new byte[regionCount][];
            for (int i = 0; i < regionCount; i++) {
                ByteString region = response.getRegionToFlush(i);
                regionsToFlush[i] = region.toByteArray();
            }
            return regionsToFlush;
        } catch (ServiceException se) {
            throw ProtobufUtil.getRemoteException(se);
        }
    }

    public String[] getMasterCoprocessors() {
        try {
            return getClusterStatus().getMasterCoprocessors();
        } catch (IOException e) {
            LOG.error("Could not getClusterStatus()", e);
            return null;
        }
    }

    /**
     * Get the current compaction state of a table or region.
     * It could be in a major compaction, a minor compaction, both, or none.
     *
     * @param tableNameOrRegionName table or region to major compact
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException
     * @return the current compaction state
     */
    public CompactionState getCompactionState(final String tableNameOrRegionName)
            throws IOException, InterruptedException {
        return getCompactionState(Bytes.toBytes(tableNameOrRegionName));
    }

    /**
     * Get the current compaction state of a table or region.
     * It could be in a major compaction, a minor compaction, both, or none.
     *
     * @param tableNameOrRegionName table or region to major compact
     * @throws IOException if a remote or network exception occurs
     * @throws InterruptedException
     * @return the current compaction state
     */
    public CompactionState getCompactionState(final byte[] tableNameOrRegionName)
            throws IOException, InterruptedException {
        CompactionState state = CompactionState.NONE;
        CatalogTracker ct = getCatalogTracker();
        try {
            Pair<HRegionInfo, ServerName> regionServerPair = getRegion(tableNameOrRegionName, ct);
            if (regionServerPair != null) {
                if (regionServerPair.getSecond() == null) {
                    throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
                } else {
                    ServerName sn = regionServerPair.getSecond();
                    AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
                    GetRegionInfoRequest request = RequestConverter
                            .buildGetRegionInfoRequest(regionServerPair.getFirst().getRegionName(), true);
                    GetRegionInfoResponse response = admin.getRegionInfo(null, request);
                    return response.getCompactionState();
                }
            } else {
                final TableName tableName = checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
                List<Pair<HRegionInfo, ServerName>> pairs = MetaReader.getTableRegionsAndLocations(ct, tableName);
                for (Pair<HRegionInfo, ServerName> pair : pairs) {
                    if (pair.getFirst().isOffline())
                        continue;
                    if (pair.getSecond() == null)
                        continue;
                    try {
                        ServerName sn = pair.getSecond();
                        AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
                        GetRegionInfoRequest request = RequestConverter
                                .buildGetRegionInfoRequest(pair.getFirst().getRegionName(), true);
                        GetRegionInfoResponse response = admin.getRegionInfo(null, request);
                        switch (response.getCompactionState()) {
                        case MAJOR_AND_MINOR:
                            return CompactionState.MAJOR_AND_MINOR;
                        case MAJOR:
                            if (state == CompactionState.MINOR) {
                                return CompactionState.MAJOR_AND_MINOR;
                            }
                            state = CompactionState.MAJOR;
                            break;
                        case MINOR:
                            if (state == CompactionState.MAJOR) {
                                return CompactionState.MAJOR_AND_MINOR;
                            }
                            state = CompactionState.MINOR;
                            break;
                        case NONE:
                        default: // nothing, continue
                        }
                    } catch (NotServingRegionException e) {
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Trying to get compaction state of " + pair.getFirst() + ": "
                                    + StringUtils.stringifyException(e));
                        }
                    } catch (RemoteException e) {
                        if (e.getMessage().indexOf(NotServingRegionException.class.getName()) >= 0) {
                            if (LOG.isDebugEnabled()) {
                                LOG.debug("Trying to get compaction state of " + pair.getFirst() + ": "
                                        + StringUtils.stringifyException(e));
                            }
                        } else {
                            throw e;
                        }
                    }
                }
            }
        } catch (ServiceException se) {
            throw ProtobufUtil.getRemoteException(se);
        } finally {
            cleanupCatalogTracker(ct);
        }
        return state;
    }

    /**
     * Take a snapshot for the given table. If the table is enabled, a FLUSH-type snapshot will be
     * taken. If the table is disabled, an offline snapshot is taken.
     * <p>
     * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
     * snapshot with the same name (even a different type or with different parameters) will fail with
     * a {@link SnapshotCreationException} indicating the duplicate naming.
     * <p>
     * Snapshot names follow the same naming constraints as tables in HBase. See
     * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
     * @param snapshotName name of the snapshot to be created
     * @param tableName name of the table for which snapshot is created
     * @throws IOException if a remote or network exception occurs
     * @throws SnapshotCreationException if snapshot creation failed
     * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
     */
    public void snapshot(final String snapshotName, final TableName tableName)
            throws IOException, SnapshotCreationException, IllegalArgumentException {
        snapshot(snapshotName, tableName, SnapshotDescription.Type.FLUSH);
    }

    public void snapshot(final String snapshotName, final String tableName)
            throws IOException, SnapshotCreationException, IllegalArgumentException {
        snapshot(snapshotName, TableName.valueOf(tableName), SnapshotDescription.Type.FLUSH);
    }

    /**
     public void snapshot(final String snapshotName,
      * Create a timestamp consistent snapshot for the given table.
                    final byte[] tableName) throws IOException,
      * <p>
      * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
      * snapshot with the same name (even a different type or with different parameters) will fail with
      * a {@link SnapshotCreationException} indicating the duplicate naming.
      * <p>
      * Snapshot names follow the same naming constraints as tables in HBase.
      * @param snapshotName name of the snapshot to be created
      * @param tableName name of the table for which snapshot is created
      * @throws IOException if a remote or network exception occurs
      * @throws SnapshotCreationException if snapshot creation failed
      * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
      */
    public void snapshot(final byte[] snapshotName, final TableName tableName)
            throws IOException, SnapshotCreationException, IllegalArgumentException {
        snapshot(Bytes.toString(snapshotName), tableName, SnapshotDescription.Type.FLUSH);
    }

    public void snapshot(final byte[] snapshotName, final byte[] tableName)
            throws IOException, SnapshotCreationException, IllegalArgumentException {
        snapshot(Bytes.toString(snapshotName), TableName.valueOf(tableName), SnapshotDescription.Type.FLUSH);
    }

    /**
     * Create typed snapshot of the table.
     * <p>
     * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
     * snapshot with the same name (even a different type or with different parameters) will fail with
     * a {@link SnapshotCreationException} indicating the duplicate naming.
     * <p>
     * Snapshot names follow the same naming constraints as tables in HBase. See
     * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
     * <p>
     * @param snapshotName name to give the snapshot on the filesystem. Must be unique from all other
     *          snapshots stored on the cluster
     * @param tableName name of the table to snapshot
     * @param type type of snapshot to take
     * @throws IOException we fail to reach the master
     * @throws SnapshotCreationException if snapshot creation failed
     * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
     */
    public void snapshot(final String snapshotName, final TableName tableName, SnapshotDescription.Type type)
            throws IOException, SnapshotCreationException, IllegalArgumentException {
        SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
        builder.setTable(tableName.getNameAsString());
        builder.setName(snapshotName);
        builder.setType(type);
        snapshot(builder.build());
    }

    public void snapshot(final String snapshotName, final String tableName, SnapshotDescription.Type type)
            throws IOException, SnapshotCreationException, IllegalArgumentException {
        snapshot(snapshotName, TableName.valueOf(tableName), type);
    }

    public void snapshot(final String snapshotName, final byte[] tableName, SnapshotDescription.Type type)
            throws IOException, SnapshotCreationException, IllegalArgumentException {
        snapshot(snapshotName, TableName.valueOf(tableName), type);
    }

    /**
     * Take a snapshot and wait for the server to complete that snapshot (blocking).
     * <p>
     * Only a single snapshot should be taken at a time for an instance of HBase, or results may be
     * undefined (you can tell multiple HBase clusters to snapshot at the same time, but only one at a
     * time for a single cluster).
     * <p>
     * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
     * snapshot with the same name (even a different type or with different parameters) will fail with
     * a {@link SnapshotCreationException} indicating the duplicate naming.
     * <p>
     * Snapshot names follow the same naming constraints as tables in HBase. See
     * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
     * <p>
     * You should probably use {@link #snapshot(String, String)} or {@link #snapshot(byte[], byte[])}
     * unless you are sure about the type of snapshot that you want to take.
     * @param snapshot snapshot to take
     * @throws IOException or we lose contact with the master.
     * @throws SnapshotCreationException if snapshot failed to be taken
     * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
     */
    public void snapshot(SnapshotDescription snapshot)
            throws IOException, SnapshotCreationException, IllegalArgumentException {
        // actually take the snapshot
        SnapshotResponse response = takeSnapshotAsync(snapshot);
        final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot).build();
        IsSnapshotDoneResponse done = null;
        long start = EnvironmentEdgeManager.currentTimeMillis();
        long max = response.getExpectedTimeout();
        long maxPauseTime = max / this.numRetries;
        int tries = 0;
        LOG.debug(
                "Waiting a max of " + max + " ms for snapshot '" + ClientSnapshotDescriptionUtils.toString(snapshot)
                        + "'' to complete. (max " + maxPauseTime + " ms per retry)");
        while (tries == 0 || ((EnvironmentEdgeManager.currentTimeMillis() - start) < max && !done.getDone())) {
            try {
                // sleep a backoff <= pauseTime amount
                long sleep = getPauseTime(tries++);
                sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
                LOG.debug("(#" + tries + ") Sleeping: " + sleep + "ms while waiting for snapshot completion.");
                Thread.sleep(sleep);
            } catch (InterruptedException e) {
                throw (InterruptedIOException) new InterruptedIOException("Interrupted").initCause(e);
            }
            LOG.debug("Getting current status of snapshot from master...");
            done = executeCallable(new MasterCallable<IsSnapshotDoneResponse>(getConnection()) {
                @Override
                public IsSnapshotDoneResponse call(int callTimeout) throws ServiceException {
                    return master.isSnapshotDone(null, request);
                }
            });
        }
        if (!done.getDone()) {
            throw new SnapshotCreationException(
                    "Snapshot '" + snapshot.getName() + "' wasn't completed in expectedTime:" + max + " ms",
                    snapshot);
        }
    }

    /**
     * Take a snapshot without waiting for the server to complete that snapshot (asynchronous)
     * <p>
     * Only a single snapshot should be taken at a time, or results may be undefined.
     * @param snapshot snapshot to take
     * @return response from the server indicating the max time to wait for the snapshot
     * @throws IOException if the snapshot did not succeed or we lose contact with the master.
     * @throws SnapshotCreationException if snapshot creation failed
     * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
     */
    public SnapshotResponse takeSnapshotAsync(SnapshotDescription snapshot)
            throws IOException, SnapshotCreationException {
        ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
        final SnapshotRequest request = SnapshotRequest.newBuilder().setSnapshot(snapshot).build();
        // run the snapshot on the master
        return executeCallable(new MasterCallable<SnapshotResponse>(getConnection()) {
            @Override
            public SnapshotResponse call(int callTimeout) throws ServiceException {
                return master.snapshot(null, request);
            }
        });
    }

    /**
     * Check the current state of the passed snapshot.
     * <p>
     * There are three possible states:
     * <ol>
     * <li>running - returns <tt>false</tt></li>
     * <li>finished - returns <tt>true</tt></li>
     * <li>finished with error - throws the exception that caused the snapshot to fail</li>
     * </ol>
     * <p>
     * The cluster only knows about the most recent snapshot. Therefore, if another snapshot has been
     * run/started since the snapshot your are checking, you will recieve an
     * {@link UnknownSnapshotException}.
     * @param snapshot description of the snapshot to check
     * @return <tt>true</tt> if the snapshot is completed, <tt>false</tt> if the snapshot is still
     *         running
     * @throws IOException if we have a network issue
     * @throws HBaseSnapshotException if the snapshot failed
     * @throws UnknownSnapshotException if the requested snapshot is unknown
     */
    public boolean isSnapshotFinished(final SnapshotDescription snapshot)
            throws IOException, HBaseSnapshotException, UnknownSnapshotException {

        return executeCallable(new MasterCallable<IsSnapshotDoneResponse>(getConnection()) {
            @Override
            public IsSnapshotDoneResponse call(int callTimeout) throws ServiceException {
                return master.isSnapshotDone(null,
                        IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot).build());
            }
        }).getDone();
    }

    /**
     * Restore the specified snapshot on the original table. (The table must be disabled)
     * If the "hbase.snapshot.restore.take.failsafe.snapshot" configuration property
     * is set to true, a snapshot of the current table is taken
     * before executing the restore operation.
     * In case of restore failure, the failsafe snapshot will be restored.
     * If the restore completes without problem the failsafe snapshot is deleted.
     *
     * @param snapshotName name of the snapshot to restore
     * @throws IOException if a remote or network exception occurs
     * @throws RestoreSnapshotException if snapshot failed to be restored
     * @throws IllegalArgumentException if the restore request is formatted incorrectly
     */
    public void restoreSnapshot(final byte[] snapshotName) throws IOException, RestoreSnapshotException {
        restoreSnapshot(Bytes.toString(snapshotName));
    }

    /**
     * Restore the specified snapshot on the original table. (The table must be disabled)
     * If the "hbase.snapshot.restore.take.failsafe.snapshot" configuration property
     * is set to true, a snapshot of the current table is taken
     * before executing the restore operation.
     * In case of restore failure, the failsafe snapshot will be restored.
     * If the restore completes without problem the failsafe snapshot is deleted.
     *
     * @param snapshotName name of the snapshot to restore
     * @throws IOException if a remote or network exception occurs
     * @throws RestoreSnapshotException if snapshot failed to be restored
     * @throws IllegalArgumentException if the restore request is formatted incorrectly
     */
    public void restoreSnapshot(final String snapshotName) throws IOException, RestoreSnapshotException {
        boolean takeFailSafeSnapshot = conf.getBoolean("hbase.snapshot.restore.take.failsafe.snapshot", false);
        restoreSnapshot(snapshotName, takeFailSafeSnapshot);
    }

    /**
     * Restore the specified snapshot on the original table. (The table must be disabled)
     * If 'takeFailSafeSnapshot' is set to true, a snapshot of the current table is taken
     * before executing the restore operation.
     * In case of restore failure, the failsafe snapshot will be restored.
     * If the restore completes without problem the failsafe snapshot is deleted.
     *
     * The failsafe snapshot name is configurable by using the property
     * "hbase.snapshot.restore.failsafe.name".
     *
     * @param snapshotName name of the snapshot to restore
     * @param takeFailSafeSnapshot true if the failsafe snapshot should be taken
     * @throws IOException if a remote or network exception occurs
     * @throws RestoreSnapshotException if snapshot failed to be restored
     * @throws IllegalArgumentException if the restore request is formatted incorrectly
     */
    public void restoreSnapshot(final byte[] snapshotName, final boolean takeFailSafeSnapshot)
            throws IOException, RestoreSnapshotException {
        restoreSnapshot(Bytes.toString(snapshotName), takeFailSafeSnapshot);
    }

    /**
     * Restore the specified snapshot on the original table. (The table must be disabled)
     * If 'takeFailSafeSnapshot' is set to true, a snapshot of the current table is taken
     * before executing the restore operation.
     * In case of restore failure, the failsafe snapshot will be restored.
     * If the restore completes without problem the failsafe snapshot is deleted.
     *
     * The failsafe snapshot name is configurable by using the property
     * "hbase.snapshot.restore.failsafe.name".
     *
     * @param snapshotName name of the snapshot to restore
     * @param takeFailSafeSnapshot true if the failsafe snapshot should be taken
     * @throws IOException if a remote or network exception occurs
     * @throws RestoreSnapshotException if snapshot failed to be restored
     * @throws IllegalArgumentException if the restore request is formatted incorrectly
     */
    public void restoreSnapshot(final String snapshotName, boolean takeFailSafeSnapshot)
            throws IOException, RestoreSnapshotException {
        TableName tableName = null;
        for (SnapshotDescription snapshotInfo : listSnapshots()) {
            if (snapshotInfo.getName().equals(snapshotName)) {
                tableName = TableName.valueOf(snapshotInfo.getTable());
                break;
            }
        }

        if (tableName == null) {
            throw new RestoreSnapshotException("Unable to find the table name for snapshot=" + snapshotName);
        }

        // The table does not exists, switch to clone.
        if (!tableExists(tableName)) {
            try {
                cloneSnapshot(snapshotName, tableName);
            } catch (InterruptedException e) {
                throw new InterruptedIOException(
                        "Interrupted when restoring a nonexistent table: " + e.getMessage());
            }
            return;
        }

        // Check if the table is disabled
        if (!isTableDisabled(tableName)) {
            throw new TableNotDisabledException(tableName);
        }

        // Take a snapshot of the current state
        String failSafeSnapshotSnapshotName = null;
        if (takeFailSafeSnapshot) {
            failSafeSnapshotSnapshotName = conf.get("hbase.snapshot.restore.failsafe.name",
                    "hbase-failsafe-{snapshot.name}-{restore.timestamp}");
            failSafeSnapshotSnapshotName = failSafeSnapshotSnapshotName.replace("{snapshot.name}", snapshotName)
                    .replace("{table.name}", tableName.toString().replace(TableName.NAMESPACE_DELIM, '.'))
                    .replace("{restore.timestamp}", String.valueOf(EnvironmentEdgeManager.currentTimeMillis()));
            LOG.info("Taking restore-failsafe snapshot: " + failSafeSnapshotSnapshotName);
            snapshot(failSafeSnapshotSnapshotName, tableName);
        }

        try {
            // Restore snapshot
            internalRestoreSnapshot(snapshotName, tableName);
        } catch (IOException e) {
            // Somthing went wrong during the restore...
            // if the pre-restore snapshot is available try to rollback
            if (takeFailSafeSnapshot) {
                try {
                    internalRestoreSnapshot(failSafeSnapshotSnapshotName, tableName);
                    String msg = "Restore snapshot=" + snapshotName + " failed. Rollback to snapshot="
                            + failSafeSnapshotSnapshotName + " succeeded.";
                    LOG.error(msg, e);
                    throw new RestoreSnapshotException(msg, e);
                } catch (IOException ex) {
                    String msg = "Failed to restore and rollback to snapshot=" + failSafeSnapshotSnapshotName;
                    LOG.error(msg, ex);
                    throw new RestoreSnapshotException(msg, e);
                }
            } else {
                throw new RestoreSnapshotException("Failed to restore snapshot=" + snapshotName, e);
            }
        }

        // If the restore is succeeded, delete the pre-restore snapshot
        if (takeFailSafeSnapshot) {
            try {
                LOG.info("Deleting restore-failsafe snapshot: " + failSafeSnapshotSnapshotName);
                deleteSnapshot(failSafeSnapshotSnapshotName);
            } catch (IOException e) {
                LOG.error("Unable to remove the failsafe snapshot: " + failSafeSnapshotSnapshotName, e);
            }
        }
    }

    /**
     * Create a new table by cloning the snapshot content.
     *
     * @param snapshotName name of the snapshot to be cloned
     * @param tableName name of the table where the snapshot will be restored
     * @throws IOException if a remote or network exception occurs
     * @throws TableExistsException if table to be created already exists
     * @throws RestoreSnapshotException if snapshot failed to be cloned
     * @throws IllegalArgumentException if the specified table has not a valid name
     */
    public void cloneSnapshot(final byte[] snapshotName, final byte[] tableName)
            throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
        cloneSnapshot(Bytes.toString(snapshotName), TableName.valueOf(tableName));
    }

    /**
     * Create a new table by cloning the snapshot content.
     *
     * @param snapshotName name of the snapshot to be cloned
     * @param tableName name of the table where the snapshot will be restored
     * @throws IOException if a remote or network exception occurs
     * @throws TableExistsException if table to be created already exists
     * @throws RestoreSnapshotException if snapshot failed to be cloned
     * @throws IllegalArgumentException if the specified table has not a valid name
     */
    public void cloneSnapshot(final byte[] snapshotName, final TableName tableName)
            throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
        cloneSnapshot(Bytes.toString(snapshotName), tableName);
    }

    /**
     * Create a new table by cloning the snapshot content.
     *
     * @param snapshotName name of the snapshot to be cloned
     * @param tableName name of the table where the snapshot will be restored
     * @throws IOException if a remote or network exception occurs
     * @throws TableExistsException if table to be created already exists
     * @throws RestoreSnapshotException if snapshot failed to be cloned
     * @throws IllegalArgumentException if the specified table has not a valid name
     */
    public void cloneSnapshot(final String snapshotName, final String tableName)
            throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
        cloneSnapshot(snapshotName, TableName.valueOf(tableName));
    }

    /**
     * Create a new table by cloning the snapshot content.
     *
     * @param snapshotName name of the snapshot to be cloned
     * @param tableName name of the table where the snapshot will be restored
     * @throws IOException if a remote or network exception occurs
     * @throws TableExistsException if table to be created already exists
     * @throws RestoreSnapshotException if snapshot failed to be cloned
     * @throws IllegalArgumentException if the specified table has not a valid name
     */
    public void cloneSnapshot(final String snapshotName, final TableName tableName)
            throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
        if (tableExists(tableName)) {
            throw new TableExistsException(tableName);
        }
        internalRestoreSnapshot(snapshotName, tableName);
        waitUntilTableIsEnabled(tableName);
    }

    /**
     * Execute a distributed procedure on a cluster.
     *
     * @param signature A distributed procedure is uniquely identified
     * by its signature (default the root ZK node name of the procedure).
     * @param instance The instance name of the procedure. For some procedures, this parameter is
     * optional.
     * @param props Property/Value pairs of properties passing to the procedure
     */
    public void execProcedure(String signature, String instance, Map<String, String> props) throws IOException {
        ProcedureDescription.Builder builder = ProcedureDescription.newBuilder();
        builder.setSignature(signature).setInstance(instance);
        for (Entry<String, String> entry : props.entrySet()) {
            NameStringPair pair = NameStringPair.newBuilder().setName(entry.getKey()).setValue(entry.getValue())
                    .build();
            builder.addConfiguration(pair);
        }

        final ExecProcedureRequest request = ExecProcedureRequest.newBuilder().setProcedure(builder.build())
                .build();
        // run the procedure on the master
        ExecProcedureResponse response = executeCallable(
                new MasterCallable<ExecProcedureResponse>(getConnection()) {
                    @Override
                    public ExecProcedureResponse call(int callTimeout) throws ServiceException {
                        return master.execProcedure(null, request);
                    }
                });

        long start = EnvironmentEdgeManager.currentTimeMillis();
        long max = response.getExpectedTimeout();
        long maxPauseTime = max / this.numRetries;
        int tries = 0;
        LOG.debug("Waiting a max of " + max + " ms for procedure '" + signature + " : " + instance
                + "'' to complete. (max " + maxPauseTime + " ms per retry)");
        boolean done = false;
        while (tries == 0 || ((EnvironmentEdgeManager.currentTimeMillis() - start) < max && !done)) {
            try {
                // sleep a backoff <= pauseTime amount
                long sleep = getPauseTime(tries++);
                sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
                LOG.debug("(#" + tries + ") Sleeping: " + sleep + "ms while waiting for procedure completion.");
                Thread.sleep(sleep);
            } catch (InterruptedException e) {
                throw (InterruptedIOException) new InterruptedIOException("Interrupted").initCause(e);
            }
            LOG.debug("Getting current status of procedure from master...");
            done = isProcedureFinished(signature, instance, props);
        }
        if (!done) {
            throw new IOException("Procedure '" + signature + " : " + instance
                    + "' wasn't completed in expectedTime:" + max + " ms");
        }
    }

    /**
     * Check the current state of the specified procedure.
     * <p>
     * There are three possible states:
     * <ol>
     * <li>running - returns <tt>false</tt></li>
     * <li>finished - returns <tt>true</tt></li>
     * <li>finished with error - throws the exception that caused the procedure to fail</li>
     * </ol>
     * <p>
     *
     * @param signature The signature that uniquely identifies a procedure
     * @param instance The instance name of the procedure
     * @param props Property/Value pairs of properties passing to the procedure
     * @return true if the specified procedure is finished successfully, false if it is still running
     * @throws IOException if the specified procedure finished with error
     */
    public boolean isProcedureFinished(String signature, String instance, Map<String, String> props)
            throws IOException {
        final ProcedureDescription.Builder builder = ProcedureDescription.newBuilder();
        builder.setSignature(signature).setInstance(instance);
        for (Entry<String, String> entry : props.entrySet()) {
            NameStringPair pair = NameStringPair.newBuilder().setName(entry.getKey()).setValue(entry.getValue())
                    .build();
            builder.addConfiguration(pair);
        }
        final ProcedureDescription desc = builder.build();
        return executeCallable(new MasterCallable<IsProcedureDoneResponse>(getConnection()) {
            @Override
            public IsProcedureDoneResponse call(int callTimeout) throws ServiceException {
                return master.isProcedureDone(null, IsProcedureDoneRequest.newBuilder().setProcedure(desc).build());
            }
        }).getDone();
    }

    /**
     * Execute Restore/Clone snapshot and wait for the server to complete (blocking).
     * To check if the cloned table exists, use {@link #isTableAvailable} -- it is not safe to
     * create an HTable instance to this table before it is available.
     * @param snapshotName snapshot to restore
     * @param tableName table name to restore the snapshot on
     * @throws IOException if a remote or network exception occurs
     * @throws RestoreSnapshotException if snapshot failed to be restored
     * @throws IllegalArgumentException if the restore request is formatted incorrectly
     */
    private void internalRestoreSnapshot(final String snapshotName, final TableName tableName)
            throws IOException, RestoreSnapshotException {
        SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName)
                .setTable(tableName.getNameAsString()).build();

        // actually restore the snapshot
        internalRestoreSnapshotAsync(snapshot);

        final IsRestoreSnapshotDoneRequest request = IsRestoreSnapshotDoneRequest.newBuilder().setSnapshot(snapshot)
                .build();
        IsRestoreSnapshotDoneResponse done = IsRestoreSnapshotDoneResponse.newBuilder().setDone(false)
                .buildPartial();
        final long maxPauseTime = 5000;
        int tries = 0;
        while (!done.getDone()) {
            try {
                // sleep a backoff <= pauseTime amount
                long sleep = getPauseTime(tries++);
                sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
                LOG.debug(tries + ") Sleeping: " + sleep + " ms while we wait for snapshot restore to complete.");
                Thread.sleep(sleep);
            } catch (InterruptedException e) {
                throw (InterruptedIOException) new InterruptedIOException("Interrupted").initCause(e);
            }
            LOG.debug("Getting current status of snapshot restore from master...");
            done = executeCallable(new MasterCallable<IsRestoreSnapshotDoneResponse>(getConnection()) {
                @Override
                public IsRestoreSnapshotDoneResponse call(int callTimeout) throws ServiceException {
                    return master.isRestoreSnapshotDone(null, request);
                }
            });
        }
        if (!done.getDone()) {
            throw new RestoreSnapshotException("Snapshot '" + snapshot.getName() + "' wasn't restored.");
        }
    }

    /**
     * Execute Restore/Clone snapshot and wait for the server to complete (asynchronous)
     * <p>
     * Only a single snapshot should be restored at a time, or results may be undefined.
     * @param snapshot snapshot to restore
     * @return response from the server indicating the max time to wait for the snapshot
     * @throws IOException if a remote or network exception occurs
     * @throws RestoreSnapshotException if snapshot failed to be restored
     * @throws IllegalArgumentException if the restore request is formatted incorrectly
     */
    private RestoreSnapshotResponse internalRestoreSnapshotAsync(final SnapshotDescription snapshot)
            throws IOException, RestoreSnapshotException {
        ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);

        final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot).build();

        // run the snapshot restore on the master
        return executeCallable(new MasterCallable<RestoreSnapshotResponse>(getConnection()) {
            @Override
            public RestoreSnapshotResponse call(int callTimeout) throws ServiceException {
                return master.restoreSnapshot(null, request);
            }
        });
    }

    /**
     * List completed snapshots.
     * @return a list of snapshot descriptors for completed snapshots
     * @throws IOException if a network error occurs
     */
    public List<SnapshotDescription> listSnapshots() throws IOException {
        return executeCallable(new MasterCallable<List<SnapshotDescription>>(getConnection()) {
            @Override
            public List<SnapshotDescription> call(int callTimeout) throws ServiceException {
                return master.getCompletedSnapshots(null, GetCompletedSnapshotsRequest.newBuilder().build())
                        .getSnapshotsList();
            }
        });
    }

    /**
     * List all the completed snapshots matching the given regular expression.
     *
     * @param regex The regular expression to match against
     * @return - returns a List of SnapshotDescription
     * @throws IOException if a remote or network exception occurs
     */
    public List<SnapshotDescription> listSnapshots(String regex) throws IOException {
        return listSnapshots(Pattern.compile(regex));
    }

    /**
     * List all the completed snapshots matching the given pattern.
     *
     * @param pattern The compiled regular expression to match against
     * @return - returns a List of SnapshotDescription
     * @throws IOException if a remote or network exception occurs
     */
    public List<SnapshotDescription> listSnapshots(Pattern pattern) throws IOException {
        List<SnapshotDescription> matched = new LinkedList<SnapshotDescription>();
        List<SnapshotDescription> snapshots = listSnapshots();
        for (SnapshotDescription snapshot : snapshots) {
            if (pattern.matcher(snapshot.getName()).matches()) {
                matched.add(snapshot);
            }
        }
        return matched;
    }

    /**
     * Delete an existing snapshot.
     * @param snapshotName name of the snapshot
     * @throws IOException if a remote or network exception occurs
     */
    public void deleteSnapshot(final byte[] snapshotName) throws IOException {
        deleteSnapshot(Bytes.toString(snapshotName));
    }

    /**
     * Delete an existing snapshot.
     * @param snapshotName name of the snapshot
     * @throws IOException if a remote or network exception occurs
     */
    public void deleteSnapshot(final String snapshotName) throws IOException {
        // make sure the snapshot is possibly valid
        TableName.isLegalFullyQualifiedTableName(Bytes.toBytes(snapshotName));
        // do the delete
        executeCallable(new MasterCallable<Void>(getConnection()) {
            @Override
            public Void call(int callTimeout) throws ServiceException {
                master.deleteSnapshot(null, DeleteSnapshotRequest.newBuilder()
                        .setSnapshot(SnapshotDescription.newBuilder().setName(snapshotName).build()).build());
                return null;
            }
        });
    }

    /**
     * Delete existing snapshots whose names match the pattern passed.
     * @param regex The regular expression to match against
     * @throws IOException if a remote or network exception occurs
     */
    public void deleteSnapshots(final String regex) throws IOException {
        deleteSnapshots(Pattern.compile(regex));
    }

    /**
     * Delete existing snapshots whose names match the pattern passed.
     * @param pattern pattern for names of the snapshot to match
     * @throws IOException if a remote or network exception occurs
     */
    public void deleteSnapshots(final Pattern pattern) throws IOException {
        List<SnapshotDescription> snapshots = listSnapshots(pattern);
        for (final SnapshotDescription snapshot : snapshots) {
            // do the delete
            executeCallable(new MasterCallable<Void>(getConnection()) {
                @Override
                public Void call(int callTimeout) throws ServiceException {
                    this.master.deleteSnapshot(null,
                            DeleteSnapshotRequest.newBuilder().setSnapshot(snapshot).build());
                    return null;
                }
            });
        }
    }

    /**
     * Parent of {@link MasterCallable} and {@link MasterCallable}.
     * Has common methods.
     * @param <V>
     */
    abstract static class MasterCallable<V> implements RetryingCallable<V>, Closeable {
        protected HConnection connection;
        protected MasterKeepAliveConnection master;

        public MasterCallable(final HConnection connection) {
            this.connection = connection;
        }

        @Override
        public void prepare(boolean reload) throws IOException {
            this.master = this.connection.getKeepAliveMasterService();
        }

        @Override
        public void close() throws IOException {
            // The above prepare could fail but this would still be called though masterAdmin is null
            if (this.master != null)
                this.master.close();
        }

        @Override
        public void throwable(Throwable t, boolean retrying) {
        }

        @Override
        public String getExceptionMessageAdditionalDetail() {
            return "";
        }

        @Override
        public long sleep(long pause, int tries) {
            return ConnectionUtils.getPauseTime(pause, tries);
        }
    }

    private <V> V executeCallable(MasterCallable<V> callable) throws IOException {
        RpcRetryingCaller<V> caller = rpcCallerFactory.newCaller();
        try {
            return caller.callWithRetries(callable, operationTimeout);
        } finally {
            callable.close();
        }
    }

    /**
     * Creates and returns a {@link com.google.protobuf.RpcChannel} instance
     * connected to the active master.
     *
     * <p>
     * The obtained {@link com.google.protobuf.RpcChannel} instance can be used to access a published
     * coprocessor {@link com.google.protobuf.Service} using standard protobuf service invocations:
     * </p>
     *
     * <div style="background-color: #cccccc; padding: 2px">
     * <blockquote><pre>
     * CoprocessorRpcChannel channel = myAdmin.coprocessorService();
     * MyService.BlockingInterface service = MyService.newBlockingStub(channel);
     * MyCallRequest request = MyCallRequest.newBuilder()
     *     ...
     *     .build();
     * MyCallResponse response = service.myCall(null, request);
     * </pre></blockquote></div>
     *
     * @return A MasterCoprocessorRpcChannel instance
     */
    public CoprocessorRpcChannel coprocessorService() {
        return new MasterCoprocessorRpcChannel(connection);
    }
}