com.cloudera.impala.service.ZooKeeperSession.java Source code

Java tutorial

Introduction

Here is the source code for com.cloudera.impala.service.ZooKeeperSession.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.cloudera.impala.service;

import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;

import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.api.ACLProvider;
import org.apache.curator.framework.imps.CuratorFrameworkState;
import org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.curator.framework.recipes.cache.PathChildrenCache;
import org.apache.curator.framework.recipes.cache.PathChildrenCache.StartMode;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.authentication.util.KerberosUtil;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.ZooDefs.Perms;
import org.apache.zookeeper.client.ZooKeeperSaslClient;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.cloudera.impala.common.InternalException;
import com.cloudera.impala.thrift.TMembershipUpdate;
import com.cloudera.impala.thrift.TMembershipUpdateType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;

/**
 * Wrapper around curator to manage zookeeper sessions. This class also
 * handles membership.
 */
public class ZooKeeperSession implements Closeable {
    private static final Logger LOGGER = LoggerFactory.getLogger(ZooKeeperSession.class.getName());

    // Location of zookeeper quorem.
    public static final String ZOOKEEPER_CONNECTION_STRING_CONF = "recordservice.zookeeper.connectString";
    public static final String ZOOKEEPER_CONNECT_TIMEOUTMILLIS_CONF = "recordservice..zookeeper.connectTimeoutMillis";

    // Root zookeeper directory.
    public static final String ZOOKEEPER_ZNODE_CONF = "recordservice.zookeeper.znode";
    public static final String ZOOKEEPER_ZNODE_DEFAULT = "/recordservice";

    // Acl to use when creating znodes, including planner, worker or delegation tokens.
    public static final String ZOOKEEPER_STORE_ACL_CONF = "recordservice.zookeeper.acl";

    // Extra Acl to use for the "planners" directory. In default this is set to
    // be {@link Ids#READ_ACL_UNSAFE} to allow planner membership discovery. One can
    // set it to be empty to disallow such.
    public static final String ZOOKEEPER_STORE_PLANNERS_ACL_CONF = "recordservice.zookeeper.planners.acl";

    // Zookeeper directory locations
    private static final String PLANNER_MEMBERSHIP_ZNODE = "planners";
    private static final String WORKER_MEMBERSHIP_ZNODE = "workers";

    // ZooKeeper property name to pick the correct JAAS conf section
    private static final String SASL_LOGIN_CONTEXT_NAME = "RecordServiceZooKeeperClient";

    // Zookeeper connection string (host/port of quorem).
    private final String zkConnectString_;

    // Session timeout.
    private final int connectTimeoutMillis_;

    // Root znode directory.
    private String rootNode_ = "";

    // Current Zookeeper session.
    private volatile CuratorFramework zkSession_;

    // Watcher for the planner membership znode.
    private PathChildrenCache plannerMembership_;

    // Watcher of the worker membership znode.
    private PathChildrenCache workerMembership_;

    // Planner membership. Only maintained for planners. (runningPlanner_ = true)
    // Access to this needs to be thread safe. If this and zkSession_ need to be
    // locked, lock zkSession_ first.
    private final Set<String> planners_ = Sets.newHashSet();

    // Worker membership. Only maintained for planners. (runningPlanner_ = true)
    // Access to this needs to be thread safe. If this and zkSession_ need to be
    // locked, lock zkSession_ first.
    private final Set<String> workers_ = Sets.newHashSet();

    // List of callbacks to invoke when the zk sessions is (re) created.
    private final List<NewSessionCb> newSessionsCbs_ = Lists.newArrayList();

    // ACLs to use when creating new znodes.
    private List<ACL> newNodeAcl_;

    // ACLs to use when creating new znodes for the "planners" dir.
    private List<ACL> plannersAcl_;

    // ID for this service.
    private final String id_;

    // Planner and worker ports. Greater than 0 if set.
    private final int plannerPort_;
    private final int workerPort_;

    /**
     * ACLProvider permissions will be used in case parent dirs need to be created
     */
    private final ACLProvider aclDefaultProvider_ = new ACLProvider() {
        @Override
        public List<ACL> getDefaultAcl() {
            return newNodeAcl_;
        }

        @Override
        public List<ACL> getAclForPath(String path) {
            return getDefaultAcl();
        }
    };

    /**
     * Connects to zookeeper and handles maintaining membership, without Kerberos.
     * Note: this can only be called when either planner or worker is running.
     * The client of this class is responsible for checking that.
     * @param conf
     * @param id - The ID for this server. This should be unique among
     *    all instances of the service.
     * @param plannerPort - If greater than 0, running the planner service.
     * @param workerPort - If greater than 0, running the worker service.
     */
    public ZooKeeperSession(Configuration conf, String id, int plannerPort, int workerPort) throws IOException {
        this(conf, id, null, null, plannerPort, workerPort);
    }

    /**
     * Connects to zookeeper and handles maintaining membership.
     * Note: this can only be called when either planner or worker is running.
     * The client of this class is responsible for checking that.
     * @param conf
     * @param id - The ID for this server. This should be unique among
     *    all instances of the service.
     * @param principal - The Kerberos principal to use. If not null and not empty,
     *    ZooKeeper nodes will be secured with Kerberos.
     * @param keytabPath - The path to the keytab file. Only used when a valid
     *    principal is provided.
     * @param plannerPort - If greater than 0, running the planner service.
     * @param workerPort - If greater than 0, running the worker service.
     */
    public ZooKeeperSession(Configuration conf, String id, String principal, String keytabPath, int plannerPort,
            int workerPort) throws IOException {
        id_ = id;
        plannerPort_ = plannerPort;
        workerPort_ = workerPort;

        zkConnectString_ = conf.get(ZOOKEEPER_CONNECTION_STRING_CONF);
        if (zkConnectString_ == null || zkConnectString_.trim().isEmpty()) {
            throw new IllegalArgumentException(
                    "Zookeeper connect string has to be specified through " + ZOOKEEPER_CONNECTION_STRING_CONF);
        }
        LOGGER.info("Connecting to zookeeper at: " + zkConnectString_ + " with id: " + id_);

        connectTimeoutMillis_ = conf.getInt(ZOOKEEPER_CONNECT_TIMEOUTMILLIS_CONF,
                CuratorFrameworkFactory.builder().getConnectionTimeoutMs());

        if (principal != null && !principal.isEmpty()) {
            newNodeAcl_ = Ids.CREATOR_ALL_ACL;
        } else {
            newNodeAcl_ = Ids.OPEN_ACL_UNSAFE;
        }

        String aclStr = conf.get(ZOOKEEPER_STORE_ACL_CONF, null);
        LOGGER.info("Zookeeper acl: " + aclStr);
        if (StringUtils.isNotBlank(aclStr))
            newNodeAcl_ = parseACLs(aclStr);

        plannersAcl_ = Ids.READ_ACL_UNSAFE;
        String plannersAclStr = conf.get(ZOOKEEPER_STORE_PLANNERS_ACL_CONF, null);
        LOGGER.info("Zookeeper planners acl: " + plannersAclStr);
        if (plannersAclStr != null)
            plannersAcl_ = parseACLs(plannersAclStr);

        rootNode_ = conf.get(ZOOKEEPER_ZNODE_CONF, ZOOKEEPER_ZNODE_DEFAULT);
        LOGGER.info("Zookeeper root: " + rootNode_);

        // Install the JAAS Configuration for the runtime, if Kerberos is enabled.
        if (principal != null && !principal.isEmpty()) {
            setupJAASConfig(principal, keytabPath);
        }
        initMembershipPaths();
    }

    /**
     * Closes the underlying ZK session. This should be called if the server is stopped
     * to remove this node from ZK more quickly (otherwise it will timeout eventually).
     */
    public void close() throws IOException {
        if (plannerMembership_ != null) {
            plannerMembership_.close();
            synchronized (planners_) {
                planners_.clear();
            }
        }
        if (workerMembership_ != null) {
            workerMembership_.close();
            synchronized (workers_) {
                workers_.clear();
            }
        }
        if (zkSession_ != null)
            zkSession_.close();
    }

    /**
     * Callback that is invoked when a sessions restarts.
     */
    public interface NewSessionCb {
        void newSession() throws IOException;
    }

    /**
     * Adds a callback to be invoked when a new ZK session is created.
     */
    public void addNewSessionCb(NewSessionCb cb) {
        synchronized (newSessionsCbs_) {
            newSessionsCbs_.add(cb);
        }
    }

    /**
     * Returns the current Curator session, reconnecting if necessary. As part of this,
     * if the connection is re(created), membership is updated accordingly.
     */
    public CuratorFramework getSession() throws IOException {
        if (zkSession_ == null || zkSession_.getState() == CuratorFrameworkState.STOPPED) {
            boolean recreatedSession = false;
            synchronized (this) {
                if (zkSession_ == null || zkSession_.getState() == CuratorFrameworkState.STOPPED) {
                    zkSession_ = CuratorFrameworkFactory.builder().connectString(zkConnectString_)
                            .connectionTimeoutMs(connectTimeoutMillis_).aclProvider(aclDefaultProvider_)
                            .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build();
                    zkSession_.start();
                    recreatedSession = true;
                }
            }
            // Just started a new session, register membership.
            if (recreatedSession) {
                if (runningPlanner())
                    registerMembership(zkSession_, true);
                if (runningWorker())
                    registerMembership(zkSession_, false);

                synchronized (newSessionsCbs_) {
                    for (NewSessionCb cb : newSessionsCbs_) {
                        cb.newSession();
                    }
                }
            }
        }
        return zkSession_;
    }

    public String rootZnode() {
        return rootNode_;
    }

    public List<ACL> newNodeAcl() {
        return newNodeAcl_;
    }

    /**
     * Returns a path relative (appended to) the root znode path.
     */
    public String getRelativePath(String path) {
        return rootNode_ + "/" + path;
    }

    /**
     * Returns the set of planner or workers, depending on whether 'planner' is true
     * or false. Only callable for planner services.
     */
    public Set<String> getMembership(boolean planner) {
        if (!runningPlanner()) {
            // TODO: we can either always listen to the membership or just get the list on
            // demand for worker-only services. Currently, there is no use case for this.
            throw new IllegalStateException("Can only call when running planner.");
        }
        Set<String> members = planner ? planners_ : workers_;
        synchronized (members) {
            return ImmutableSet.copyOf(members);
        }
    }

    /**
     * Returns the size for planners or workers, depending on whether 'planner' is
     * true or false.
     */
    public int getMembershipSize(boolean planner) {
        return planner ? planners_.size() : workers_.size();
    }

    /**
     * Create a path if it does not already exist ("mkdir -p")
     * @param path string with '/' separator
     * @param acl list of ACL entries
     */
    public void ensurePath(String path, List<ACL> acl) throws IOException {
        try {
            CuratorFramework zk = getSession();
            String node = zk.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).withACL(acl)
                    .forPath(path);
            LOGGER.info("Created path: {} ", node);
        } catch (KeeperException.NodeExistsException e) {
            // node already exists
            LOGGER.debug("ZNode already exists: {} ", path);
        } catch (Exception e) {
            throw new IOException("Error creating path " + path, e);
        }
    }

    /**
     * Deletes the znode at 'path'. Idempotent.
     */
    public void zkDelete(String path) throws IOException {
        CuratorFramework zk = getSession();
        try {
            zk.delete().forPath(path);
        } catch (KeeperException.NoNodeException ex) {
            // already deleted
        } catch (Exception e) {
            throw new IOException("Error deleting " + path, e);
        }
    }

    /**
     * Lists all the children in 'path'.
     */
    public List<String> zkGetChildren(String path) throws IOException {
        CuratorFramework zk = getSession();
        try {
            return zk.getChildren().forPath(path);
        } catch (Exception e) {
            throw new IOException("Error getting children for " + path, e);
        }
    }

    /**
     * Returns the data at 'nodePath'. Returns null if there is no data (but
     * the path exists).
     */
    public byte[] zkGetData(String nodePath) throws IOException {
        CuratorFramework zk = getSession();
        try {
            return zk.getData().forPath(nodePath);
        } catch (KeeperException.NoNodeException ex) {
            return null;
        } catch (Exception e) {
            throw new IOException("Error reading " + nodePath, e);
        }
    }

    /**
     * Set ACL for 'path' to be 'acl. Throws IOException if error happens.
     */
    public void zkSetACL(String path, List<ACL> acl) throws IOException {
        try {
            CuratorFramework zk = getSession();
            zk.setACL().withACL(acl).forPath(path);
        } catch (Exception e) {
            throw new IOException("Error setting acl " + acl + " for path " + path, e);
        }
    }

    /**
     * Updates the worker or planner membership, including calling into the BE.
     */
    private void updateMembership(TMembershipUpdateType type, String member, boolean planner)
            throws InternalException {
        PathChildrenCache membership = planner ? plannerMembership_ : workerMembership_;
        Set<String> members = planner ? planners_ : workers_;
        synchronized (members) {
            TMembershipUpdate update = new TMembershipUpdate(planner, type, new ArrayList<String>());
            switch (type) {
            case ADD:
                Preconditions.checkNotNull(member);
                members.add(member);
                update.membership.add(member);
                break;
            case REMOVE:
                Preconditions.checkNotNull(member);
                members.remove(member);
                update.membership.add(member);
                break;
            case FULL_LIST:
                Preconditions.checkState(member == null);
                members.clear();
                for (ChildData d : membership.getCurrentData()) {
                    members.add(d.getPath());
                }
                update.membership.addAll(members);
                break;
            default:
                Preconditions.checkState(false);
            }
            FeSupport.UpdateMembership(update);
        }
    }

    /**
     * Adds this service in the planner and/or membership directory. This is done
     * in Zookeeper by creating a ephemeral znode with 'id'. ephemeral means that
     * if the session closes (zookeeper manages timeouts), the znode is deleted.
     */
    private void registerMembership(CuratorFramework zk, boolean planner) throws IOException {
        if (planner) {
            // This is the planner meaning we also want to listen to the membership.
            initMembership(zk, true);
            initMembership(zk, false);
        }

        String serviceId = id_ + ":" + (planner ? plannerPort_ : workerPort_);
        String path = getRelativePath(
                (planner ? PLANNER_MEMBERSHIP_ZNODE : WORKER_MEMBERSHIP_ZNODE) + "/" + serviceId);
        // First delete the current path. This is required for correctness. For example, if
        // this service is restarted, the previous znode might still be there (hasn't timed
        // out yet) and this function would not be able to create the znode with this
        // session.
        zkDelete(path);
        try {
            String node = zk.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).withACL(newNodeAcl_)
                    .forPath(path);
            LOGGER.info("Created path: {} ", node);
        } catch (Exception e) {
            throw new IOException("Could not create znode at " + path, e);
        }
    }

    /**
     * Initiate membership for planner or worker, depending on whether 'planner' is
     * true or false. This reconstruct the membership information based on the current
     * state, and add listener for future membership updates.
     */
    private void initMembership(CuratorFramework zk, final boolean planner) throws IOException {
        PathChildrenCache membership = planner ? plannerMembership_ : workerMembership_;
        Set<String> members = planner ? planners_ : workers_;
        String znode = planner ? PLANNER_MEMBERSHIP_ZNODE : WORKER_MEMBERSHIP_ZNODE;
        synchronized (zkSession_) {
            if (membership != null)
                membership.close();
            membership = new PathChildrenCache(zk, getRelativePath(znode), true);
            if (planner)
                plannerMembership_ = membership;
            else
                workerMembership_ = membership;
            try {
                Preconditions.checkState(members.isEmpty());
                membership.start(StartMode.BUILD_INITIAL_CACHE);
                updateMembership(TMembershipUpdateType.FULL_LIST, null, planner);
            } catch (Exception e) {
                throw new IOException("Could not watch " + (planner ? "planner" : "worker") + " membership.", e);
            }
        }

        // Add a listener to the worker or planner directory for updates.
        membership.getListenable().addListener(new PathChildrenCacheListener() {
            @Override
            public void childEvent(CuratorFramework zk, PathChildrenCacheEvent arg) throws Exception {
                synchronized (zkSession_) {
                    // This session is no longer the active one, ignore this update.
                    if (zk != zkSession_)
                        return;
                    switch (arg.getType()) {
                    case CHILD_ADDED:
                        updateMembership(TMembershipUpdateType.ADD, arg.getData().getPath(), planner);
                        break;
                    case CHILD_REMOVED:
                        updateMembership(TMembershipUpdateType.REMOVE, arg.getData().getPath(), planner);
                        break;

                    case INITIALIZED:
                    default:
                        // TODO: what do the other types mean? Should be safe to just
                        // reconstruct.
                        LOGGER.info("Found unexpected event type {}. Reconstruct the membership.", arg.getType());
                        updateMembership(TMembershipUpdateType.FULL_LIST, null, planner);
                        break;
                    }
                }
            }
        });
    }

    /**
     * Initializes the paths for storing membership. This creates the directories
     * for planners and workers.
     */
    private void initMembershipPaths() throws IOException {
        ensurePath(getRelativePath(PLANNER_MEMBERSHIP_ZNODE), newNodeAcl_);
        List<ACL> acl = new ArrayList<ACL>(newNodeAcl_);
        acl.addAll(plannersAcl_);
        zkSetACL(getRelativePath(PLANNER_MEMBERSHIP_ZNODE), acl);
        ensurePath(getRelativePath(WORKER_MEMBERSHIP_ZNODE), newNodeAcl_);
    }

    /**
     * Whether this is running as a planner service.
     */
    private boolean runningPlanner() {
        return plannerPort_ > 0;
    }

    /**
     * Whether this is running as a worker service.
     */
    private boolean runningWorker() {
        return workerPort_ > 0;
    }

    /**
     * Parse comma separated list of ACL entries to secure generated nodes, e.g.
     * <code>sasl:recordservice/host1@MY.DOMAIN:cdrwa,</code>
     */
    private static List<ACL> parseACLs(String aclString) {
        String[] aclComps = StringUtils.splitByWholeSeparator(aclString, ",");
        List<ACL> acl = new ArrayList<ACL>(aclComps.length);
        for (String a : aclComps) {
            if (StringUtils.isBlank(a)) {
                continue;
            }
            a = a.trim();
            // from ZooKeeperMain private method
            int firstColon = a.indexOf(':');
            int lastColon = a.lastIndexOf(':');
            if (firstColon == -1 || lastColon == -1 || firstColon == lastColon) {
                LOGGER.error(a + " does not have the form scheme:id:perm");
                continue;
            }
            ACL newAcl = new ACL();
            newAcl.setId(new Id(a.substring(0, firstColon), a.substring(firstColon + 1, lastColon)));
            newAcl.setPerms(getPermFromString(a.substring(lastColon + 1)));
            acl.add(newAcl);
        }
        return acl;
    }

    /**
     * Parse ACL permission string, from ZooKeeperMain private method
     */
    private static int getPermFromString(String permString) {
        int perm = 0;
        for (int i = 0; i < permString.length(); i++) {
            switch (permString.charAt(i)) {
            case 'r':
                perm |= ZooDefs.Perms.READ;
                break;
            case 'w':
                perm |= ZooDefs.Perms.WRITE;
                break;
            case 'c':
                perm |= ZooDefs.Perms.CREATE;
                break;
            case 'd':
                perm |= ZooDefs.Perms.DELETE;
                break;
            case 'a':
                perm |= ZooDefs.Perms.ADMIN;
                break;
            default:
                LOGGER.error("Unknown perm type: " + permString.charAt(i));
            }
        }
        return perm;
    }

    /**
     * A JAAS configuration for ZooKeeper clients intended to use for SASL
     * Kerberos.
     */
    private static class JaasConfiguration extends javax.security.auth.login.Configuration {
        // Current installed Configuration
        private final javax.security.auth.login.Configuration baseConfig = javax.security.auth.login.Configuration
                .getConfiguration();
        private final String loginContextName;
        private final String principal;
        private final String keyTabFile;

        public JaasConfiguration(String hiveLoginContextName, String principal, String keyTabFile) {
            this.loginContextName = hiveLoginContextName;
            this.principal = principal;
            this.keyTabFile = keyTabFile;
        }

        @Override
        public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
            if (loginContextName.equals(appName)) {
                Map<String, String> krbOptions = new HashMap<String, String>();
                krbOptions.put("doNotPrompt", "true");
                krbOptions.put("storeKey", "true");
                krbOptions.put("useKeyTab", "true");
                krbOptions.put("principal", principal);
                krbOptions.put("keyTab", keyTabFile);
                krbOptions.put("refreshKrb5Config", "true");
                AppConfigurationEntry hiveZooKeeperClientEntry = new AppConfigurationEntry(
                        KerberosUtil.getKrb5LoginModuleName(), LoginModuleControlFlag.REQUIRED, krbOptions);
                return new AppConfigurationEntry[] { hiveZooKeeperClientEntry };
            }
            // Try the base config
            if (baseConfig != null) {
                return baseConfig.getAppConfigurationEntry(appName);
            }
            return null;
        }
    }

    /**
     * Setup configuration to connect to Zookeeper using kerberos.
     */
    private void setupJAASConfig(String principal, String keytab) throws IOException {
        Preconditions.checkArgument(principal != null && !principal.isEmpty());
        if (keytab == null || keytab.trim().isEmpty()) {
            throw new IOException("Keytab must be set to connect using kerberos.");
        }
        LOGGER.debug("Authenticating with principal {} and keytab {}", principal, keytab);
        System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY, SASL_LOGIN_CONTEXT_NAME);
        principal = SecurityUtil.getServerPrincipal(principal, "0.0.0.0");
        JaasConfiguration jaasConf = new JaasConfiguration(SASL_LOGIN_CONTEXT_NAME, principal, keytab);
        // Install the Configuration in the runtime.
        javax.security.auth.login.Configuration.setConfiguration(jaasConf);
    }
}