org.apache.sentry.tests.e2e.solr.db.integration.AbstractSolrSentryTestWithDbProvider.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.sentry.tests.e2e.solr.db.integration.AbstractSolrSentryTestWithDbProvider.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.sentry.tests.e2e.solr.db.integration;

import static org.apache.sentry.core.model.search.SearchModelAuthorizable.AuthorizableType.Collection;

import java.io.File;
import java.io.FileOutputStream;
import java.util.Comparator;
import java.util.List;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.TimeoutException;

import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.sentry.SentryUserException;
import org.apache.sentry.binding.solr.HdfsTestUtil;
import org.apache.sentry.binding.solr.conf.SolrAuthzConf.AuthzConfVars;
import org.apache.sentry.core.common.Action;
import org.apache.sentry.core.model.search.SearchConstants;
import org.apache.sentry.provider.db.generic.SentryGenericProviderBackend;
import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClient;
import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClientFactory;
import org.apache.sentry.provider.db.generic.service.thrift.TAuthorizable;
import org.apache.sentry.provider.db.generic.service.thrift.TSentryGrantOption;
import org.apache.sentry.provider.db.generic.service.thrift.TSentryPrivilege;
import org.apache.sentry.provider.file.LocalGroupResourceAuthorizationProvider;
import org.apache.sentry.provider.file.PolicyFile;
import org.apache.sentry.service.thrift.SentryService;
import org.apache.sentry.service.thrift.SentryServiceFactory;
import org.apache.sentry.service.thrift.ServiceConstants.ClientConfig;
import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
import org.apache.sentry.tests.e2e.solr.AbstractSolrSentryTestBase;
import org.apache.sentry.tests.e2e.solr.ModifiableUserAuthenticationFilter;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;

/**
 * This class used to test the Solr integration with DB store.
 * It will set up a miniSolrCloud, miniHDFS and Sentry service in a JVM process.
 */
public class AbstractSolrSentryTestWithDbProvider extends AbstractSolrSentryTestBase {
    private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSolrSentryTestWithDbProvider.class);

    protected static final String SERVER_HOST = NetUtils.createSocketAddr("localhost:80").getAddress()
            .getCanonicalHostName();
    protected static final int PORT = 8038;
    protected static final String ADMIN_GROUP = "admin_group";
    protected static final String ADMIN_ROLE = "admin_role";
    protected static final String ADMIN_COLLECTION_NAME = "admin";
    protected static final String COMPONENT_SOLR = "solr";
    protected static final String CLUSTER_NAME = SearchConstants.SENTRY_SEARCH_CLUSTER_DEFAULT;

    protected static final Configuration conf = new Configuration(false);

    protected static SentryService server;
    protected static SentryGenericServiceClient client;

    protected static File baseDir;
    protected static File hdfsDir;
    protected static File dbDir;
    protected static File policyFilePath;
    protected static File sentrySitePath;

    protected static PolicyFile policyFile;

    /**
     * Overwrite the method from super class AbstractSolrSentryTestBase
     * take over the management of miniSolrCloudCluster and dfsCluster
     */
    @BeforeClass
    public static void beforeTestSimpleSolrEndToEnd() throws Exception {
        setupConf();
        startHDFS();
        startSolrWithDbProvider();
        startSentryService();
        connectToSentryService();
        setGroupsAndRoles();
    }

    @AfterClass
    public static void teardownClass() throws Exception {
        stopAllService();
        FileUtils.deleteDirectory(baseDir);
        unsetSystemProperties();
    }

    public static void setupConf() throws Exception {
        baseDir = createTempDir();
        hdfsDir = new File(baseDir, "hdfs");
        dbDir = new File(baseDir, "sentry_policy_db");
        policyFilePath = new File(baseDir, "local_policy_file.ini");
        sentrySitePath = new File(baseDir, "sentry-site.xml");
        policyFile = new PolicyFile();

        conf.set(ServerConfig.SECURITY_MODE, ServerConfig.SECURITY_MODE_NONE);
        conf.set(ServerConfig.SENTRY_VERIFY_SCHEM_VERSION, "false");
        conf.set(ServerConfig.ADMIN_GROUPS, ADMIN_GROUP + ",solr");
        conf.set(ServerConfig.RPC_ADDRESS, SERVER_HOST);
        conf.set(ServerConfig.RPC_PORT, String.valueOf(PORT));
        conf.set(ServerConfig.SENTRY_STORE_JDBC_URL,
                "jdbc:derby:;databaseName=" + dbDir.getPath() + ";create=true");
        conf.set(ServerConfig.SENTRY_STORE_JDBC_PASS, "dummy");
        conf.set(ServerConfig.SENTRY_STORE_GROUP_MAPPING_RESOURCE, policyFilePath.getPath());
        server = new SentryServiceFactory().create(conf);

        conf.set(ClientConfig.SERVER_RPC_ADDRESS, server.getAddress().getHostName());
        conf.set(ClientConfig.SERVER_RPC_PORT, String.valueOf(server.getAddress().getPort()));
        conf.set(ServerConfig.SENTRY_STORE_GROUP_MAPPING, ServerConfig.SENTRY_STORE_LOCAL_GROUP_MAPPING);
        conf.set(AuthzConfVars.AUTHZ_PROVIDER.getVar(), LocalGroupResourceAuthorizationProvider.class.getName());
        conf.set(AuthzConfVars.AUTHZ_PROVIDER_BACKEND.getVar(), SentryGenericProviderBackend.class.getName());
        conf.set(AuthzConfVars.AUTHZ_PROVIDER_RESOURCE.getVar(), policyFilePath.getPath());
    }

    public static File createTempDir() {
        File baseDir = new File(System.getProperty("java.io.tmpdir"));
        String baseName = "solr-integration-db-";
        File tempDir = new File(baseDir, baseName + UUID.randomUUID().toString());
        if (tempDir.mkdir()) {
            return tempDir;
        }
        throw new IllegalStateException("Failed to create temp directory");
    }

    public static void configureWithSolr() throws Exception {
        conf.set(ServerConfig.SECURITY_USE_UGI_TRANSPORT, "true");
        //save configuration to sentry-site.xml
        conf.writeXml(new FileOutputStream(sentrySitePath));
        setSystemProperties();
        extraRequestFilters = new TreeMap<Class, String>(new Comparator<Class>() {
            // There's only one class, make this as simple as possible
            @Override
            public int compare(Class o1, Class o2) {
                return 0;
            }

            @Override
            public boolean equals(Object obj) {
                return true;
            }
        });
        extraRequestFilters.put(ModifiableUserAuthenticationFilter.class, "*");

        //set the solr for the loginUser and belongs to solr group
        addGroupsToUser("solr", "solr");
        UserGroupInformation
                .setLoginUser(UserGroupInformation.createUserForTesting("solr", new String[] { "solr" }));
    }

    public static void startHDFS() throws Exception {
        dfsCluster = HdfsTestUtil.setupClass(hdfsDir.getPath());
        conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY,
                dfsCluster.getFileSystem().getConf().get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY));
    }

    public static void startSolrWithDbProvider() throws Exception {
        LOGGER.info("starting Solr authorization via Sentry Service");
        configureWithSolr();
        miniSolrCloudCluster = new MiniSolrCloudCluster(NUM_SERVERS, null,
                new File(RESOURCES_DIR, "solr-no-core.xml"), null, extraRequestFilters);
    }

    public static void startSentryService() throws Exception {
        server.start();
        final long start = System.currentTimeMillis();
        while (!server.isRunning()) {
            Thread.sleep(1000);
            if (System.currentTimeMillis() - start > 60000L) {
                throw new TimeoutException("Server did not start after 60 seconds");
            }
        }
    }

    public static void connectToSentryService() throws Exception {
        client = SentryGenericServiceClientFactory.create(conf);
    }

    public static void stopAllService() throws Exception {
        if (miniSolrCloudCluster != null) {
            miniSolrCloudCluster.shutdown();
            miniSolrCloudCluster = null;
        }
        if (dfsCluster != null) {
            HdfsTestUtil.teardownClass(dfsCluster);
            dfsCluster = null;
        }
        if (client != null) {
            client.close();
            client = null;
        }
        if (server != null) {
            server.stop();
            server = null;
        }
    }

    public static void addGroupsToUser(String user, String... groupNames) {
        policyFile.addGroupsToUser(user, groupNames);
    }

    public static void writePolicyFile() throws Exception {
        policyFile.write(policyFilePath);
        FileSystem clusterFs = dfsCluster.getFileSystem();
        clusterFs.copyFromLocalFile(false, new Path(policyFilePath.getPath()), new Path(policyFilePath.getPath()));
    }

    public static void setSystemProperties() throws Exception {
        System.setProperty("solr.xml.persist", "true");
        // Disable the block cache because we can run out of memory
        // on a MiniCluster.
        System.setProperty("solr.hdfs.blockcache.enabled", "false");
        System.setProperty("solr.hdfs.home", dfsCluster.getURI().toString() + "/solr");
        System.setProperty("solr.authorization.sentry.site",
                sentrySitePath.toURI().toURL().toString().substring("file:".length()));
    }

    public static void unsetSystemProperties() {
        System.clearProperty("solr.xml.persist");
        System.clearProperty("solr.hdfs.blockcache.enabled");
        System.clearProperty("solr.hdfs.home");
        System.clearProperty("solr.authorization.sentry.site");
    }

    public static void setGroupsAndRoles() throws Exception {
        /**set local group mapping
         * user0->group0->role0
         * user1->group1->role1
         * user2->group2->role2
         * user3->group3->role3
         */
        String[] users = { "user0", "user1", "user2", "user3" };
        String[] groups = { "group0", "group1", "group2", "group3" };
        String[] roles = { "role0", "role1", "role2", "role3" };

        for (int i = 0; i < users.length; i++) {
            addGroupsToUser(users[i], groups[i]);
        }
        addGroupsToUser(ADMIN_USER, ADMIN_GROUP);
        writePolicyFile();

        for (int i = 0; i < roles.length; i++) {
            client.createRole(ADMIN_USER, roles[i], COMPONENT_SOLR);
            client.addRoleToGroups(ADMIN_USER, roles[i], COMPONENT_SOLR, Sets.newHashSet(groups[i]));
        }

        /**
         * user[admin]->group[admin]->role[admin]
         * grant ALL privilege on collection ALL to role admin
         */
        client.createRole(ADMIN_USER, ADMIN_ROLE, COMPONENT_SOLR);
        client.addRoleToGroups(ADMIN_USER, ADMIN_ROLE, COMPONENT_SOLR, Sets.newHashSet(ADMIN_GROUP));
        grantCollectionPrivilege(SearchConstants.ALL, ADMIN_USER, ADMIN_ROLE, SearchConstants.ALL);
    }

    protected static void grantCollectionPrivilege(String collection, String requestor, String roleName,
            String action) throws SentryUserException {
        TSentryPrivilege tPrivilege = toTSentryPrivilege(collection, action);
        client.grantPrivilege(requestor, roleName, COMPONENT_SOLR, tPrivilege);
    }

    protected static void revokeCollectionPrivilege(String collection, String requestor, String roleName,
            String action) throws SentryUserException {
        TSentryPrivilege tPrivilege = toTSentryPrivilege(collection, action);
        client.revokePrivilege(requestor, roleName, COMPONENT_SOLR, tPrivilege);
    }

    protected static void dropCollectionPrivilege(String collection, String requestor) throws SentryUserException {
        final TSentryPrivilege tPrivilege = toTSentryPrivilege(collection, Action.ALL);
        client.dropPrivilege(requestor, COMPONENT_SOLR, tPrivilege);
    }

    private static TSentryPrivilege toTSentryPrivilege(String collection, String action) {
        TSentryPrivilege tPrivilege = new TSentryPrivilege();
        tPrivilege.setComponent(COMPONENT_SOLR);
        tPrivilege.setServiceName(CLUSTER_NAME);
        tPrivilege.setAction(action);
        tPrivilege.setGrantOption(TSentryGrantOption.FALSE);

        List<TAuthorizable> authorizables = Lists.newArrayList(new TAuthorizable(Collection.name(), collection));
        tPrivilege.setAuthorizables(authorizables);
        return tPrivilege;
    }
}