Java tutorial
package com.thinkbiganalytics.datalake.authorization; /*- * #%L * thinkbig-hadoop-authorization-ranger * %% * Copyright (C) 2017 ThinkBig Analytics * %% * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * #L% */ import com.thinkbiganalytics.datalake.authorization.config.AuthorizationConfiguration; import com.thinkbiganalytics.datalake.authorization.config.RangerConnection; import com.thinkbiganalytics.datalake.authorization.model.HadoopAuthorizationGroup; import com.thinkbiganalytics.datalake.authorization.rest.client.RangerRestClient; import com.thinkbiganalytics.datalake.authorization.rest.client.RangerRestClientConfig; import com.thinkbiganalytics.datalake.authorization.rest.model.RangerCreateOrUpdatePolicy; import com.thinkbiganalytics.datalake.authorization.rest.model.RangerGroup; import com.thinkbiganalytics.datalake.authorization.rest.model.RangerPolicy; import com.thinkbiganalytics.datalake.authorization.service.BaseHadoopAuthorizationService; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; public class RangerAuthorizationService extends BaseHadoopAuthorizationService { private static final Logger log = LoggerFactory.getLogger(RangerAuthorizationService.class); private static final String HADOOP_AUTHORIZATION_TYPE_RANGER = "RANGER"; private static final String HDFS_REPOSITORY_TYPE = "hdfs"; private static final String HIVE_REPOSITORY_TYPE = "hive"; private static final String IsEnable = "true"; private static final String IsRecursive = "true"; private static final String IsAuditable = "true"; private static final String REPOSITORY_TYPE = "repositoryType"; private static final String POLICY_NAME = "policyName"; private static final String HIVE_COLUMN_PERMISSION = "*"; private static final String HDFS_READ_ONLY_PERMISSION = "read"; private static final String HIVE_READ_ONLY_PERMISSION = "select"; private static final String KYLO_POLICY_PREFIX = "kylo"; private RangerRestClient rangerRestClient; private RangerConnection rangerConnection; /** * @return : comma separated string */ private static String convertListToString(List<String> list, String delim) { StringBuilder sb = new StringBuilder(); String loopDelim = ""; for (String input : list) { sb.append(loopDelim); sb.append(input); loopDelim = delim; } return sb.toString(); } /** * Implement Ranger Authentication Service. Initiate RangerClient and RangerClientConfig for initializing service and invoke different methods of it. */ @Override public void initialize(AuthorizationConfiguration config) { rangerConnection = (RangerConnection) config; RangerRestClientConfig rangerClientConfiguration = new RangerRestClientConfig( rangerConnection.getHostName(), rangerConnection.getUsername(), rangerConnection.getPassword()); rangerClientConfiguration.setPort(rangerConnection.getPort()); rangerRestClient = new RangerRestClient(rangerClientConfiguration); } @Override public RangerGroup getGroupByName(String groupName) { return rangerRestClient.getGroupByName(groupName); } @Override public List<HadoopAuthorizationGroup> getAllGroups() { return rangerRestClient.getAllGroups(); } @Override public void createOrUpdateReadOnlyHivePolicy(String categoryName, String feedName, List<String> securityGroupNames, String datebaseName, List<String> tableNames) { String rangerHivePolicyName = getHivePolicyName(categoryName, feedName); Map<String, Object> searchHiveCriteria = new HashMap<>(); searchHiveCriteria.put(POLICY_NAME, rangerHivePolicyName); searchHiveCriteria.put(REPOSITORY_TYPE, HIVE_REPOSITORY_TYPE); List<RangerPolicy> hadoopPolicyList = this.searchPolicy(searchHiveCriteria); if (hadoopPolicyList.size() > 1) { throw new RuntimeException("More than 1 unique Hive policy was found for " + rangerHivePolicyName); } else if (hadoopPolicyList.size() == 0) { createReadOnlyHivePolicy(categoryName, feedName, securityGroupNames, datebaseName, tableNames); } else { updateReadOnlyHivePolicy(categoryName, feedName, securityGroupNames, datebaseName, tableNames); } } @Override public void createOrUpdateReadOnlyHdfsPolicy(String categoryName, String feedName, List<String> securityGroupNames, List<String> hdfsPaths) { String rangerHdfsPolicyName = getHdfsPolicyName(categoryName, feedName); Map<String, Object> searchHDFSCriteria = new HashMap<>(); searchHDFSCriteria.put(POLICY_NAME, rangerHdfsPolicyName); searchHDFSCriteria.put(REPOSITORY_TYPE, HDFS_REPOSITORY_TYPE); List<RangerPolicy> hadoopPolicyList = this.searchPolicy(searchHDFSCriteria); if (hadoopPolicyList.size() > 1) { throw new RuntimeException("More than 1 unique HDFS policy was found for " + rangerHdfsPolicyName); } else if (hadoopPolicyList.size() == 0) { createReadOnlyHdfsPolicy(categoryName, feedName, securityGroupNames, hdfsPaths); } else { updateReadOnlyHdfsPolicy(categoryName, feedName, securityGroupNames, hdfsPaths); } } @Override public void createReadOnlyHivePolicy(String categoryName, String feedName, List<String> securityGroupNames, String databaseName, List<String> tableNames) { RangerCreateOrUpdatePolicy rangerCreateOrUpdatePolicy = new RangerCreateOrUpdatePolicy(); List<String> hivePermissions = new ArrayList<>(); hivePermissions.add(HIVE_READ_ONLY_PERMISSION); String rangerHivePolicyName = getHivePolicyName(categoryName, feedName); String hiveDescription = "Ranger policy created for group list " + securityGroupNames.toString() + " for resource " + tableNames.toString(); String hiveTables = convertListToString(tableNames, ","); rangerCreateOrUpdatePolicy = new RangerCreateOrUpdatePolicy(); rangerCreateOrUpdatePolicy.setPolicyName(rangerHivePolicyName); rangerCreateOrUpdatePolicy.setDatabases(databaseName); rangerCreateOrUpdatePolicy.setTables(hiveTables); rangerCreateOrUpdatePolicy.setColumns(HIVE_COLUMN_PERMISSION); rangerCreateOrUpdatePolicy.setUdfs(""); rangerCreateOrUpdatePolicy.setDescription(hiveDescription); rangerCreateOrUpdatePolicy.setRepositoryName(rangerConnection.getHiveRepositoryName()); rangerCreateOrUpdatePolicy.setRepositoryType(HIVE_REPOSITORY_TYPE); rangerCreateOrUpdatePolicy.setIsAuditEnabled(IsAuditable); rangerCreateOrUpdatePolicy.setIsEnabled(IsEnable); rangerCreateOrUpdatePolicy.setPermMapList(securityGroupNames, hivePermissions); try { rangerRestClient.createPolicy(rangerCreateOrUpdatePolicy); } catch (Exception e) { log.error("Error creating Hive Ranger policy", e); throw new RuntimeException("Error creating Hive Ranger policy", e); } } @Override public void createReadOnlyHdfsPolicy(String categoryName, String feedName, List<String> securityGroupNames, List<String> hdfsPaths) { RangerCreateOrUpdatePolicy rangerCreateOrUpdatePolicy = new RangerCreateOrUpdatePolicy(); /** * Create HDFS Policy */ List<String> hdfsPermissions = new ArrayList<>(); hdfsPermissions.add(HDFS_READ_ONLY_PERMISSION); String rangerHdfsPolicyName = getHdfsPolicyName(categoryName, feedName); String description = "Ranger policy created for group list " + securityGroupNames.toString() + " for resource " + hdfsPaths.toString(); String hdfsResource = convertListToString(hdfsPaths, ","); rangerCreateOrUpdatePolicy.setPolicyName(rangerHdfsPolicyName); rangerCreateOrUpdatePolicy.setResourceName(hdfsResource); rangerCreateOrUpdatePolicy.setDescription(description); rangerCreateOrUpdatePolicy.setRepositoryName(rangerConnection.getHdfsRepositoryName()); rangerCreateOrUpdatePolicy.setRepositoryType(HDFS_REPOSITORY_TYPE); rangerCreateOrUpdatePolicy.setIsEnabled(IsEnable); rangerCreateOrUpdatePolicy.setIsRecursive(IsRecursive); rangerCreateOrUpdatePolicy.setIsAuditEnabled(IsAuditable); rangerCreateOrUpdatePolicy.setPermMapList(securityGroupNames, hdfsPermissions); try { rangerRestClient.createPolicy(rangerCreateOrUpdatePolicy); } catch (Exception e) { log.error("Error creating HDFS Ranger policy", e); throw new RuntimeException("Error creating HDFS Ranger policy", e); } } @Override public void updateReadOnlyHivePolicy(String categoryName, String feedName, List<String> groups, String datebaseNames, List<String> tableNames) { int policyId; String rangerHivePolicyName = getHivePolicyName(categoryName, feedName); Map<String, Object> searchHiveCriteria = new HashMap<>(); searchHiveCriteria.put(POLICY_NAME, rangerHivePolicyName); searchHiveCriteria.put(REPOSITORY_TYPE, HIVE_REPOSITORY_TYPE); List<RangerPolicy> hadoopPolicyList = this.searchPolicy(searchHiveCriteria); policyId = 0; if (hadoopPolicyList.size() == 0) { throw new UnsupportedOperationException("Ranger Plugin : Unable to get ID for Ranger Hive Policy"); } else { if (hadoopPolicyList.size() > 1) { throw new RuntimeException("Unable to find Hive unique policy."); } else { for (RangerPolicy hadoopPolicy : hadoopPolicyList) { policyId = hadoopPolicy.getId(); } } } List<String> hivePermissions = new ArrayList<>(); hivePermissions.add(HIVE_READ_ONLY_PERMISSION); String hiveDescription = "Ranger policy updated for group list " + groups.toString() + " for resource " + datebaseNames; String hiveTables = convertListToString(tableNames, ","); RangerCreateOrUpdatePolicy rangerUpdatePolicy = new RangerCreateOrUpdatePolicy(); rangerUpdatePolicy.setPolicyName(rangerHivePolicyName); rangerUpdatePolicy.setDatabases(datebaseNames); rangerUpdatePolicy.setTables(hiveTables); rangerUpdatePolicy.setColumns(HIVE_COLUMN_PERMISSION); rangerUpdatePolicy.setUdfs(""); rangerUpdatePolicy.setDescription(hiveDescription); rangerUpdatePolicy.setRepositoryName(rangerConnection.getHiveRepositoryName()); rangerUpdatePolicy.setRepositoryType(HIVE_REPOSITORY_TYPE); rangerUpdatePolicy.setIsAuditEnabled(IsAuditable); rangerUpdatePolicy.setIsEnabled(IsEnable); rangerUpdatePolicy.setPermMapList(groups, hivePermissions); try { rangerRestClient.updatePolicy(rangerUpdatePolicy, policyId); } catch (Exception e) { log.error("Failed to update Hive Policy", e); throw new RuntimeException("Failed to update Hive Policy", e); } } @Override public void updateReadOnlyHdfsPolicy(String categoryName, String feedName, List<String> groups, List<String> hdfsPaths) { int policyId = 0; String rangerHdfsPolicyName = getHdfsPolicyName(categoryName, feedName); Map<String, Object> searchHDFSCriteria = new HashMap<>(); searchHDFSCriteria.put(POLICY_NAME, rangerHdfsPolicyName); searchHDFSCriteria.put(REPOSITORY_TYPE, HDFS_REPOSITORY_TYPE); List<RangerPolicy> hadoopPolicyList = this.searchPolicy(searchHDFSCriteria); if (hadoopPolicyList.size() == 0) { throw new UnsupportedOperationException("Ranger Plugin : Unable to get ID for Ranger HDFS Policy"); } else { if (hadoopPolicyList.size() > 1) { throw new RuntimeException("Unable to find HDFS unique policy."); } else { for (RangerPolicy hadoopPolicy : hadoopPolicyList) { policyId = hadoopPolicy.getId(); } } } RangerCreateOrUpdatePolicy rangerUpdatePolicy = new RangerCreateOrUpdatePolicy(); /** * Update HDFS Policy */ List<String> hdfsPermissions = new ArrayList<>(); hdfsPermissions.add(HDFS_READ_ONLY_PERMISSION); String description = "Ranger policy updated for group list " + groups.toString() + " for resource " + hdfsPaths.toString(); String hdfs_resource = convertListToString(hdfsPaths, ","); rangerUpdatePolicy.setPolicyName(rangerHdfsPolicyName); rangerUpdatePolicy.setResourceName(hdfs_resource); rangerUpdatePolicy.setDescription(description); rangerUpdatePolicy.setRepositoryName(rangerConnection.getHdfsRepositoryName()); rangerUpdatePolicy.setRepositoryType(HDFS_REPOSITORY_TYPE); rangerUpdatePolicy.setIsEnabled(IsEnable); rangerUpdatePolicy.setIsRecursive(IsRecursive); rangerUpdatePolicy.setIsAuditEnabled(IsAuditable); rangerUpdatePolicy.setPermMapList(groups, hdfsPermissions); try { rangerRestClient.updatePolicy(rangerUpdatePolicy, policyId); } catch (Exception e) { log.error("Failed to update HDFS policy", e); throw new RuntimeException("Failed to update HDFS policy", e); } } @Override /** * If no security policy exists it will be created. If a policy exists it will be updated */ public void updateSecurityGroupsForAllPolicies(String categoryName, String feedName, List<String> securityGroupNames, Map<String, Object> feedProperties) { String rangerHdfsPolicyName = getHdfsPolicyName(categoryName, feedName); RangerPolicy hdfsPolicy = searchHdfsPolicy(rangerHdfsPolicyName); String rangerHivePolicyName = getHivePolicyName(categoryName, feedName); RangerPolicy hivePolicy = searchHivePolicy(rangerHivePolicyName); if (securityGroupNames == null || securityGroupNames.isEmpty()) { // Only delete if the policies exists. It's possibile that someone adds a security group right after feed creation and before initial ingestion if (hivePolicy != null) { deleteHivePolicy(categoryName, feedName); } if (hdfsPolicy != null) { deleteHdfsPolicy(categoryName, feedName, null); } } else { if (hdfsPolicy == null) { // If a feed hasn't run yet the metadata won't exists if (!StringUtils.isEmpty((String) feedProperties.get(REGISTRATION_HDFS_FOLDERS))) { String hdfsFoldersWithCommas = ((String) feedProperties.get(REGISTRATION_HDFS_FOLDERS)) .replace("\n", ","); List<String> hdfsFolders = Stream.of(hdfsFoldersWithCommas).collect(Collectors.toList()); createReadOnlyHdfsPolicy(categoryName, feedName, securityGroupNames, hdfsFolders); } } else { List<String> hdfsPermissions = new ArrayList<>(); hdfsPermissions.add(HDFS_READ_ONLY_PERMISSION); RangerCreateOrUpdatePolicy updatePolicy = new RangerCreateOrUpdatePolicy(); BeanUtils.copyProperties(hdfsPolicy, updatePolicy); updatePolicy.setPermMapList(securityGroupNames, hdfsPermissions); rangerRestClient.updatePolicy(updatePolicy, hdfsPolicy.getId()); } if (hivePolicy == null) { // If a feed hasn't run yet the metadata won't exists if (!StringUtils.isEmpty((String) feedProperties.get(REGISTRATION_HIVE_TABLES))) { String hiveTablesWithCommas = ((String) feedProperties.get(REGISTRATION_HIVE_TABLES)) .replace("\n", ","); List<String> hiveTables = Stream.of(hiveTablesWithCommas).collect(Collectors.toList()); String hiveSchema = ((String) feedProperties.get(REGISTRATION_HIVE_SCHEMA)); createOrUpdateReadOnlyHivePolicy(categoryName, feedName, securityGroupNames, hiveSchema, hiveTables); } } else { List<String> hivePermissions = new ArrayList<>(); hivePermissions.add(HIVE_READ_ONLY_PERMISSION); RangerCreateOrUpdatePolicy updatePolicy = new RangerCreateOrUpdatePolicy(); BeanUtils.copyProperties(hivePolicy, updatePolicy); updatePolicy.setPermMapList(securityGroupNames, hivePermissions); rangerRestClient.updatePolicy(updatePolicy, hivePolicy.getId()); } } } private String getHdfsPolicyName(String categoryName, String feedName) { return KYLO_POLICY_PREFIX + "_" + categoryName + "_" + feedName + "_" + HDFS_REPOSITORY_TYPE; } private String getHivePolicyName(String categoryName, String feedName) { return KYLO_POLICY_PREFIX + "_" + categoryName + "_" + feedName + "_" + HIVE_REPOSITORY_TYPE; } private RangerPolicy searchHdfsPolicy(String hdfsPolicyName) { RangerPolicy result = null; Map<String, Object> searchHDFSCriteria = new HashMap<>(); searchHDFSCriteria.put(POLICY_NAME, hdfsPolicyName); searchHDFSCriteria.put(REPOSITORY_TYPE, HDFS_REPOSITORY_TYPE); List<RangerPolicy> hadoopPolicyList = this.searchPolicy(searchHDFSCriteria); if (hadoopPolicyList.size() > 1) { throw new RuntimeException("More than 1 unique HDFS policy was found for " + hdfsPolicyName); } else if (hadoopPolicyList != null && !hadoopPolicyList.isEmpty()) { result = hadoopPolicyList.get(0); } return result; } private RangerPolicy searchHivePolicy(String hivePolicyName) { RangerPolicy result = null; Map<String, Object> searchHiveCriteria = new HashMap<>(); searchHiveCriteria.put(POLICY_NAME, hivePolicyName); searchHiveCriteria.put(REPOSITORY_TYPE, HIVE_REPOSITORY_TYPE); List<RangerPolicy> hadoopPolicyList = this.searchPolicy(searchHiveCriteria); if (hadoopPolicyList.size() > 1) { throw new RuntimeException("More than 1 unique Hive policy was found for " + hivePolicyName); } else if (hadoopPolicyList != null && !hadoopPolicyList.isEmpty()) { result = hadoopPolicyList.get(0); } return result; } private List<RangerPolicy> searchPolicy(Map<String, Object> searchCriteria) { return rangerRestClient.searchPolicies(searchCriteria); } @Override public void deleteHivePolicy(String categoryName, String feedName) { String rangerHivePolicyName = getHivePolicyName(categoryName, feedName); Map<String, Object> searchHiveCriteria = new HashMap<>(); searchHiveCriteria.put(POLICY_NAME, rangerHivePolicyName); searchHiveCriteria.put(REPOSITORY_TYPE, HIVE_REPOSITORY_TYPE); List<RangerPolicy> hadoopPolicyList = this.searchPolicy(searchHiveCriteria); if (hadoopPolicyList.size() == 0) { log.info("Ranger Plugin : Unable to find Ranger Hive policy " + rangerHivePolicyName + " ... Ignoring"); } else if (hadoopPolicyList.size() > 1) { throw new RuntimeException("Unable to find Hive unique policy."); } else { for (RangerPolicy hadoopPolicy : hadoopPolicyList) { try { rangerRestClient.deletePolicy(hadoopPolicy.getId()); } catch (Exception e) { log.error("Unable to delete policy", e); throw new RuntimeException("Unable to delete policy for " + rangerHivePolicyName, e); } } } } @Override public void deleteHdfsPolicy(String categoryName, String feedName, List<String> hdfsPaths) { String rangerHdfsPolicyName = getHdfsPolicyName(categoryName, feedName); Map<String, Object> searchHdfsCriteria = new HashMap<>(); searchHdfsCriteria.put(POLICY_NAME, rangerHdfsPolicyName); searchHdfsCriteria.put(REPOSITORY_TYPE, HDFS_REPOSITORY_TYPE); List<RangerPolicy> hadoopPolicyList = this.searchPolicy(searchHdfsCriteria); if (hadoopPolicyList.size() == 0) { log.info("Ranger Plugin : Unable to find Ranger HDFS policy " + rangerHdfsPolicyName + " ... Ignoring"); } else if (hadoopPolicyList.size() > 1) { throw new RuntimeException("Unable to find HDFS unique policy."); } else { for (RangerPolicy hadoopPolicy : hadoopPolicyList) { try { rangerRestClient.deletePolicy(hadoopPolicy.getId()); } catch (Exception e) { log.error("Unable to delete policy", e); throw new RuntimeException("Unable to delete policy for " + rangerHdfsPolicyName, e); } } } } @Override public String getType() { return HADOOP_AUTHORIZATION_TYPE_RANGER; } }