Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.ranger.services.hdfs.client; import java.io.FileNotFoundException; import java.io.IOException; import java.net.UnknownHostException; import java.security.PrivilegedExceptionAction; import java.util.*; import javax.security.auth.Subject; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.security.SecureClientLogin; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ranger.plugin.client.BaseClient; import org.apache.ranger.plugin.client.HadoopException; public class HdfsClient extends BaseClient { private static final Log LOG = LogFactory.getLog(HdfsClient.class); private Configuration conf; public HdfsClient(String serviceName, Map<String, String> connectionProperties) { super(serviceName, connectionProperties, "hdfs-client"); conf = new Configuration(); Set<String> rangerInternalPropertyKeys = getConfigHolder().getRangerInternalPropertyKeys(); for (Map.Entry<String, String> entry : connectionProperties.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); if (!rangerInternalPropertyKeys.contains(key)) { conf.set(key, value); } } } private List<String> listFilesInternal(String baseDir, String fileMatching, final List<String> pathList) throws HadoopException { List<String> fileList = new ArrayList<String>(); String errMsg = " You can still save the repository and start creating " + "policies, but you would not be able to use autocomplete for " + "resource names. Check ranger_admin.log for more info."; try { String dirPrefix = (baseDir.endsWith("/") ? baseDir : (baseDir + "/")); String filterRegEx = null; if (fileMatching != null && fileMatching.trim().length() > 0) { filterRegEx = fileMatching.trim(); } UserGroupInformation.setConfiguration(conf); FileSystem fs = null; try { fs = FileSystem.get(conf); Path basePath = new Path(baseDir); FileStatus[] fileStats = fs.listStatus(basePath); if (LOG.isDebugEnabled()) { LOG.debug("<== HdfsClient fileStatus : " + fileStats.length + " PathList :" + pathList); } if (fileStats != null) { if (fs.exists(basePath) && ArrayUtils.isEmpty(fileStats)) { fileList.add(basePath.toString()); } else { for (FileStatus stat : fileStats) { Path path = stat.getPath(); String pathComponent = path.getName(); String prefixedPath = dirPrefix + pathComponent; if (pathList != null && pathList.contains(prefixedPath)) { continue; } if (filterRegEx == null) { fileList.add(prefixedPath); } else if (FilenameUtils.wildcardMatch(pathComponent, fileMatching)) { fileList.add(prefixedPath); } } } } } catch (UnknownHostException uhe) { String msgDesc = "listFilesInternal: Unable to connect using given config parameters" + " of Hadoop environment [" + getSerivceName() + "]."; HadoopException hdpException = new HadoopException(msgDesc, uhe); hdpException.generateResponseDataMap(false, getMessage(uhe), msgDesc + errMsg, null, null); if (LOG.isDebugEnabled()) { LOG.debug("<== HdfsClient listFilesInternal Error : " + uhe); } throw hdpException; } catch (FileNotFoundException fne) { String msgDesc = "listFilesInternal: Unable to locate files using given config parameters " + "of Hadoop environment [" + getSerivceName() + "]."; HadoopException hdpException = new HadoopException(msgDesc, fne); hdpException.generateResponseDataMap(false, getMessage(fne), msgDesc + errMsg, null, null); if (LOG.isDebugEnabled()) { LOG.debug("<== HdfsClient listFilesInternal Error : " + fne); } throw hdpException; } } catch (IOException ioe) { String msgDesc = "listFilesInternal: Unable to get listing of files for directory " + baseDir + fileMatching + "] from Hadoop environment [" + getSerivceName() + "]."; HadoopException hdpException = new HadoopException(msgDesc, ioe); hdpException.generateResponseDataMap(false, getMessage(ioe), msgDesc + errMsg, null, null); if (LOG.isDebugEnabled()) { LOG.debug("<== HdfsClient listFilesInternal Error : " + ioe); } throw hdpException; } catch (IllegalArgumentException iae) { String msgDesc = "Unable to get listing of files for directory [" + baseDir + "] from Hadoop environment [" + getSerivceName() + "]."; HadoopException hdpException = new HadoopException(msgDesc, iae); hdpException.generateResponseDataMap(false, getMessage(iae), msgDesc + errMsg, null, null); if (LOG.isDebugEnabled()) { LOG.debug("<== HdfsClient listFilesInternal Error : " + iae); } throw hdpException; } return fileList; } public List<String> listFiles(final String baseDir, final String fileMatching, final List<String> pathList) throws Exception { PrivilegedExceptionAction<List<String>> action = new PrivilegedExceptionAction<List<String>>() { @Override public List<String> run() throws Exception { return listFilesInternal(baseDir, fileMatching, pathList); } }; return Subject.doAs(getLoginSubject(), action); } public static final void main(String[] args) { if (args.length < 2) { System.err.println("USAGE: java " + HdfsClient.class.getName() + " repositoryName basedirectory [filenameToMatch]"); System.exit(1); } String repositoryName = args[0]; String baseDir = args[1]; String fileNameToMatch = (args.length == 2 ? null : args[2]); HdfsClient fs = new HdfsClient(repositoryName, new HashMap<String, String>()); List<String> fsList = null; try { fsList = fs.listFiles(baseDir, fileNameToMatch, null); } catch (Exception e) { e.printStackTrace(); } if (fsList != null && fsList.size() > 0) { for (String s : fsList) { System.out.println(s); } } else { System.err.println("Unable to get file listing for [" + baseDir + (baseDir.endsWith("/") ? "" : "/") + fileNameToMatch + "] in repository [" + repositoryName + "]"); } } public static HashMap<String, Object> connectionTest(String serviceName, Map<String, String> configs) throws Exception { LOG.info("===> HdfsClient.testConnection()"); HashMap<String, Object> responseData = new HashMap<String, Object>(); boolean connectivityStatus = false; String validateConfigsMsg = null; try { validateConnectionConfigs(configs); } catch (IllegalArgumentException e) { validateConfigsMsg = e.getMessage(); } if (validateConfigsMsg == null) { HdfsClient connectionObj = new HdfsClient(serviceName, configs); if (connectionObj != null) { List<String> testResult = null; try { testResult = connectionObj.listFiles("/", null, null); } catch (HadoopException e) { LOG.error("<== HdfsClient.testConnection() error " + e.getMessage(), e); throw e; } if (testResult != null && testResult.size() != 0) { connectivityStatus = true; } } } String testconnMsg = null; if (connectivityStatus) { testconnMsg = "ConnectionTest Successful"; generateResponseDataMap(connectivityStatus, testconnMsg, testconnMsg, null, null, responseData); } else { testconnMsg = "Unable to retrieve any files using given parameters, " + "You can still save the repository and start creating policies, " + "but you would not be able to use autocomplete for resource names. " + "Check ranger_admin.log for more info. "; String additionalMsg = (validateConfigsMsg != null) ? validateConfigsMsg : testconnMsg; generateResponseDataMap(connectivityStatus, testconnMsg, additionalMsg, null, null, responseData); } LOG.info("<== HdfsClient.testConnection(): Status " + testconnMsg); return responseData; } public static void validateConnectionConfigs(Map<String, String> configs) throws IllegalArgumentException { String lookupPrincipal = null; try { lookupPrincipal = SecureClientLogin.getPrincipal(configs.get("lookupprincipal"), java.net.InetAddress.getLocalHost().getCanonicalHostName()); } catch (Exception e) { //do nothing } String lookupKeytab = configs.get("lookupkeytab"); if (StringUtils.isEmpty(lookupPrincipal) || StringUtils.isEmpty(lookupKeytab)) { // username String username = configs.get("username"); if ((username == null || username.isEmpty())) { throw new IllegalArgumentException("Value for username not specified"); } // password String password = configs.get("password"); if ((password == null || password.isEmpty())) { throw new IllegalArgumentException("Value for password not specified"); } } // hadoop.security.authentication String authentication = configs.get("hadoop.security.authentication"); if ((authentication == null || authentication.isEmpty())) { throw new IllegalArgumentException("Value for hadoop.security.authentication not specified"); } String fsDefaultName = configs.get("fs.default.name"); fsDefaultName = (fsDefaultName == null) ? "" : fsDefaultName.trim(); if (fsDefaultName.isEmpty()) { throw new IllegalArgumentException("Value for neither fs.default.name is specified"); } String dfsNameservices = configs.get("dfs.nameservices"); dfsNameservices = (dfsNameservices == null) ? "" : dfsNameservices.trim(); if (!dfsNameservices.isEmpty()) { String proxyProvider = configs.get("dfs.client.failover.proxy.provider." + dfsNameservices); proxyProvider = (proxyProvider == null) ? "" : proxyProvider.trim(); if (proxyProvider.isEmpty()) { throw new IllegalArgumentException( "Value for " + "dfs.client.failover.proxy.provider." + dfsNameservices + " not specified"); } String dfsNameNodes = configs.get("dfs.ha.namenodes." + dfsNameservices); dfsNameNodes = (dfsNameNodes == null) ? "" : dfsNameNodes.trim(); if (dfsNameNodes.isEmpty()) { throw new IllegalArgumentException( "Value for " + "dfs.ha.namenodes." + proxyProvider + " not specified"); } String[] dfsNameNodeElements = dfsNameNodes.split(","); for (String dfsNameNodeElement : dfsNameNodeElements) { String nameNodeUrlKey = "dfs.namenode.rpc-address." + dfsNameservices + "." + dfsNameNodeElement.trim(); String nameNodeUrl = configs.get(nameNodeUrlKey); nameNodeUrl = (nameNodeUrl == null) ? "" : nameNodeUrl.trim(); if (nameNodeUrl.isEmpty()) { throw new IllegalArgumentException("Value for " + nameNodeUrlKey + " not specified"); } } } } }