Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the * License. You may obtain a copy of the License at * <p> * http://www.apache.org/licenses/LICENSE-2.0 * <p> * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.buaa.cfs.security; import com.buaa.cfs.conf.Configuration; import com.buaa.cfs.constant.IdMappingConstant; import com.buaa.cfs.utils.Time; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import org.apache.commons.io.Charsets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import java.io.*; import java.nio.charset.Charset; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * A simple shell-based implementation of {@link IdMappingServiceProvider} Map id to user name or group name. It does * update every 15 minutes. Only a single instance of this class is expected to be on the server. * <p> * The maps are incrementally updated as described below: 1. Initialize the maps as empty. 2. Incrementally update the * maps - When ShellBasedIdMapping is requested for user or group name given an ID, or for ID given a user or group * name, do look up in the map first, if it doesn't exist, find the corresponding entry with shell command, and insert * the entry to the maps. - When group ID is requested for a given group name, and if the group name is numerical, the * full group map is loaded. Because we don't have a good way to find the entry for a numerical group name, loading the * full map helps to get in all entries. 3. Periodically refresh the maps for both user and group, e.g, do step 1. Note: * for testing purpose, step 1 may initial the maps with full mapping when using constructor */ public class ShellBasedIdMapping implements IdMappingServiceProvider { private static final Log LOG = LogFactory.getLog(ShellBasedIdMapping.class); private final static String OS = System.getProperty("os.name"); /** Shell commands to get users and groups */ static final String GET_ALL_USERS_CMD = "getent passwd | cut -d: -f1,3"; static final String GET_ALL_GROUPS_CMD = "getent group | cut -d: -f1,3"; static final String MAC_GET_ALL_USERS_CMD = "dscl . -list /Users UniqueID"; static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups PrimaryGroupID"; private final File staticMappingFile; private StaticMapping staticMapping = null; // Last time the static map was modified, measured time difference in // milliseconds since midnight, January 1, 1970 UTC private long lastModificationTimeStaticMap = 0; private boolean constructFullMapAtInit = false; // Used for parsing the static mapping file. private static final Pattern EMPTY_LINE = Pattern.compile("^\\s*$"); private static final Pattern COMMENT_LINE = Pattern.compile("^\\s*#.*$"); private static final Pattern MAPPING_LINE = Pattern.compile("^(uid|gid)\\s+(\\d+)\\s+(\\d+)\\s*(#.*)?$"); final private long timeout; // Maps for id to name map. Guarded by this object monitor lock private BiMap<Integer, String> uidNameMap = HashBiMap.create(); private BiMap<Integer, String> gidNameMap = HashBiMap.create(); private long lastUpdateTime = 0; // Last time maps were updated /* * Constructor * @param conf the configuration * @param constructFullMapAtInit initialize the maps with full mapping when * true, otherwise initialize the maps to empty. This parameter is * intended for testing only, its default is false. */ @VisibleForTesting public ShellBasedIdMapping(Configuration conf, boolean constructFullMapAtInit) throws IOException { this.constructFullMapAtInit = constructFullMapAtInit; long updateTime = conf.getLong(IdMappingConstant.USERGROUPID_UPDATE_MILLIS_KEY, IdMappingConstant.USERGROUPID_UPDATE_MILLIS_DEFAULT); // Minimal interval is 1 minute if (updateTime < IdMappingConstant.USERGROUPID_UPDATE_MILLIS_MIN) { LOG.info("User configured user account update time is less" + " than 1 minute. Use 1 minute instead."); timeout = IdMappingConstant.USERGROUPID_UPDATE_MILLIS_MIN; } else { timeout = updateTime; } String staticFilePath = conf.get(IdMappingConstant.STATIC_ID_MAPPING_FILE_KEY, IdMappingConstant.STATIC_ID_MAPPING_FILE_DEFAULT); staticMappingFile = new File(staticFilePath); updateStaticMapping(); updateMaps(); } /** * Constructor initialize user and group maps to empty * * @param conf the configuration */ public ShellBasedIdMapping(Configuration conf) throws IOException { this(conf, false); } @VisibleForTesting public long getTimeout() { return timeout; } @VisibleForTesting public BiMap<Integer, String> getUidNameMap() { return uidNameMap; } @VisibleForTesting public BiMap<Integer, String> getGidNameMap() { return gidNameMap; } @VisibleForTesting synchronized public void clearNameMaps() { uidNameMap.clear(); gidNameMap.clear(); lastUpdateTime = Time.monotonicNow(); } synchronized private boolean isExpired() { return Time.monotonicNow() - lastUpdateTime > timeout; } // If can't update the maps, will keep using the old ones private void checkAndUpdateMaps() { if (isExpired()) { LOG.info("Update cache now"); try { updateMaps(); } catch (IOException e) { LOG.error("Can't update the maps. Will use the old ones," + " which can potentially cause problem.", e); } } } private static final String DUPLICATE_NAME_ID_DEBUG_INFO = "NFS gateway could have problem starting with duplicate name or id on the host system.\n" + "This is because HDFS (non-kerberos cluster) uses name as the only way to identify a user or group.\n" + "The host system with duplicated user/group name or id might work fine most of the time by itself.\n" + "However when NFS gateway talks to HDFS, HDFS accepts only user and group name.\n" + "Therefore, same name means the same user or same group. To find the duplicated names/ids, one can do:\n" + "<getent passwd | cut -d: -f1,3> and <getent group | cut -d: -f1,3> on Linux systems,\n" + "<dscl . -list /Users UniqueID> and <dscl . -list /Groups PrimaryGroupID> on MacOS."; private static void reportDuplicateEntry(final String header, final Integer key, final String value, final Integer ekey, final String evalue) { LOG.warn("\n" + header + String.format("new entry (%d, %s), existing entry: (%d, %s).%n%s%n%s", key, value, ekey, evalue, "The new entry is to be ignored for the following reason.", DUPLICATE_NAME_ID_DEBUG_INFO)); } /** * uid and gid are defined as uint32 in linux. Some systems create (intended or unintended) <nfsnobody, 4294967294> * kind of <name,Id> mapping, where 4294967294 is 2**32-2 as unsigned int32. As an example, * https://bugzilla.redhat.com/show_bug.cgi?id=511876. Because user or group id are treated as Integer (signed * integer or int32) here, the number 4294967294 is out of range. The solution is to convert uint32 to int32, so to * map the out-of-range ID to the negative side of Integer, e.g. 4294967294 maps to -2 and 4294967295 maps to -1. */ private static Integer parseId(final String idStr) { Long longVal = Long.parseLong(idStr); int intVal = longVal.intValue(); return Integer.valueOf(intVal); } /** * Get the list of users or groups returned by the specified command, and save them in the corresponding map. * * @throws IOException */ @VisibleForTesting public static boolean updateMapInternal(BiMap<Integer, String> map, String mapName, String command, String regex, Map<Integer, Integer> staticMapping) throws IOException { boolean updated = false; BufferedReader br = null; try { Process process = Runtime.getRuntime().exec(new String[] { "bash", "-c", command }); br = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.defaultCharset())); String line = null; while ((line = br.readLine()) != null) { String[] nameId = line.split(regex); if ((nameId == null) || (nameId.length != 2)) { throw new IOException("Can't parse " + mapName + " list entry:" + line); } LOG.debug("add to " + mapName + "map:" + nameId[0] + " id:" + nameId[1]); // HDFS can't differentiate duplicate names with simple authentication final Integer key = staticMapping.get(parseId(nameId[1])); final String value = nameId[0]; if (map.containsKey(key)) { final String prevValue = map.get(key); if (value.equals(prevValue)) { // silently ignore equivalent entries continue; } reportDuplicateEntry("Got multiple names associated with the same id: ", key, value, key, prevValue); continue; } if (map.containsValue(value)) { final Integer prevKey = map.inverse().get(value); reportDuplicateEntry("Got multiple ids associated with the same name: ", key, value, prevKey, value); continue; } map.put(key, value); updated = true; } LOG.debug("Updated " + mapName + " map size: " + map.size()); } catch (IOException e) { LOG.error("Can't update " + mapName + " map"); throw e; } finally { if (br != null) { try { br.close(); } catch (IOException e1) { LOG.error("Can't close BufferedReader of command result", e1); } } } return updated; } private boolean checkSupportedPlatform() { if (!OS.startsWith("Linux") && !OS.startsWith("Mac")) { LOG.error("Platform is not supported:" + OS + ". Can't update user map and group map and" + " 'nobody' will be used for any user and group."); return false; } return true; } private static boolean isInteger(final String s) { try { Integer.parseInt(s); } catch (NumberFormatException e) { return false; } // only got here if we didn't return false return true; } private synchronized void updateStaticMapping() throws IOException { final boolean init = (staticMapping == null); // // if the static mapping file // - was modified after last update, load the map again; // - did not exist but was added since last update, load the map; // - existed before but deleted since last update, clear the map // if (staticMappingFile.exists()) { // check modification time, reload the file if the last modification // time changed since prior load. long lmTime = staticMappingFile.lastModified(); if (lmTime != lastModificationTimeStaticMap) { LOG.info(init ? "Using " : "Reloading " + "'" + staticMappingFile + "' for static UID/GID mapping..."); lastModificationTimeStaticMap = lmTime; staticMapping = parseStaticMap(staticMappingFile); } } else { if (init) { staticMapping = new StaticMapping(new HashMap<Integer, Integer>(), new HashMap<Integer, Integer>()); } if (lastModificationTimeStaticMap != 0 || init) { // print the following log at initialization or when the static // mapping file was deleted after prior load LOG.info("Not doing static UID/GID mapping because '" + staticMappingFile + "' does not exist."); } lastModificationTimeStaticMap = 0; staticMapping.clear(); } } /** * Refresh static map, and reset the other maps to empty. For testing code, a full map may be re-constructed here * when the object was created with constructFullMapAtInit being set to true. */ synchronized public void updateMaps() throws IOException { if (!checkSupportedPlatform()) { return; } if (constructFullMapAtInit) { loadFullMaps(); // set constructFullMapAtInit to false to allow testing code to // do incremental update to maps after initial construction constructFullMapAtInit = false; } else { updateStaticMapping(); clearNameMaps(); } } synchronized private void loadFullUserMap() throws IOException { BiMap<Integer, String> uMap = HashBiMap.create(); if (OS.startsWith("Mac")) { updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+", staticMapping.uidMapping); } else { updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":", staticMapping.uidMapping); } uidNameMap = uMap; lastUpdateTime = Time.monotonicNow(); } synchronized private void loadFullGroupMap() throws IOException { BiMap<Integer, String> gMap = HashBiMap.create(); if (OS.startsWith("Mac")) { updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+", staticMapping.gidMapping); } else { updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":", staticMapping.gidMapping); } gidNameMap = gMap; lastUpdateTime = Time.monotonicNow(); } synchronized private void loadFullMaps() throws IOException { loadFullUserMap(); loadFullGroupMap(); } // search for id with given name, return "<name>:<id>" // return // getent group <name> | cut -d: -f1,3 // OR // id -u <name> | awk '{print "<name>:"$1 }' // private String getName2IdCmdLinux(final String name, final boolean isGrp) { String cmd; if (isGrp) { cmd = "getent group " + name + " | cut -d: -f1,3"; } else { cmd = "id -u " + name + " | awk '{print \"" + name + ":\"$1 }'"; } return cmd; } // search for name with given id, return "<name>:<id>" private String getId2NameCmdLinux(final int id, final boolean isGrp) { String cmd = "getent "; cmd += isGrp ? "group " : "passwd "; cmd += String.valueOf(id) + " | cut -d: -f1,3"; return cmd; } // "dscl . -read /Users/<name> | grep UniqueID" returns "UniqueId: <id>", // "dscl . -read /Groups/<name> | grep PrimaryGroupID" returns "PrimaryGoupID: <id>" // The following method returns a command that uses awk to process the result, // of these commands, and returns "<name> <id>", to simulate one entry returned by // MAC_GET_ALL_USERS_CMD or MAC_GET_ALL_GROUPS_CMD. // Specificially, this method returns: // id -u <name> | awk '{print "<name>:"$1 }' // OR // dscl . -read /Groups/<name> | grep PrimaryGroupID | awk '($1 == "PrimaryGroupID:") { print "<name> " $2 }' // private String getName2IdCmdMac(final String name, final boolean isGrp) { String cmd; if (isGrp) { cmd = "dscl . -read /Groups/" + name; cmd += " | grep PrimaryGroupID | awk '($1 == \"PrimaryGroupID:\") "; cmd += "{ print \"" + name + " \" $2 }'"; } else { cmd = "id -u " + name + " | awk '{print \"" + name + " \"$1 }'"; } return cmd; } // "dscl . -search /Users UniqueID <id>" returns // <name> UniqueID = ( // <id> // ) // "dscl . -search /Groups PrimaryGroupID <id>" returns // <name> PrimaryGroupID = ( // <id> // ) // The following method returns a command that uses sed to process the // the result and returns "<name> <id>" to simulate one entry returned // by MAC_GET_ALL_USERS_CMD or MAC_GET_ALL_GROUPS_CMD. // For certain negative id case like nfsnobody, the <id> is quoted as // "<id>", added one sed section to remove the quote. // Specifically, the method returns: // dscl . -search /Users UniqueID <id> | sed 'N;s/\\n//g;N;s/\\n//g' | sed 's/UniqueID =//g' | sed 's/)//g' | sed 's/\"//g' // OR // dscl . -search /Groups PrimaryGroupID <id> | sed 'N;s/\\n//g;N;s/\\n//g' | sed 's/PrimaryGroupID =//g' | sed 's/)//g' | sed 's/\"//g' // private String getId2NameCmdMac(final int id, final boolean isGrp) { String cmd = "dscl . -search /"; cmd += isGrp ? "Groups PrimaryGroupID " : "Users UniqueID "; cmd += String.valueOf(id); cmd += " | sed 'N;s/\\n//g;N;s/\\n//g' | sed 's/"; cmd += isGrp ? "PrimaryGroupID" : "UniqueID"; cmd += " = (//g' | sed 's/)//g' | sed 's/\\\"//g'"; return cmd; } synchronized private void updateMapIncr(final String name, final boolean isGrp) throws IOException { if (!checkSupportedPlatform()) { return; } if (isInteger(name) && isGrp) { loadFullGroupMap(); return; } boolean updated = false; updateStaticMapping(); if (OS.startsWith("Linux")) { if (isGrp) { updated = updateMapInternal(gidNameMap, "group", getName2IdCmdLinux(name, true), ":", staticMapping.gidMapping); } else { updated = updateMapInternal(uidNameMap, "user", getName2IdCmdLinux(name, false), ":", staticMapping.uidMapping); } } else { // Mac if (isGrp) { updated = updateMapInternal(gidNameMap, "group", getName2IdCmdMac(name, true), "\\s+", staticMapping.gidMapping); } else { updated = updateMapInternal(uidNameMap, "user", getName2IdCmdMac(name, false), "\\s+", staticMapping.uidMapping); } } if (updated) { lastUpdateTime = Time.monotonicNow(); } } synchronized private void updateMapIncr(final int id, final boolean isGrp) throws IOException { if (!checkSupportedPlatform()) { return; } boolean updated = false; updateStaticMapping(); if (OS.startsWith("Linux")) { if (isGrp) { updated = updateMapInternal(gidNameMap, "group", getId2NameCmdLinux(id, true), ":", staticMapping.gidMapping); } else { updated = updateMapInternal(uidNameMap, "user", getId2NameCmdLinux(id, false), ":", staticMapping.uidMapping); } } else { // Mac if (isGrp) { updated = updateMapInternal(gidNameMap, "group", getId2NameCmdMac(id, true), "\\s+", staticMapping.gidMapping); } else { updated = updateMapInternal(uidNameMap, "user", getId2NameCmdMac(id, false), "\\s+", staticMapping.uidMapping); } } if (updated) { lastUpdateTime = Time.monotonicNow(); } } @SuppressWarnings("serial") static final class PassThroughMap<K> extends HashMap<K, K> { public PassThroughMap() { this(new HashMap<K, K>()); } public PassThroughMap(Map<K, K> mapping) { super(); for (Entry<K, K> entry : mapping.entrySet()) { super.put(entry.getKey(), entry.getValue()); } } @SuppressWarnings("unchecked") @Override public K get(Object key) { if (super.containsKey(key)) { return super.get(key); } else { return (K) key; } } } @VisibleForTesting static final class StaticMapping { final Map<Integer, Integer> uidMapping; final Map<Integer, Integer> gidMapping; public StaticMapping(Map<Integer, Integer> uidMapping, Map<Integer, Integer> gidMapping) { this.uidMapping = new PassThroughMap<Integer>(uidMapping); this.gidMapping = new PassThroughMap<Integer>(gidMapping); } public void clear() { uidMapping.clear(); gidMapping.clear(); } public boolean isNonEmpty() { return uidMapping.size() > 0 || gidMapping.size() > 0; } } static StaticMapping parseStaticMap(File staticMapFile) throws IOException { Map<Integer, Integer> uidMapping = new HashMap<Integer, Integer>(); Map<Integer, Integer> gidMapping = new HashMap<Integer, Integer>(); BufferedReader in = new BufferedReader( new InputStreamReader(new FileInputStream(staticMapFile), Charsets.UTF_8)); try { String line = null; while ((line = in.readLine()) != null) { // Skip entirely empty and comment lines. LOG.info("--- the idmaping read line is : " + line); if (EMPTY_LINE.matcher(line).matches() || COMMENT_LINE.matcher(line).matches()) { continue; } Matcher lineMatcher = MAPPING_LINE.matcher(line); if (!lineMatcher.matches()) { LOG.warn("Could not parse line '" + line + "'. Lines should be of " + "the form '[uid|gid] [remote id] [local id]'. Blank lines and " + "everything following a '#' on a line will be ignored."); continue; } // We know the line is fine to parse without error checking like this // since it matched the regex above. String firstComponent = lineMatcher.group(1); LOG.info("--- the idmaping read line group(1) is : " + firstComponent); int remoteId = parseId(lineMatcher.group(2)); int localId = parseId(lineMatcher.group(3)); if (firstComponent.equals("uid")) { uidMapping.put(localId, remoteId); } else { gidMapping.put(localId, remoteId); } } } finally { in.close(); } return new StaticMapping(uidMapping, gidMapping); } synchronized public int getUid(String user) throws IOException { checkAndUpdateMaps(); Integer id = uidNameMap.inverse().get(user); if (id == null) { updateMapIncr(user, false); id = uidNameMap.inverse().get(user); if (id == null) { throw new IOException("User just deleted?:" + user); } } return id.intValue(); } synchronized public int getGid(String group) throws IOException { checkAndUpdateMaps(); Integer id = gidNameMap.inverse().get(group); if (id == null) { updateMapIncr(group, true); id = gidNameMap.inverse().get(group); if (id == null) { throw new IOException("No such group:" + group); } } return id.intValue(); } synchronized public String getUserName(int uid, String unknown) { checkAndUpdateMaps(); String uname = uidNameMap.get(uid); if (uname == null) { try { updateMapIncr(uid, false); } catch (Exception e) { } uname = uidNameMap.get(uid); if (uname == null) { LOG.warn("Can't find user name for uid " + uid + ". Use default user name " + unknown); uname = unknown; } } return uname; } synchronized public String getGroupName(int gid, String unknown) { checkAndUpdateMaps(); String gname = gidNameMap.get(gid); if (gname == null) { try { updateMapIncr(gid, true); } catch (Exception e) { } gname = gidNameMap.get(gid); if (gname == null) { LOG.warn("Can't find group name for gid " + gid + ". Use default group name " + unknown); gname = unknown; } } return gname; } // When can't map user, return user name's string hashcode public int getUidAllowingUnknown(String user) { checkAndUpdateMaps(); int uid; try { uid = getUid(user); } catch (IOException e) { uid = user.hashCode(); LOG.info("Can't map user " + user + ". Use its string hashcode:" + uid); } return uid; } // When can't map group, return group name's string hashcode public int getGidAllowingUnknown(String group) { checkAndUpdateMaps(); int gid; try { gid = getGid(group); } catch (IOException e) { gid = group.hashCode(); LOG.info("Can't map group " + group + ". Use its string hashcode:" + gid); } return gid; } }