Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * <p> * http://www.apache.org/licenses/LICENSE-2.0 * <p> * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hive.metastore.utils; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Objects; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import org.apache.commons.lang.ArrayUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclEntryScope; import org.apache.hadoop.fs.permission.AclEntryType; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.client.HdfsAdmin; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.tools.DistCp; import org.apache.hadoop.tools.DistCpOptions; import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.security.auth.login.LoginException; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; public class HdfsUtils { private static final Logger LOG = LoggerFactory.getLogger(HdfsUtils.class); private static final String DISTCP_OPTIONS_PREFIX = "distcp.options."; // TODO: this relies on HDFS not changing the format; we assume if we could get inode ID, this // is still going to work. Otherwise, file IDs can be turned off. Later, we should use // as public utility method in HDFS to obtain the inode-based path. private static final String HDFS_ID_PATH_PREFIX = "/.reserved/.inodes/"; /** * Check the permissions on a file. * @param fs Filesystem the file is contained in * @param stat Stat info for the file * @param action action to be performed * @throws IOException If thrown by Hadoop * @throws AccessControlException if the file cannot be accessed */ public static void checkFileAccess(FileSystem fs, FileStatus stat, FsAction action) throws IOException, LoginException { checkFileAccess(fs, stat, action, SecurityUtils.getUGI()); } /** * Check the permissions on a file * @param fs Filesystem the file is contained in * @param stat Stat info for the file * @param action action to be performed * @param ugi user group info for the current user. This is passed in so that tests can pass * in mock ones. * @throws IOException If thrown by Hadoop * @throws AccessControlException if the file cannot be accessed */ @VisibleForTesting static void checkFileAccess(FileSystem fs, FileStatus stat, FsAction action, UserGroupInformation ugi) throws IOException { String user = ugi.getShortUserName(); String[] groups = ugi.getGroupNames(); if (groups != null) { String superGroupName = fs.getConf().get("dfs.permissions.supergroup", ""); if (arrayContains(groups, superGroupName)) { LOG.debug("User \"" + user + "\" belongs to super-group \"" + superGroupName + "\". " + "Permission granted for action: " + action + "."); return; } } FsPermission dirPerms = stat.getPermission(); if (user.equals(stat.getOwner())) { if (dirPerms.getUserAction().implies(action)) { return; } } else if (arrayContains(groups, stat.getGroup())) { if (dirPerms.getGroupAction().implies(action)) { return; } } else if (dirPerms.getOtherAction().implies(action)) { return; } throw new AccessControlException( "action " + action + " not permitted on path " + stat.getPath() + " for user " + user); } public static boolean isPathEncrypted(Configuration conf, URI fsUri, Path path) throws IOException { Path fullPath; if (path.isAbsolute()) { fullPath = path; } else { fullPath = path.getFileSystem(conf).makeQualified(path); } if (!"hdfs".equalsIgnoreCase(path.toUri().getScheme())) { return false; } try { HdfsAdmin hdfsAdmin = new HdfsAdmin(fsUri, conf); return (hdfsAdmin.getEncryptionZoneForPath(fullPath) != null); } catch (FileNotFoundException fnfe) { LOG.debug("Failed to get EZ for non-existent path: " + fullPath, fnfe); return false; } } private static boolean arrayContains(String[] array, String value) { if (array == null) return false; for (String element : array) { if (element.equals(value)) return true; } return false; } public static boolean runDistCpAs(List<Path> srcPaths, Path dst, Configuration conf, String doAsUser) throws IOException { UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(doAsUser, UserGroupInformation.getLoginUser()); try { return proxyUser.doAs(new PrivilegedExceptionAction<Boolean>() { @Override public Boolean run() throws Exception { return runDistCp(srcPaths, dst, conf); } }); } catch (InterruptedException e) { throw new IOException(e); } } public static boolean runDistCp(List<Path> srcPaths, Path dst, Configuration conf) throws IOException { DistCpOptions options = new DistCpOptions.Builder(srcPaths, dst).withSyncFolder(true).withCRC(true) .preserve(FileAttribute.BLOCKSIZE).build(); // Creates the command-line parameters for distcp List<String> params = constructDistCpParams(srcPaths, dst, conf); try { conf.setBoolean("mapred.mapper.new-api", true); DistCp distcp = new DistCp(conf, options); // HIVE-13704 states that we should use run() instead of execute() due to a hadoop known issue // added by HADOOP-10459 if (distcp.run(params.toArray(new String[params.size()])) == 0) { return true; } else { return false; } } catch (Exception e) { throw new IOException("Cannot execute DistCp process: " + e, e); } finally { conf.setBoolean("mapred.mapper.new-api", false); } } private static List<String> constructDistCpParams(List<Path> srcPaths, Path dst, Configuration conf) { List<String> params = new ArrayList<>(); for (Map.Entry<String, String> entry : conf.getPropsWithPrefix(DISTCP_OPTIONS_PREFIX).entrySet()) { String distCpOption = entry.getKey(); String distCpVal = entry.getValue(); params.add("-" + distCpOption); if ((distCpVal != null) && (!distCpVal.isEmpty())) { params.add(distCpVal); } } if (params.size() == 0) { // if no entries were added via conf, we initiate our defaults params.add("-update"); params.add("-pbx"); } for (Path src : srcPaths) { params.add(src.toString()); } params.add(dst.toString()); return params; } public static Path getFileIdPath(FileSystem fileSystem, Path path, long fileId) { return (fileSystem instanceof DistributedFileSystem) ? new Path(HDFS_ID_PATH_PREFIX + fileId) : path; } public static long getFileId(FileSystem fs, String path) throws IOException { return ensureDfs(fs).getClient().getFileInfo(path).getFileId(); } private static DistributedFileSystem ensureDfs(FileSystem fs) { if (!(fs instanceof DistributedFileSystem)) { throw new UnsupportedOperationException("Only supported for DFS; got " + fs.getClass()); } return (DistributedFileSystem) fs; } public static class HadoopFileStatus { private final FileStatus fileStatus; private final AclStatus aclStatus; public HadoopFileStatus(Configuration conf, FileSystem fs, Path file) throws IOException { FileStatus fileStatus = fs.getFileStatus(file); AclStatus aclStatus = null; if (Objects.equal(conf.get("dfs.namenode.acls.enabled"), "true")) { //Attempt extended Acl operations only if its enabled, but don't fail the operation regardless. try { aclStatus = fs.getAclStatus(file); } catch (Exception e) { LOG.info("Skipping ACL inheritance: File system for path " + file + " " + "does not support ACLs but dfs.namenode.acls.enabled is set to true. "); LOG.debug("The details are: " + e, e); } } this.fileStatus = fileStatus; this.aclStatus = aclStatus; } public FileStatus getFileStatus() { return fileStatus; } List<AclEntry> getAclEntries() { return aclStatus == null ? null : Collections.unmodifiableList(aclStatus.getEntries()); } @VisibleForTesting AclStatus getAclStatus() { return this.aclStatus; } } /** * Copy the permissions, group, and ACLs from a source {@link HadoopFileStatus} to a target {@link Path}. This method * will only log a warning if permissions cannot be set, no exception will be thrown. * * @param conf the {@link Configuration} used when setting permissions and ACLs * @param sourceStatus the source {@link HadoopFileStatus} to copy permissions and ACLs from * @param targetGroup the group of the target {@link Path}, if this is set and it is equal to the source group, an * extra set group operation is avoided * @param fs the {@link FileSystem} that contains the target {@link Path} * @param target the {@link Path} to copy permissions, group, and ACLs to * @param recursion recursively set permissions and ACLs on the target {@link Path} */ public static void setFullFileStatus(Configuration conf, HdfsUtils.HadoopFileStatus sourceStatus, String targetGroup, FileSystem fs, Path target, boolean recursion) { setFullFileStatus(conf, sourceStatus, targetGroup, fs, target, recursion, recursion ? new FsShell() : null); } @VisibleForTesting static void setFullFileStatus(Configuration conf, HdfsUtils.HadoopFileStatus sourceStatus, String targetGroup, FileSystem fs, Path target, boolean recursion, FsShell fsShell) { try { FileStatus fStatus = sourceStatus.getFileStatus(); String group = fStatus.getGroup(); boolean aclEnabled = Objects.equal(conf.get("dfs.namenode.acls.enabled"), "true"); FsPermission sourcePerm = fStatus.getPermission(); List<AclEntry> aclEntries = null; if (aclEnabled) { if (sourceStatus.getAclEntries() != null) { LOG.trace(sourceStatus.getAclStatus().toString()); aclEntries = new ArrayList<>(sourceStatus.getAclEntries()); removeBaseAclEntries(aclEntries); //the ACL api's also expect the tradition user/group/other permission in the form of ACL aclEntries .add(newAclEntry(AclEntryScope.ACCESS, AclEntryType.USER, sourcePerm.getUserAction())); aclEntries.add( newAclEntry(AclEntryScope.ACCESS, AclEntryType.GROUP, sourcePerm.getGroupAction())); aclEntries.add( newAclEntry(AclEntryScope.ACCESS, AclEntryType.OTHER, sourcePerm.getOtherAction())); } } if (recursion) { //use FsShell to change group, permissions, and extended ACL's recursively fsShell.setConf(conf); //If there is no group of a file, no need to call chgrp if (group != null && !group.isEmpty()) { run(fsShell, new String[] { "-chgrp", "-R", group, target.toString() }); } if (aclEnabled) { if (null != aclEntries) { //Attempt extended Acl operations only if its enabled, 8791but don't fail the operation regardless. try { //construct the -setfacl command String aclEntry = Joiner.on(",").join(aclEntries); run(fsShell, new String[] { "-setfacl", "-R", "--set", aclEntry, target.toString() }); } catch (Exception e) { LOG.info("Skipping ACL inheritance: File system for path " + target + " " + "does not support ACLs but dfs.namenode.acls.enabled is set to true. "); LOG.debug("The details are: " + e, e); } } } else { String permission = Integer.toString(sourcePerm.toShort(), 8); run(fsShell, new String[] { "-chmod", "-R", permission, target.toString() }); } } else { if (group != null && !group.isEmpty()) { if (targetGroup == null || !group.equals(targetGroup)) { fs.setOwner(target, null, group); } } if (aclEnabled) { if (null != aclEntries) { fs.setAcl(target, aclEntries); } } else { fs.setPermission(target, sourcePerm); } } } catch (Exception e) { LOG.warn("Unable to inherit permissions for file " + target + " from file " + sourceStatus.getFileStatus().getPath(), e.getMessage()); LOG.debug("Exception while inheriting permissions", e); } } /** * Removes basic permission acls (unamed acls) from the list of acl entries * @param entries acl entries to remove from. */ private static void removeBaseAclEntries(List<AclEntry> entries) { Iterables.removeIf(entries, new Predicate<AclEntry>() { @Override public boolean apply(AclEntry input) { if (input.getName() == null) { return true; } return false; } }); } /** * Create a new AclEntry with scope, type and permission (no name). * * @param scope * AclEntryScope scope of the ACL entry * @param type * AclEntryType ACL entry type * @param permission * FsAction set of permissions in the ACL entry * @return AclEntry new AclEntry */ private static AclEntry newAclEntry(AclEntryScope scope, AclEntryType type, FsAction permission) { return new AclEntry.Builder().setScope(scope).setType(type).setPermission(permission).build(); } private static void run(FsShell shell, String[] command) throws Exception { LOG.debug(ArrayUtils.toString(command)); int retval = shell.run(command); LOG.debug("Return value is :" + retval); } }