Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hdfs; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; import static org.apache.hadoop.fs.CreateFlag.CREATE; import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST; import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.io.InterruptedIOException; import java.io.PrintStream; import java.io.RandomAccessFile; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.Socket; import java.net.SocketException; import java.net.URI; import java.net.URL; import java.net.URLConnection; import java.nio.ByteBuffer; import java.security.NoSuchAlgorithmException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Random; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import com.google.common.base.Charsets; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.CacheFlag; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem.Statistics; import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclEntryScope; import org.apache.hadoop.fs.permission.AclEntryType; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.MiniDFSCluster.NameNodeInfo; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder; import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LayoutVersion; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.datatransfer.Sender; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.balancer.NameNodeConnector; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSEditLog; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.LeaseManager; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.Namesystem; import org.apache.hadoop.hdfs.server.namenode.XAttrStorage; import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStatus; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.tools.DFSAdmin; import org.apache.hadoop.hdfs.tools.JMXGet; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.net.unix.TemporarySocketDirectory; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.RefreshUserMappingsProtocol; import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.Whitebox; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.VersionInfo; import org.apache.log4j.Level; import org.junit.Assert; import org.junit.Assume; import org.apache.hadoop.util.ToolRunner; import com.google.common.annotations.VisibleForTesting; /** Utilities for HDFS tests */ public class DFSTestUtil { private static final Logger LOG = LoggerFactory.getLogger(DFSTestUtil.class); private static final Random gen = new Random(); private static final String[] dirNames = { "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine" }; private final int maxLevels; private final int maxSize; private final int minSize; private final int nFiles; private MyFile[] files; /** Creates a new instance of DFSTestUtil * * @param nFiles Number of files to be created * @param maxLevels Maximum number of directory levels * @param maxSize Maximum size for file * @param minSize Minimum size for file */ private DFSTestUtil(int nFiles, int maxLevels, int maxSize, int minSize) { this.nFiles = nFiles; this.maxLevels = maxLevels; this.maxSize = maxSize; this.minSize = minSize; } /** Creates a new instance of DFSTestUtil * * @param testName Name of the test from where this utility is used * @param nFiles Number of files to be created * @param maxLevels Maximum number of directory levels * @param maxSize Maximum size for file * @param minSize Minimum size for file */ public DFSTestUtil(String testName, int nFiles, int maxLevels, int maxSize, int minSize) { this.nFiles = nFiles; this.maxLevels = maxLevels; this.maxSize = maxSize; this.minSize = minSize; } /** * when formatting a namenode - we must provide clusterid. * @param conf * @throws IOException */ public static void formatNameNode(Configuration conf) throws IOException { String clusterId = StartupOption.FORMAT.getClusterId(); if (clusterId == null || clusterId.isEmpty()) StartupOption.FORMAT.setClusterId("testClusterID"); // Use a copy of conf as it can be altered by namenode during format. NameNode.format(new Configuration(conf)); } /** * Create a new HA-enabled configuration. */ public static Configuration newHAConfiguration(final String logicalName) { Configuration conf = new Configuration(); addHAConfiguration(conf, logicalName); return conf; } /** * Add a new HA configuration. */ public static void addHAConfiguration(Configuration conf, final String logicalName) { String nsIds = conf.get(DFSConfigKeys.DFS_NAMESERVICES); if (nsIds == null) { conf.set(DFSConfigKeys.DFS_NAMESERVICES, logicalName); } else { // append the nsid conf.set(DFSConfigKeys.DFS_NAMESERVICES, nsIds + "," + logicalName); } conf.set(DFSUtil.addKeySuffixes(HdfsClientConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, logicalName), "nn1,nn2"); conf.set(HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + "." + logicalName, ConfiguredFailoverProxyProvider.class.getName()); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); } public static void setFakeHttpAddresses(Configuration conf, final String logicalName) { conf.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, logicalName, "nn1"), "127.0.0.1:12345"); conf.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, logicalName, "nn2"), "127.0.0.1:12346"); } public static void setEditLogForTesting(FSNamesystem fsn, FSEditLog newLog) { // spies are shallow copies, must allow async log to restart its thread // so it has the new copy newLog.restart(); Whitebox.setInternalState(fsn.getFSImage(), "editLog", newLog); Whitebox.setInternalState(fsn.getFSDirectory(), "editLog", newLog); } public static void enableAllECPolicies(DistributedFileSystem fs) throws IOException { // Enable all available EC policies for (ErasureCodingPolicy ecPolicy : SystemErasureCodingPolicies.getPolicies()) { fs.enableErasureCodingPolicy(ecPolicy.getName()); } } public static ErasureCodingPolicyState getECPolicyState(final ErasureCodingPolicy policy) { final ErasureCodingPolicyInfo[] policyInfos = ErasureCodingPolicyManager.getInstance().getPolicies(); for (ErasureCodingPolicyInfo pi : policyInfos) { if (pi.getPolicy().equals(policy)) { return pi.getState(); } } throw new IllegalArgumentException("ErasureCodingPolicy <" + policy + "> doesn't exist in the policies:" + Arrays.toString(policyInfos)); } /** class MyFile contains enough information to recreate the contents of * a single file. */ private class MyFile { private String name = ""; private final int size; private final long seed; MyFile() { int nLevels = gen.nextInt(maxLevels); if (nLevels != 0) { int[] levels = new int[nLevels]; for (int idx = 0; idx < nLevels; idx++) { levels[idx] = gen.nextInt(10); } StringBuffer sb = new StringBuffer(); for (int idx = 0; idx < nLevels; idx++) { sb.append(dirNames[levels[idx]]); sb.append("/"); } name = sb.toString(); } long fidx = -1; while (fidx < 0) { fidx = gen.nextLong(); } name = name + Long.toString(fidx); size = minSize + gen.nextInt(maxSize - minSize); seed = gen.nextLong(); } String getName() { return name; } int getSize() { return size; } long getSeed() { return seed; } } public void createFiles(FileSystem fs, String topdir) throws IOException { createFiles(fs, topdir, (short) 3); } public static byte[] readFileAsBytes(FileSystem fs, Path fileName) throws IOException { try (ByteArrayOutputStream os = new ByteArrayOutputStream()) { IOUtils.copyBytes(fs.open(fileName), os, 1024); return os.toByteArray(); } } /** create nFiles with random names and directory hierarchies * with random (but reproducible) data in them. */ public void createFiles(FileSystem fs, String topdir, short replicationFactor) throws IOException { files = new MyFile[nFiles]; for (int idx = 0; idx < nFiles; idx++) { files[idx] = new MyFile(); } Path root = new Path(topdir); for (int idx = 0; idx < nFiles; idx++) { createFile(fs, new Path(root, files[idx].getName()), files[idx].getSize(), replicationFactor, files[idx].getSeed()); } } public static String readFile(FileSystem fs, Path fileName) throws IOException { byte buf[] = readFileBuffer(fs, fileName); return new String(buf, 0, buf.length); } public static byte[] readFileBuffer(FileSystem fs, Path fileName) throws IOException { try (ByteArrayOutputStream os = new ByteArrayOutputStream(); FSDataInputStream in = fs.open(fileName)) { IOUtils.copyBytes(in, os, 1024, true); return os.toByteArray(); } } public static void createFile(FileSystem fs, Path fileName, long fileLen, short replFactor, long seed) throws IOException { createFile(fs, fileName, 1024, fileLen, fs.getDefaultBlockSize(fileName), replFactor, seed); } public static void createFile(FileSystem fs, Path fileName, int bufferLen, long fileLen, long blockSize, short replFactor, long seed) throws IOException { createFile(fs, fileName, false, bufferLen, fileLen, blockSize, replFactor, seed, false); } public static void createFile(FileSystem fs, Path fileName, boolean isLazyPersist, int bufferLen, long fileLen, long blockSize, short replFactor, long seed, boolean flush) throws IOException { createFile(fs, fileName, isLazyPersist, bufferLen, fileLen, blockSize, replFactor, seed, flush, null); } public static void createFile(FileSystem fs, Path fileName, boolean isLazyPersist, int bufferLen, long fileLen, long blockSize, short replFactor, long seed, boolean flush, InetSocketAddress[] favoredNodes) throws IOException { assert bufferLen > 0; if (!fs.mkdirs(fileName.getParent())) { throw new IOException("Mkdirs failed to create " + fileName.getParent().toString()); } EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE); createFlags.add(OVERWRITE); if (isLazyPersist) { createFlags.add(LAZY_PERSIST); } try (FSDataOutputStream out = (favoredNodes == null) ? fs.create(fileName, FsPermission.getFileDefault(), createFlags, fs.getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, 4096), replFactor, blockSize, null) : ((DistributedFileSystem) fs).create(fileName, FsPermission.getDefault(), true, bufferLen, replFactor, blockSize, null, favoredNodes)) { if (fileLen > 0) { byte[] toWrite = new byte[bufferLen]; Random rb = new Random(seed); long bytesToWrite = fileLen; while (bytesToWrite > 0) { rb.nextBytes(toWrite); int bytesToWriteNext = (bufferLen < bytesToWrite) ? bufferLen : (int) bytesToWrite; out.write(toWrite, 0, bytesToWriteNext); bytesToWrite -= bytesToWriteNext; } if (flush) { out.hsync(); } } } } public static byte[] calculateFileContentsFromSeed(long seed, int length) { Random rb = new Random(seed); byte val[] = new byte[length]; rb.nextBytes(val); return val; } /** check if the files have been copied correctly. */ public boolean checkFiles(FileSystem fs, String topdir) throws IOException { Path root = new Path(topdir); for (int idx = 0; idx < nFiles; idx++) { Path fPath = new Path(root, files[idx].getName()); try (FSDataInputStream in = fs.open(fPath)) { byte[] toRead = new byte[files[idx].getSize()]; byte[] toCompare = new byte[files[idx].getSize()]; Random rb = new Random(files[idx].getSeed()); rb.nextBytes(toCompare); in.readFully(0, toRead); for (int i = 0; i < toRead.length; i++) { if (toRead[i] != toCompare[i]) { return false; } } } } return true; } void setReplication(FileSystem fs, String topdir, short value) throws IOException { Path root = new Path(topdir); for (int idx = 0; idx < nFiles; idx++) { Path fPath = new Path(root, files[idx].getName()); fs.setReplication(fPath, value); } } /* * Waits for the replication factor of all files to reach the * specified target. */ public void waitReplication(FileSystem fs, String topdir, short value) throws IOException, InterruptedException, TimeoutException { Path root = new Path(topdir); /** wait for the replication factor to settle down */ for (int idx = 0; idx < nFiles; idx++) { waitReplication(fs, new Path(root, files[idx].getName()), value); } } /* * Check if the given block in the given file is corrupt. */ public static boolean allBlockReplicasCorrupt(MiniDFSCluster cluster, Path file, int blockNo) throws IOException { try (DFSClient client = new DFSClient(new InetSocketAddress("localhost", cluster.getNameNodePort()), cluster.getConfiguration(0))) { LocatedBlocks blocks; blocks = client.getNamenode().getBlockLocations(file.toString(), 0, Long.MAX_VALUE); return blocks.get(blockNo).isCorrupt(); } } /* * Wait up to 20s for the given block to be replicated across * the requested number of racks, with the requested number of * replicas, and the requested number of replicas still needed. */ public static void waitForReplication(MiniDFSCluster cluster, ExtendedBlock b, int racks, int replicas, int neededReplicas) throws TimeoutException, InterruptedException { int curRacks = 0; int curReplicas = 0; int curNeededReplicas = 0; int count = 0; final int ATTEMPTS = 20; do { Thread.sleep(1000); int[] r = BlockManagerTestUtil.getReplicaInfo(cluster.getNamesystem(), b.getLocalBlock()); curRacks = r[0]; curReplicas = r[1]; curNeededReplicas = r[2]; count++; } while ((curRacks != racks || curReplicas != replicas || curNeededReplicas != neededReplicas) && count < ATTEMPTS); if (count == ATTEMPTS) { throw new TimeoutException("Timed out waiting for replication." + " Needed replicas = " + neededReplicas + " Cur needed replicas = " + curNeededReplicas + " Replicas = " + replicas + " Cur replicas = " + curReplicas + " Racks = " + racks + " Cur racks = " + curRacks); } } public static void waitForReplication(final DistributedFileSystem dfs, final Path file, final short replication, int waitForMillis) throws TimeoutException, InterruptedException { GenericTestUtils.waitFor(new Supplier<Boolean>() { @Override public Boolean get() { try { FileStatus stat = dfs.getFileStatus(file); BlockLocation[] locs = dfs.getFileBlockLocations(stat, 0, stat.getLen()); for (BlockLocation loc : locs) { if (replication != loc.getHosts().length) { return false; } } return true; } catch (IOException e) { LOG.info("getFileStatus on path " + file + " failed!", e); return false; } } }, 100, waitForMillis); } /** * Keep accessing the given file until the namenode reports that the * given block in the file contains the given number of corrupt replicas. */ public static void waitCorruptReplicas(FileSystem fs, FSNamesystem ns, Path file, ExtendedBlock b, int corruptRepls) throws TimeoutException, InterruptedException { int count = 0; final int ATTEMPTS = 50; int repls = ns.getBlockManager().numCorruptReplicas(b.getLocalBlock()); while (repls != corruptRepls && count < ATTEMPTS) { try { IOUtils.copyBytes(fs.open(file), new IOUtils.NullOutputStream(), 512, true); } catch (IOException e) { // Swallow exceptions } System.out.println("Waiting for " + corruptRepls + " corrupt replicas"); count++; // check more often so corrupt block reports are not easily missed for (int i = 0; i < 10; i++) { repls = ns.getBlockManager().numCorruptReplicas(b.getLocalBlock()); Thread.sleep(100); if (repls == corruptRepls) { break; } } } if (count == ATTEMPTS) { throw new TimeoutException("Timed out waiting for corrupt replicas." + " Waiting for " + corruptRepls + ", but only found " + repls); } } /* * Wait up to 20s for the given DN (IP:port) to be decommissioned */ public static void waitForDecommission(FileSystem fs, String name) throws IOException, InterruptedException, TimeoutException { DatanodeInfo dn = null; int count = 0; final int ATTEMPTS = 20; do { Thread.sleep(1000); DistributedFileSystem dfs = (DistributedFileSystem) fs; for (DatanodeInfo info : dfs.getDataNodeStats()) { if (name.equals(info.getXferAddr())) { dn = info; } } count++; } while ((dn == null || dn.isDecommissionInProgress() || !dn.isDecommissioned()) && count < ATTEMPTS); if (count == ATTEMPTS) { throw new TimeoutException("Timed out waiting for datanode " + name + " to decommission."); } } /* * Returns the index of the first datanode which has a copy * of the given block, or -1 if no such datanode exists. */ public static int firstDnWithBlock(MiniDFSCluster cluster, ExtendedBlock b) throws IOException { int numDatanodes = cluster.getDataNodes().size(); for (int i = 0; i < numDatanodes; i++) { String blockContent = cluster.readBlockOnDataNode(i, b); if (blockContent != null) { return i; } } return -1; } /* * Return the total capacity of all live DNs. */ public static long getLiveDatanodeCapacity(DatanodeManager dm) { final List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>(); dm.fetchDatanodes(live, null, false); long capacity = 0; for (final DatanodeDescriptor dn : live) { capacity += dn.getCapacity(); } return capacity; } /* * Return the capacity of the given live DN. */ public static long getDatanodeCapacity(DatanodeManager dm, int index) { final List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>(); dm.fetchDatanodes(live, null, false); return live.get(index).getCapacity(); } /* * Wait for the given # live/dead DNs, total capacity, and # vol failures. */ public static void waitForDatanodeStatus(DatanodeManager dm, int expectedLive, int expectedDead, long expectedVolFails, long expectedTotalCapacity, long timeout) throws InterruptedException, TimeoutException { final List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>(); final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>(); final int ATTEMPTS = 10; int count = 0; long currTotalCapacity = 0; int volFails = 0; do { Thread.sleep(timeout); live.clear(); dead.clear(); dm.fetchDatanodes(live, dead, false); currTotalCapacity = 0; volFails = 0; for (final DatanodeDescriptor dd : live) { currTotalCapacity += dd.getCapacity(); volFails += dd.getVolumeFailures(); } count++; } while ((expectedLive != live.size() || expectedDead != dead.size() || expectedTotalCapacity != currTotalCapacity || expectedVolFails != volFails) && count < ATTEMPTS); if (count == ATTEMPTS) { throw new TimeoutException("Timed out waiting for capacity." + " Live = " + live.size() + " Expected = " + expectedLive + " Dead = " + dead.size() + " Expected = " + expectedDead + " Total capacity = " + currTotalCapacity + " Expected = " + expectedTotalCapacity + " Vol Fails = " + volFails + " Expected = " + expectedVolFails); } } /* * Wait for the given DN to consider itself dead. */ public static void waitForDatanodeDeath(DataNode dn) throws InterruptedException, TimeoutException { final int ATTEMPTS = 10; int count = 0; do { Thread.sleep(1000); count++; } while (dn.isDatanodeUp() && count < ATTEMPTS); if (count == ATTEMPTS) { throw new TimeoutException("Timed out waiting for DN to die"); } } /** return list of filenames created as part of createFiles */ public String[] getFileNames(String topDir) { if (nFiles == 0) return new String[] {}; else { String[] fileNames = new String[nFiles]; for (int idx = 0; idx < nFiles; idx++) { fileNames[idx] = topDir + "/" + files[idx].getName(); } return fileNames; } } /** * Wait for the given file to reach the given replication factor. * @throws TimeoutException if we fail to sufficiently replicate the file */ public static void waitReplication(FileSystem fs, Path fileName, short replFactor) throws IOException, InterruptedException, TimeoutException { boolean correctReplFactor; final int ATTEMPTS = 40; int count = 0; do { correctReplFactor = true; BlockLocation locs[] = fs.getFileBlockLocations(fs.getFileStatus(fileName), 0, Long.MAX_VALUE); count++; for (int j = 0; j < locs.length; j++) { String[] hostnames = locs[j].getNames(); if (hostnames.length != replFactor) { correctReplFactor = false; System.out.println( "Block " + j + " of file " + fileName + " has replication factor " + hostnames.length + " (desired " + replFactor + "); locations " + Joiner.on(' ').join(hostnames)); Thread.sleep(1000); break; } } if (correctReplFactor) { System.out.println( "All blocks of file " + fileName + " verified to have replication factor " + replFactor); } } while (!correctReplFactor && count < ATTEMPTS); if (count == ATTEMPTS) { throw new TimeoutException( "Timed out waiting for " + fileName + " to reach " + replFactor + " replicas"); } } /** delete directory and everything underneath it.*/ public void cleanup(FileSystem fs, String topdir) throws IOException { Path root = new Path(topdir); fs.delete(root, true); files = null; } public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException { try (HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path)) { in.readByte(); return in.getCurrentBlock(); } } public static List<LocatedBlock> getAllBlocks(FSDataInputStream in) throws IOException { return ((HdfsDataInputStream) in).getAllBlocks(); } public static List<LocatedBlock> getAllBlocks(FileSystem fs, Path path) throws IOException { try (HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path)) { return in.getAllBlocks(); } } public static Token<BlockTokenIdentifier> getBlockToken(FSDataOutputStream out) { return ((DFSOutputStream) out.getWrappedStream()).getBlockToken(); } public static String readFile(File f) throws IOException { try (BufferedReader in = new BufferedReader(new FileReader(f))) { StringBuilder b = new StringBuilder(); int c; while ((c = in.read()) != -1) { b.append((char) c); } return b.toString(); } } public static byte[] readFileAsBytes(File f) throws IOException { try (ByteArrayOutputStream os = new ByteArrayOutputStream()) { IOUtils.copyBytes(new FileInputStream(f), os, 1024); return os.toByteArray(); } } /* Write the given bytes to the given file */ public static void writeFile(FileSystem fs, Path p, byte[] bytes) throws IOException { if (fs.exists(p)) { fs.delete(p, true); } try (InputStream is = new ByteArrayInputStream(bytes); FSDataOutputStream os = fs.create(p)) { IOUtils.copyBytes(is, os, bytes.length); } } /* Write the given bytes to the given file using the specified blockSize */ public static void writeFile(FileSystem fs, Path p, byte[] bytes, long blockSize) throws IOException { if (fs.exists(p)) { fs.delete(p, true); } try (InputStream is = new ByteArrayInputStream(bytes); FSDataOutputStream os = fs.create(p, false, 4096, fs.getDefaultReplication(p), blockSize)) { IOUtils.copyBytes(is, os, bytes.length); } } /* Write the given string to the given file */ public static void writeFile(FileSystem fs, Path p, String s) throws IOException { writeFile(fs, p, s.getBytes()); } /* Append the given string to the given file */ public static void appendFile(FileSystem fs, Path p, String s) throws IOException { assert fs.exists(p); try (InputStream is = new ByteArrayInputStream(s.getBytes()); FSDataOutputStream os = fs.append(p)) { IOUtils.copyBytes(is, os, s.length()); } } /** * Append specified length of bytes to a given file * @param fs The file system * @param p Path of the file to append * @param length Length of bytes to append to the file * @throws IOException */ public static void appendFile(FileSystem fs, Path p, int length) throws IOException { assert fs.exists(p); assert length >= 0; byte[] toAppend = new byte[length]; Random random = new Random(); random.nextBytes(toAppend); try (FSDataOutputStream out = fs.append(p)) { out.write(toAppend); } } /** * Append specified length of bytes to a given file, starting with new block. * @param fs The file system * @param p Path of the file to append * @param length Length of bytes to append to the file * @throws IOException */ public static void appendFileNewBlock(DistributedFileSystem fs, Path p, int length) throws IOException { assert length >= 0; byte[] toAppend = new byte[length]; Random random = new Random(); random.nextBytes(toAppend); appendFileNewBlock(fs, p, toAppend); } /** * Append specified bytes to a given file, starting with new block. * * @param fs The file system * @param p Path of the file to append * @param bytes The data to append * @throws IOException */ public static void appendFileNewBlock(DistributedFileSystem fs, Path p, byte[] bytes) throws IOException { assert fs.exists(p); try (FSDataOutputStream out = fs.append(p, EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null)) { out.write(bytes); } } /** * @return url content as string (UTF-8 encoding assumed) */ public static String urlGet(URL url) throws IOException { return new String(urlGetBytes(url), Charsets.UTF_8); } /** * @return URL contents as a byte array */ public static byte[] urlGetBytes(URL url) throws IOException { URLConnection conn = url.openConnection(); HttpURLConnection hc = (HttpURLConnection) conn; assertEquals(HttpURLConnection.HTTP_OK, hc.getResponseCode()); ByteArrayOutputStream out = new ByteArrayOutputStream(); IOUtils.copyBytes(conn.getInputStream(), out, 4096, true); return out.toByteArray(); } /** * mock class to get group mapping for fake users * */ static class MockUnixGroupsMapping extends ShellBasedUnixGroupsMapping { static Map<String, String[]> fakeUser2GroupsMap; private static final List<String> defaultGroups; static { defaultGroups = new ArrayList<String>(1); defaultGroups.add("supergroup"); fakeUser2GroupsMap = new HashMap<String, String[]>(); } @Override public List<String> getGroups(String user) throws IOException { boolean found = false; // check to see if this is one of fake users List<String> l = new ArrayList<String>(); for (String u : fakeUser2GroupsMap.keySet()) { if (user.equals(u)) { found = true; for (String gr : fakeUser2GroupsMap.get(u)) { l.add(gr); } } } // default if (!found) { l = super.getGroups(user); if (l.size() == 0) { System.out.println("failed to get real group for " + user + "; using default"); return defaultGroups; } } return l; } } /** * update the configuration with fake class for mapping user to groups * @param conf * @param map - user to groups mapping */ static public void updateConfWithFakeGroupMapping(Configuration conf, Map<String, String[]> map) { if (map != null) { MockUnixGroupsMapping.fakeUser2GroupsMap = map; } // fake mapping user to groups conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, DFSTestUtil.MockUnixGroupsMapping.class, ShellBasedUnixGroupsMapping.class); } /** * Get a FileSystem instance as specified user in a doAs block. */ static public FileSystem getFileSystemAs(UserGroupInformation ugi, final Configuration conf) throws IOException { try { return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { @Override public FileSystem run() throws Exception { return FileSystem.get(conf); } }); } catch (InterruptedException e) { throw (InterruptedIOException) new InterruptedIOException().initCause(e); } } public static byte[] generateSequentialBytes(int start, int length) { byte[] result = new byte[length]; for (int i = 0; i < length; i++) { result[i] = (byte) ((start + i) % 127); } return result; } public static Statistics getStatistics(FileSystem fs) { return FileSystem.getStatistics(fs.getUri().getScheme(), fs.getClass()); } /** * Load file into byte[] */ public static byte[] loadFile(String filename) throws IOException { File file = new File(filename); try (DataInputStream in = new DataInputStream(new FileInputStream(file))) { byte[] content = new byte[(int) file.length()]; in.readFully(content); return content; } } /** For {@link TestTransferRbw} */ public static BlockOpResponseProto transferRbw(final ExtendedBlock b, final DFSClient dfsClient, final DatanodeInfo... datanodes) throws IOException { assertEquals(2, datanodes.length); final long writeTimeout = dfsClient.getDatanodeWriteTimeout(datanodes.length); try (Socket s = DataStreamer.createSocketForPipeline(datanodes[0], datanodes.length, dfsClient); DataOutputStream out = new DataOutputStream( new BufferedOutputStream(NetUtils.getOutputStream(s, writeTimeout), DFSUtilClient.getSmallBufferSize(dfsClient.getConfiguration()))); DataInputStream in = new DataInputStream(NetUtils.getInputStream(s))) { // send the request new Sender(out).transferBlock(b, new Token<BlockTokenIdentifier>(), dfsClient.clientName, new DatanodeInfo[] { datanodes[1] }, new StorageType[] { StorageType.DEFAULT }, new String[0]); out.flush(); return BlockOpResponseProto.parseDelimitedFrom(in); } } public static void setFederatedConfiguration(MiniDFSCluster cluster, Configuration conf) { Set<String> nameservices = new HashSet<String>(); for (NameNodeInfo info : cluster.getNameNodeInfos()) { assert info.nameserviceId != null; nameservices.add(info.nameserviceId); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, info.nameserviceId), DFSUtil .createUri(HdfsConstants.HDFS_URI_SCHEME, info.nameNode.getNameNodeAddress()).toString()); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, info.nameserviceId), DFSUtil .createUri(HdfsConstants.HDFS_URI_SCHEME, info.nameNode.getNameNodeAddress()).toString()); } conf.set(DFSConfigKeys.DFS_NAMESERVICES, Joiner.on(",").join(nameservices)); } public static void setFederatedHAConfiguration(MiniDFSCluster cluster, Configuration conf) { Map<String, List<String>> nameservices = Maps.newHashMap(); for (NameNodeInfo info : cluster.getNameNodeInfos()) { Preconditions.checkState(info.nameserviceId != null); List<String> nns = nameservices.get(info.nameserviceId); if (nns == null) { nns = Lists.newArrayList(); nameservices.put(info.nameserviceId, nns); } nns.add(info.nnId); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, info.nameserviceId, info.nnId), DFSUtil .createUri(HdfsConstants.HDFS_URI_SCHEME, info.nameNode.getNameNodeAddress()).toString()); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, info.nameserviceId, info.nnId), DFSUtil.createUri(HdfsConstants.HDFS_URI_SCHEME, info.nameNode.getNameNodeAddress()) .toString()); } for (Map.Entry<String, List<String>> entry : nameservices.entrySet()) { conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, entry.getKey()), Joiner.on(",").join(entry.getValue())); conf.set(HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + "." + entry.getKey(), ConfiguredFailoverProxyProvider.class.getName()); } conf.set(DFSConfigKeys.DFS_NAMESERVICES, Joiner.on(",").join(nameservices.keySet())); } private static DatanodeID getDatanodeID(String ipAddr) { return new DatanodeID(ipAddr, "localhost", UUID.randomUUID().toString(), DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT); } public static DatanodeID getLocalDatanodeID() { return getDatanodeID("127.0.0.1"); } public static DatanodeID getLocalDatanodeID(int port) { return new DatanodeID("127.0.0.1", "localhost", UUID.randomUUID().toString(), port, port, port, port); } public static DatanodeDescriptor getLocalDatanodeDescriptor() { return new DatanodeDescriptor(getLocalDatanodeID()); } public static DatanodeInfo getLocalDatanodeInfo() { return new DatanodeInfoBuilder().setNodeID(getLocalDatanodeID()).build(); } public static DatanodeInfo getDatanodeInfo(String ipAddr) { return new DatanodeInfoBuilder().setNodeID(getDatanodeID(ipAddr)).build(); } public static DatanodeInfo getLocalDatanodeInfo(int port) { return new DatanodeInfoBuilder().setNodeID(getLocalDatanodeID(port)).build(); } public static DatanodeInfo getDatanodeInfo(String ipAddr, String host, int port) { return new DatanodeInfoBuilder().setNodeID(new DatanodeID(ipAddr, host, UUID.randomUUID().toString(), port, DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT)).build(); } public static DatanodeInfo getLocalDatanodeInfo(String ipAddr, String hostname, AdminStates adminState) { return new DatanodeInfoBuilder().setIpAddr(ipAddr).setHostName(hostname).setDatanodeUuid("") .setXferPort(DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT) .setInfoPort(DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT) .setInfoSecurePort(DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT) .setIpcPort(DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT).setCapacity(1L).setDfsUsed(2L) .setRemaining(3L).setBlockPoolUsed(4L).setCacheCapacity(0L).setCacheUsed(0L).setLastUpdate(0L) .setLastUpdateMonotonic(5).setXceiverCount(6).setNetworkLocation("local").setAdminState(adminState) .build(); } public static DatanodeDescriptor getDatanodeDescriptor(String ipAddr, String rackLocation) { return getDatanodeDescriptor(ipAddr, DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT, rackLocation); } public static DatanodeDescriptor getDatanodeDescriptor(String ipAddr, String rackLocation, String hostname) { return getDatanodeDescriptor(ipAddr, DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT, rackLocation, hostname); } public static DatanodeStorageInfo createDatanodeStorageInfo(String storageID, String ip) { return createDatanodeStorageInfo(storageID, ip, "defaultRack", "host"); } public static DatanodeStorageInfo[] createDatanodeStorageInfos(String[] racks) { return createDatanodeStorageInfos(racks, null); } public static DatanodeStorageInfo[] createDatanodeStorageInfos(String[] racks, String[] hostnames) { return createDatanodeStorageInfos(racks.length, racks, hostnames); } public static DatanodeStorageInfo[] createDatanodeStorageInfos(int n) { return createDatanodeStorageInfos(n, null, null); } public static DatanodeStorageInfo[] createDatanodeStorageInfos(int n, String[] racks, String[] hostnames) { return createDatanodeStorageInfos(n, racks, hostnames, null); } public static DatanodeStorageInfo[] createDatanodeStorageInfos(int n, String[] racks, String[] hostnames, StorageType[] types) { DatanodeStorageInfo[] storages = new DatanodeStorageInfo[n]; for (int i = storages.length; i > 0;) { final String storageID = "s" + i; final String ip = i + "." + i + "." + i + "." + i; i--; final String rack = (racks != null && i < racks.length) ? racks[i] : "defaultRack"; final String hostname = (hostnames != null && i < hostnames.length) ? hostnames[i] : "host"; final StorageType type = (types != null && i < types.length) ? types[i] : StorageType.DEFAULT; storages[i] = createDatanodeStorageInfo(storageID, ip, rack, hostname, type, null); } return storages; } public static DatanodeStorageInfo createDatanodeStorageInfo(String storageID, String ip, String rack, String hostname) { return createDatanodeStorageInfo(storageID, ip, rack, hostname, StorageType.DEFAULT, null); } public static DatanodeStorageInfo createDatanodeStorageInfo(String storageID, String ip, String rack, String hostname, StorageType type, String upgradeDomain) { final DatanodeStorage storage = new DatanodeStorage(storageID, DatanodeStorage.State.NORMAL, type); final DatanodeDescriptor dn = BlockManagerTestUtil.getDatanodeDescriptor(ip, rack, storage, hostname); if (upgradeDomain != null) { dn.setUpgradeDomain(upgradeDomain); } return BlockManagerTestUtil.newDatanodeStorageInfo(dn, storage); } public static DatanodeDescriptor[] toDatanodeDescriptor(DatanodeStorageInfo[] storages) { DatanodeDescriptor[] datanodes = new DatanodeDescriptor[storages.length]; for (int i = 0; i < datanodes.length; i++) { datanodes[i] = storages[i].getDatanodeDescriptor(); } return datanodes; } public static DatanodeDescriptor getDatanodeDescriptor(String ipAddr, int port, String rackLocation, String hostname) { DatanodeID dnId = new DatanodeID(ipAddr, hostname, UUID.randomUUID().toString(), port, DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT); return new DatanodeDescriptor(dnId, rackLocation); } public static DatanodeDescriptor getDatanodeDescriptor(String ipAddr, int port, String rackLocation) { return getDatanodeDescriptor(ipAddr, port, rackLocation, "host"); } public static DatanodeRegistration getLocalDatanodeRegistration() { return new DatanodeRegistration(getLocalDatanodeID(), new StorageInfo(NodeType.DATA_NODE), new ExportedBlockKeys(), VersionInfo.getVersion()); } /** Copy one file's contents into the other **/ public static void copyFile(File src, File dest) throws IOException { FileUtils.copyFile(src, dest); } public static class Builder { private int maxLevels = 3; private int maxSize = 8 * 1024; private int minSize = 1; private int nFiles = 1; public Builder() { } public Builder setName(String string) { return this; } public Builder setNumFiles(int nFiles) { this.nFiles = nFiles; return this; } public Builder setMaxLevels(int maxLevels) { this.maxLevels = maxLevels; return this; } public Builder setMaxSize(int maxSize) { this.maxSize = maxSize; return this; } public Builder setMinSize(int minSize) { this.minSize = minSize; return this; } public DFSTestUtil build() { return new DFSTestUtil(nFiles, maxLevels, maxSize, minSize); } } /** * Run a set of operations and generate all edit logs */ public static void runOperations(MiniDFSCluster cluster, DistributedFileSystem filesystem, Configuration conf, long blockSize, int nnIndex) throws IOException { // create FileContext for rename2 FileContext fc = FileContext.getFileContext(cluster.getURI(0), conf); // OP_ADD 0 final Path pathFileCreate = new Path("/file_create"); FSDataOutputStream s = filesystem.create(pathFileCreate); // OP_CLOSE 9 s.close(); // OP_APPEND 47 FSDataOutputStream s2 = filesystem.append(pathFileCreate, 4096, null); s2.close(); // OP_UPDATE_BLOCKS 25 final String updateBlockFile = "/update_blocks"; FSDataOutputStream fout = filesystem.create(new Path(updateBlockFile), true, 4096, (short) 1, 4096L); fout.write(1); fout.hflush(); long fileId = ((DFSOutputStream) fout.getWrappedStream()).getFileId(); DFSClient dfsclient = DFSClientAdapter.getDFSClient(filesystem); LocatedBlocks blocks = dfsclient.getNamenode().getBlockLocations(updateBlockFile, 0, Integer.MAX_VALUE); dfsclient.getNamenode().abandonBlock(blocks.get(0).getBlock(), fileId, updateBlockFile, dfsclient.clientName); fout.close(); // OP_SET_STORAGE_POLICY 45 filesystem.setStoragePolicy(pathFileCreate, HdfsConstants.HOT_STORAGE_POLICY_NAME); // OP_RENAME_OLD 1 final Path pathFileMoved = new Path("/file_moved"); filesystem.rename(pathFileCreate, pathFileMoved); // OP_DELETE 2 filesystem.delete(pathFileMoved, false); // OP_MKDIR 3 Path pathDirectoryMkdir = new Path("/directory_mkdir"); filesystem.mkdirs(pathDirectoryMkdir); // OP_ALLOW_SNAPSHOT 29 filesystem.allowSnapshot(pathDirectoryMkdir); // OP_DISALLOW_SNAPSHOT 30 filesystem.disallowSnapshot(pathDirectoryMkdir); // OP_CREATE_SNAPSHOT 26 String ssName = "snapshot1"; filesystem.allowSnapshot(pathDirectoryMkdir); filesystem.createSnapshot(pathDirectoryMkdir, ssName); // OP_RENAME_SNAPSHOT 28 String ssNewName = "snapshot2"; filesystem.renameSnapshot(pathDirectoryMkdir, ssName, ssNewName); // OP_DELETE_SNAPSHOT 27 filesystem.deleteSnapshot(pathDirectoryMkdir, ssNewName); // OP_SET_REPLICATION 4 s = filesystem.create(pathFileCreate); s.close(); filesystem.setReplication(pathFileCreate, (short) 1); // OP_SET_PERMISSIONS 7 Short permission = 0777; filesystem.setPermission(pathFileCreate, new FsPermission(permission)); // OP_SET_OWNER 8 filesystem.setOwner(pathFileCreate, new String("newOwner"), null); // OP_CLOSE 9 see above // OP_SET_GENSTAMP 10 see above // OP_SET_NS_QUOTA 11 obsolete // OP_CLEAR_NS_QUOTA 12 obsolete // OP_TIMES 13 long mtime = 1285195527000L; // Wed, 22 Sep 2010 22:45:27 GMT long atime = mtime; filesystem.setTimes(pathFileCreate, mtime, atime); // OP_SET_QUOTA 14 filesystem.setQuota(pathDirectoryMkdir, 1000L, HdfsConstants.QUOTA_DONT_SET); // OP_SET_QUOTA_BY_STORAGETYPE filesystem.setQuotaByStorageType(pathDirectoryMkdir, StorageType.SSD, 888L); // OP_RENAME 15 fc.rename(pathFileCreate, pathFileMoved, Rename.NONE); // OP_CONCAT_DELETE 16 Path pathConcatTarget = new Path("/file_concat_target"); Path[] pathConcatFiles = new Path[2]; pathConcatFiles[0] = new Path("/file_concat_0"); pathConcatFiles[1] = new Path("/file_concat_1"); long length = blockSize * 3; // multiple of blocksize for concat short replication = 1; long seed = 1; DFSTestUtil.createFile(filesystem, pathConcatTarget, length, replication, seed); DFSTestUtil.createFile(filesystem, pathConcatFiles[0], length, replication, seed); DFSTestUtil.createFile(filesystem, pathConcatFiles[1], length, replication, seed); filesystem.concat(pathConcatTarget, pathConcatFiles); // OP_TRUNCATE 46 length = blockSize * 2; DFSTestUtil.createFile(filesystem, pathFileCreate, length, replication, seed); filesystem.truncate(pathFileCreate, blockSize); // OP_SYMLINK 17 Path pathSymlink = new Path("/file_symlink"); fc.createSymlink(pathConcatTarget, pathSymlink, false); // OP_REASSIGN_LEASE 22 String filePath = "/hard-lease-recovery-test"; byte[] bytes = "foo-bar-baz".getBytes(); DFSClientAdapter.stopLeaseRenewer(filesystem); FSDataOutputStream leaseRecoveryPath = filesystem.create(new Path(filePath)); leaseRecoveryPath.write(bytes); leaseRecoveryPath.hflush(); // Set the hard lease timeout to 1 second. cluster.setLeasePeriod(60 * 1000, 1000, nnIndex); // wait for lease recovery to complete LocatedBlocks locatedBlocks; do { try { Thread.sleep(1000); } catch (InterruptedException e) { } locatedBlocks = DFSClientAdapter.callGetBlockLocations(cluster.getNameNodeRpc(nnIndex), filePath, 0L, bytes.length); } while (locatedBlocks.isUnderConstruction()); // OP_ADD_CACHE_POOL filesystem.addCachePool(new CachePoolInfo("pool1")); // OP_MODIFY_CACHE_POOL filesystem.modifyCachePool(new CachePoolInfo("pool1").setLimit(99l)); // OP_ADD_PATH_BASED_CACHE_DIRECTIVE long id = filesystem.addCacheDirective(new CacheDirectiveInfo.Builder().setPath(new Path("/path")) .setReplication((short) 1).setPool("pool1").build(), EnumSet.of(CacheFlag.FORCE)); // OP_MODIFY_PATH_BASED_CACHE_DIRECTIVE filesystem.modifyCacheDirective( new CacheDirectiveInfo.Builder().setId(id).setReplication((short) 2).build(), EnumSet.of(CacheFlag.FORCE)); // OP_REMOVE_PATH_BASED_CACHE_DIRECTIVE filesystem.removeCacheDirective(id); // OP_REMOVE_CACHE_POOL filesystem.removeCachePool("pool1"); // OP_SET_ACL List<AclEntry> aclEntryList = Lists.newArrayList(); aclEntryList.add(new AclEntry.Builder().setPermission(FsAction.READ_WRITE).setScope(AclEntryScope.ACCESS) .setType(AclEntryType.USER).build()); aclEntryList.add(new AclEntry.Builder().setName("user").setPermission(FsAction.READ_WRITE) .setScope(AclEntryScope.ACCESS).setType(AclEntryType.USER).build()); aclEntryList.add(new AclEntry.Builder().setPermission(FsAction.WRITE).setScope(AclEntryScope.ACCESS) .setType(AclEntryType.GROUP).build()); aclEntryList.add(new AclEntry.Builder().setPermission(FsAction.NONE).setScope(AclEntryScope.ACCESS) .setType(AclEntryType.OTHER).build()); filesystem.setAcl(pathConcatTarget, aclEntryList); // OP_SET_XATTR filesystem.setXAttr(pathConcatTarget, "user.a1", new byte[] { 0x31, 0x32, 0x33 }); filesystem.setXAttr(pathConcatTarget, "user.a2", new byte[] { 0x37, 0x38, 0x39 }); // OP_REMOVE_XATTR filesystem.removeXAttr(pathConcatTarget, "user.a2"); // OP_ADD_ERASURE_CODING_POLICY ErasureCodingPolicy newPolicy1 = new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA, 8 * 1024); ErasureCodingPolicy[] policyArray = new ErasureCodingPolicy[] { newPolicy1 }; AddErasureCodingPolicyResponse[] responses = filesystem.addErasureCodingPolicies(policyArray); newPolicy1 = responses[0].getPolicy(); // OP_ADD_ERASURE_CODING_POLICY - policy with extra options Map<String, String> extraOptions = new HashMap<String, String>(); extraOptions.put("dummyKey", "dummyValue"); ECSchema schema = new ECSchema(ErasureCodeConstants.RS_CODEC_NAME, 6, 10, extraOptions); ErasureCodingPolicy newPolicy2 = new ErasureCodingPolicy(schema, 4 * 1024); policyArray = new ErasureCodingPolicy[] { newPolicy2 }; responses = filesystem.addErasureCodingPolicies(policyArray); newPolicy2 = responses[0].getPolicy(); // OP_ENABLE_ERASURE_CODING_POLICY filesystem.enableErasureCodingPolicy(newPolicy1.getName()); filesystem.enableErasureCodingPolicy(newPolicy2.getName()); // OP_DISABLE_ERASURE_CODING_POLICY filesystem.disableErasureCodingPolicy(newPolicy1.getName()); filesystem.disableErasureCodingPolicy(newPolicy2.getName()); // OP_REMOVE_ERASURE_CODING_POLICY filesystem.removeErasureCodingPolicy(newPolicy1.getName()); filesystem.removeErasureCodingPolicy(newPolicy2.getName()); // OP_ADD on erasure coding directory Path ecDir = new Path("/ec"); filesystem.mkdirs(ecDir); final ErasureCodingPolicy defaultEcPolicy = SystemErasureCodingPolicies .getByID(SystemErasureCodingPolicies.RS_6_3_POLICY_ID); final ErasureCodingPolicy ecPolicyRS32 = SystemErasureCodingPolicies .getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID); filesystem.enableErasureCodingPolicy(ecPolicyRS32.getName()); filesystem.enableErasureCodingPolicy(defaultEcPolicy.getName()); filesystem.setErasureCodingPolicy(ecDir, defaultEcPolicy.getName()); try (FSDataOutputStream out = filesystem.createFile(new Path(ecDir, "replicated")).replicate().build()) { out.write("replicated".getBytes()); } try (FSDataOutputStream out = filesystem.createFile(new Path(ecDir, "RS-3-2")) .ecPolicyName(ecPolicyRS32.getName()).blockSize(1024 * 1024).build()) { out.write("RS-3-2".getBytes()); } } public static void abortStream(DFSOutputStream out) throws IOException { out.abort(); } public static void setPipeline(DFSOutputStream out, LocatedBlock lastBlock) throws IOException { out.getStreamer().setPipelineInConstruction(lastBlock); } public static byte[] asArray(ByteBuffer buf) { byte arr[] = new byte[buf.remaining()]; buf.duplicate().get(arr); return arr; } /** * Blocks until cache usage hits the expected new value. */ public static long verifyExpectedCacheUsage(final long expectedCacheUsed, final long expectedBlocks, final FsDatasetSpi<?> fsd) throws Exception { GenericTestUtils.waitFor(new Supplier<Boolean>() { private int tries = 0; @Override public Boolean get() { long curCacheUsed = fsd.getCacheUsed(); long curBlocks = fsd.getNumBlocksCached(); if ((curCacheUsed != expectedCacheUsed) || (curBlocks != expectedBlocks)) { if (tries++ > 10) { LOG.info("verifyExpectedCacheUsage: have " + curCacheUsed + "/" + expectedCacheUsed + " bytes cached; " + curBlocks + "/" + expectedBlocks + " blocks cached. " + "memlock limit = " + NativeIO.POSIX.getCacheManipulator().getMemlockLimit() + ". Waiting..."); } return false; } LOG.info("verifyExpectedCacheUsage: got " + curCacheUsed + "/" + expectedCacheUsed + " bytes cached; " + curBlocks + "/" + expectedBlocks + " blocks cached. " + "memlock limit = " + NativeIO.POSIX.getCacheManipulator().getMemlockLimit()); return true; } }, 100, 120000); return expectedCacheUsed; } /** * Round a long value up to a multiple of a factor. * * @param val The value. * @param factor The factor to round up to. Must be > 1. * @return The rounded value. */ public static long roundUpToMultiple(long val, int factor) { assert (factor > 1); long c = (val + factor - 1) / factor; return c * factor; } public static void checkComponentsEquals(byte[][] expected, byte[][] actual) { assertEquals("expected: " + DFSUtil.byteArray2PathString(expected) + ", actual: " + DFSUtil.byteArray2PathString(actual), expected.length, actual.length); int i = 0; for (byte[] e : expected) { byte[] actualComponent = actual[i++]; assertTrue( "expected: " + DFSUtil.bytes2String(e) + ", actual: " + DFSUtil.bytes2String(actualComponent), Arrays.equals(e, actualComponent)); } } /** * A short-circuit test context which makes it easier to get a short-circuit * configuration and set everything up. */ public static class ShortCircuitTestContext implements Closeable { private final String testName; private final TemporarySocketDirectory sockDir; private boolean closed = false; private final boolean formerTcpReadsDisabled; public ShortCircuitTestContext(String testName) { this.testName = testName; this.sockDir = new TemporarySocketDirectory(); DomainSocket.disableBindPathValidation(); formerTcpReadsDisabled = DFSInputStream.tcpReadsDisabledForTesting; Assume.assumeTrue(DomainSocket.getLoadingFailureReason() == null); } public Configuration newConfiguration() { Configuration conf = new Configuration(); conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true); conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), testName + "._PORT.sock").getAbsolutePath()); return conf; } public String getTestName() { return testName; } public void close() throws IOException { if (closed) return; closed = true; DFSInputStream.tcpReadsDisabledForTesting = formerTcpReadsDisabled; sockDir.close(); } } /** * Verify that two files have the same contents. * * @param fs The file system containing the two files. * @param p1 The path of the first file. * @param p2 The path of the second file. * @param len The length of the two files. * @throws IOException */ public static void verifyFilesEqual(FileSystem fs, Path p1, Path p2, int len) throws IOException { try (FSDataInputStream in1 = fs.open(p1); FSDataInputStream in2 = fs.open(p2)) { for (int i = 0; i < len; i++) { assertEquals("Mismatch at byte " + i, in1.read(), in2.read()); } } } /** * Verify that two files have different contents. * * @param fs The file system containing the two files. * @param p1 The path of the first file. * @param p2 The path of the second file. * @param len The length of the two files. * @throws IOException */ public static void verifyFilesNotEqual(FileSystem fs, Path p1, Path p2, int len) throws IOException { try (FSDataInputStream in1 = fs.open(p1); FSDataInputStream in2 = fs.open(p2)) { for (int i = 0; i < len; i++) { if (in1.read() != in2.read()) { return; } } fail("files are equal, but should not be"); } } /** * Helper function that verified blocks of a file are placed on the * expected storage type. * * @param fs The file system containing the the file. * @param client The DFS client used to access the file * @param path name to the file to verify * @param storageType expected storage type * @returns true if file exists and its blocks are located on the expected * storage type. * false otherwise. */ public static boolean verifyFileReplicasOnStorageType(FileSystem fs, DFSClient client, Path path, StorageType storageType) throws IOException { if (!fs.exists(path)) { LOG.info("verifyFileReplicasOnStorageType: file " + path + "does not exist"); return false; } long fileLength = client.getFileInfo(path.toString()).getLen(); LocatedBlocks locatedBlocks = client.getLocatedBlocks(path.toString(), 0, fileLength); for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) { if (locatedBlock.getStorageTypes()[0] != storageType) { LOG.info("verifyFileReplicasOnStorageType: for file " + path + ". Expect blk" + locatedBlock + " on Type: " + storageType + ". Actual Type: " + locatedBlock.getStorageTypes()[0]); return false; } } return true; } /** * Verify the aggregated {@link ClientProtocol#getStats()} block counts equal * the sum of {@link ClientProtocol#getReplicatedBlockStats()} and * {@link ClientProtocol#getECBlockGroupStats()}. * @throws Exception */ public static void verifyClientStats(Configuration conf, MiniDFSCluster cluster) throws Exception { ClientProtocol client = NameNodeProxies .createProxy(conf, cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy(); long[] aggregatedStats = cluster.getNameNode().getRpcServer().getStats(); ReplicatedBlockStats replicatedBlockStats = client.getReplicatedBlockStats(); ECBlockGroupStats ecBlockGroupStats = client.getECBlockGroupStats(); assertEquals("Under replicated stats not matching!", aggregatedStats[ClientProtocol.GET_STATS_LOW_REDUNDANCY_IDX], aggregatedStats[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX]); assertEquals("Low redundancy stats not matching!", aggregatedStats[ClientProtocol.GET_STATS_LOW_REDUNDANCY_IDX], replicatedBlockStats.getLowRedundancyBlocks() + ecBlockGroupStats.getLowRedundancyBlockGroups()); assertEquals("Corrupt blocks stats not matching!", aggregatedStats[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX], replicatedBlockStats.getCorruptBlocks() + ecBlockGroupStats.getCorruptBlockGroups()); assertEquals("Missing blocks stats not matching!", aggregatedStats[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX], replicatedBlockStats.getMissingReplicaBlocks() + ecBlockGroupStats.getMissingBlockGroups()); assertEquals("Missing blocks with replication factor one not matching!", aggregatedStats[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX], replicatedBlockStats.getMissingReplicationOneBlocks()); assertEquals("Bytes in future blocks stats not matching!", aggregatedStats[ClientProtocol.GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX], replicatedBlockStats.getBytesInFutureBlocks() + ecBlockGroupStats.getBytesInFutureBlockGroups()); assertEquals("Pending deletion blocks stats not matching!", aggregatedStats[ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX], replicatedBlockStats.getPendingDeletionBlocks() + ecBlockGroupStats.getPendingDeletionBlocks()); } /** * Helper function to create a key in the Key Provider. Defaults * to the first indexed NameNode's Key Provider. * * @param keyName The name of the key to create * @param cluster The cluster to create it in * @param conf Configuration to use */ public static void createKey(String keyName, MiniDFSCluster cluster, Configuration conf) throws NoSuchAlgorithmException, IOException { createKey(keyName, cluster, 0, conf); } /** * Helper function to create a key in the Key Provider. * * @param keyName The name of the key to create * @param cluster The cluster to create it in * @param idx The NameNode index * @param conf Configuration to use */ public static void createKey(String keyName, MiniDFSCluster cluster, int idx, Configuration conf) throws NoSuchAlgorithmException, IOException { NameNode nn = cluster.getNameNode(idx); KeyProvider provider = nn.getNamesystem().getProvider(); final KeyProvider.Options options = KeyProvider.options(conf); options.setDescription(keyName); options.setBitLength(128); provider.createKey(keyName, options); provider.flush(); } /** * @return the node which is expected to run the recovery of the * given block, which is known to be under construction inside the * given NameNOde. */ public static DatanodeDescriptor getExpectedPrimaryNode(NameNode nn, ExtendedBlock blk) { BlockManager bm0 = nn.getNamesystem().getBlockManager(); BlockInfo storedBlock = bm0.getStoredBlock(blk.getLocalBlock()); assertTrue("Block " + blk + " should be under construction, " + "got: " + storedBlock, !storedBlock.isComplete()); // We expect that the replica with the most recent heart beat will be // the one to be in charge of the synchronization / recovery protocol. final DatanodeStorageInfo[] storages = storedBlock.getUnderConstructionFeature() .getExpectedStorageLocations(); DatanodeStorageInfo expectedPrimary = storages[0]; long mostRecentLastUpdate = expectedPrimary.getDatanodeDescriptor().getLastUpdateMonotonic(); for (int i = 1; i < storages.length; i++) { final long lastUpdate = storages[i].getDatanodeDescriptor().getLastUpdateMonotonic(); if (lastUpdate > mostRecentLastUpdate) { expectedPrimary = storages[i]; mostRecentLastUpdate = lastUpdate; } } return expectedPrimary.getDatanodeDescriptor(); } public static void toolRun(Tool tool, String cmd, int retcode, String contain) throws Exception { String[] cmds = StringUtils.split(cmd, ' '); System.out.flush(); System.err.flush(); PrintStream origOut = System.out; PrintStream origErr = System.err; String output = null; int ret = 0; try { ByteArrayOutputStream bs = new ByteArrayOutputStream(1024); try (PrintStream out = new PrintStream(bs)) { System.setOut(out); System.setErr(out); ret = tool.run(cmds); System.out.flush(); System.err.flush(); } output = bs.toString(); } finally { System.setOut(origOut); System.setErr(origErr); } System.out.println("Output for command: " + cmd + " retcode: " + ret); if (output != null) { System.out.println(output); } assertEquals(retcode, ret); if (contain != null) { assertTrue("The real output is: " + output + ".\n It should contain: " + contain, output.contains(contain)); } } public static void FsShellRun(String cmd, int retcode, String contain, Configuration conf) throws Exception { FsShell shell = new FsShell(new Configuration(conf)); toolRun(shell, cmd, retcode, contain); } public static void DFSAdminRun(String cmd, int retcode, String contain, Configuration conf) throws Exception { DFSAdmin admin = new DFSAdmin(new Configuration(conf)); toolRun(admin, cmd, retcode, contain); } public static void FsShellRun(String cmd, Configuration conf) throws Exception { FsShellRun(cmd, 0, null, conf); } public static void addDataNodeLayoutVersion(final int lv, final String description) throws NoSuchFieldException, IllegalAccessException { Preconditions.checkState(lv < DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION); // Override {@link DataNodeLayoutVersion#CURRENT_LAYOUT_VERSION} via reflection. Field modifiersField = Field.class.getDeclaredField("modifiers"); modifiersField.setAccessible(true); Field field = DataNodeLayoutVersion.class.getField("CURRENT_LAYOUT_VERSION"); field.setAccessible(true); modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); field.setInt(null, lv); field = HdfsServerConstants.class.getField("DATANODE_LAYOUT_VERSION"); field.setAccessible(true); modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); field.setInt(null, lv); // Inject the feature into the FEATURES map. final LayoutVersion.FeatureInfo featureInfo = new LayoutVersion.FeatureInfo(lv, lv + 1, description, false); final LayoutVersion.LayoutFeature feature = new LayoutVersion.LayoutFeature() { @Override public LayoutVersion.FeatureInfo getInfo() { return featureInfo; } }; // Update the FEATURES map with the new layout version. LayoutVersion.updateMap(DataNodeLayoutVersion.FEATURES, new LayoutVersion.LayoutFeature[] { feature }); } /** * Wait for datanode to reach alive or dead state for waitTime given in * milliseconds. */ public static void waitForDatanodeState(final MiniDFSCluster cluster, final String nodeID, final boolean alive, int waitTime) throws TimeoutException, InterruptedException { GenericTestUtils.waitFor(new Supplier<Boolean>() { @Override public Boolean get() { FSNamesystem namesystem = cluster.getNamesystem(); final DatanodeDescriptor dd = BlockManagerTestUtil.getDatanode(namesystem, nodeID); return (dd.isAlive() == alive); } }, 100, waitTime); } /** * Change the length of a block at datanode dnIndex. */ public static boolean changeReplicaLength(MiniDFSCluster cluster, ExtendedBlock blk, int dnIndex, int lenDelta) throws IOException { File blockFile = cluster.getBlockFile(dnIndex, blk); if (blockFile != null && blockFile.exists()) { try (RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw")) { raFile.setLength(raFile.length() + lenDelta); } return true; } LOG.info("failed to change length of block " + blk); return false; } public static void setNameNodeLogLevel(Level level) { GenericTestUtils.setLogLevel(FSNamesystem.LOG, level); GenericTestUtils.setLogLevel(BlockManager.LOG, level); GenericTestUtils.setLogLevel(LeaseManager.LOG, level); GenericTestUtils.setLogLevel(NameNode.LOG, level); GenericTestUtils.setLogLevel(NameNode.stateChangeLog, level); GenericTestUtils.setLogLevel(NameNode.blockStateChangeLog, level); } /** * Get the NamenodeProtocol RPC proxy for the NN associated with this * DFSClient object * * @param nameNodeUri the URI of the NN to get a proxy for. * * @return the Namenode RPC proxy associated with this DFSClient object */ @VisibleForTesting public static NamenodeProtocol getNamenodeProtocolProxy(Configuration conf, URI nameNodeUri, UserGroupInformation ugi) throws IOException { return NameNodeProxies .createNonHAProxy(conf, DFSUtilClient.getNNAddress(nameNodeUri), NamenodeProtocol.class, ugi, false) .getProxy(); } /** * Get the RefreshUserMappingsProtocol RPC proxy for the NN associated with * this DFSClient object * * @param nameNodeUri the URI of the NN to get a proxy for. * * @return the RefreshUserMappingsProtocol RPC proxy associated with this * DFSClient object */ @VisibleForTesting public static RefreshUserMappingsProtocol getRefreshUserMappingsProtocolProxy(Configuration conf, URI nameNodeUri) throws IOException { final AtomicBoolean nnFallbackToSimpleAuth = new AtomicBoolean(false); return NameNodeProxies .createProxy(conf, nameNodeUri, RefreshUserMappingsProtocol.class, nnFallbackToSimpleAuth) .getProxy(); } /** * Set the datanode dead */ public static void setDatanodeDead(DatanodeInfo dn) { dn.setLastUpdate(0); // Set this to a large negative value. // On short-lived VMs, the monotonic time can be less than the heartbeat // expiry time. Setting this to 0 will fail to immediately mark the DN as // dead. dn.setLastUpdateMonotonic(Long.MIN_VALUE / 2); } /** * Update lastUpdate and lastUpdateMonotonic with some offset. */ public static void resetLastUpdatesWithOffset(DatanodeInfo dn, long offset) { dn.setLastUpdate(Time.now() + offset); dn.setLastUpdateMonotonic(Time.monotonicNow() + offset); } /** * This method takes a set of block locations and fills the provided buffer * with expected bytes based on simulated content from * {@link SimulatedFSDataset}. * * @param lbs The block locations of a file * @param expected The buffer to be filled with expected bytes on the above * locations. */ public static void fillExpectedBuf(LocatedBlocks lbs, byte[] expected) { Block[] blks = new Block[lbs.getLocatedBlocks().size()]; for (int i = 0; i < lbs.getLocatedBlocks().size(); i++) { blks[i] = lbs.getLocatedBlocks().get(i).getBlock().getLocalBlock(); } int bufPos = 0; for (Block b : blks) { for (long blkPos = 0; blkPos < b.getNumBytes(); blkPos++) { assert bufPos < expected.length; expected[bufPos++] = SimulatedFSDataset.simulatedByte(b, blkPos); } } } public static StorageReceivedDeletedBlocks[] makeReportForReceivedBlock(Block block, BlockStatus blockStatus, DatanodeStorage storage) { ReceivedDeletedBlockInfo[] receivedBlocks = new ReceivedDeletedBlockInfo[1]; receivedBlocks[0] = new ReceivedDeletedBlockInfo(block, blockStatus, null); StorageReceivedDeletedBlocks[] reports = new StorageReceivedDeletedBlocks[1]; reports[0] = new StorageReceivedDeletedBlocks(storage, receivedBlocks); return reports; } /** * Creates the metadata of a file in striped layout. This method only * manipulates the NameNode state without injecting data to DataNode. * You should disable periodical heartbeat before use this. * @param file Path of the file to create * @param dir Parent path of the file * @param numBlocks Number of striped block groups to add to the file * @param numStripesPerBlk Number of striped cells in each block * @param toMkdir */ public static void createStripedFile(MiniDFSCluster cluster, Path file, Path dir, int numBlocks, int numStripesPerBlk, boolean toMkdir) throws Exception { createStripedFile(cluster, file, dir, numBlocks, numStripesPerBlk, toMkdir, StripedFileTestUtil.getDefaultECPolicy()); } /** * Creates the metadata of a file in striped layout. This method only * manipulates the NameNode state without injecting data to DataNode. * You should disable periodical heartbeat before use this. * @param file Path of the file to create * @param dir Parent path of the file * @param numBlocks Number of striped block groups to add to the file * @param numStripesPerBlk Number of striped cells in each block * @param toMkdir * @param ecPolicy erasure coding policy apply to created file. A null value * means using default erasure coding policy. */ public static void createStripedFile(MiniDFSCluster cluster, Path file, Path dir, int numBlocks, int numStripesPerBlk, boolean toMkdir, ErasureCodingPolicy ecPolicy) throws Exception { DistributedFileSystem dfs = cluster.getFileSystem(); // If outer test already set EC policy, dir should be left as null if (toMkdir) { assert dir != null; dfs.mkdirs(dir); try { dfs.getClient().setErasureCodingPolicy(dir.toString(), ecPolicy.getName()); } catch (IOException e) { if (!e.getMessage().contains("non-empty directory")) { throw e; } } } cluster.getNameNodeRpc().create(file.toString(), new FsPermission((short) 0755), dfs.getClient().getClientName(), new EnumSetWritable<>(EnumSet.of(CreateFlag.CREATE)), false, (short) 1, 128 * 1024 * 1024L, null, null); FSNamesystem ns = cluster.getNamesystem(); FSDirectory fsdir = ns.getFSDirectory(); INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile(); ExtendedBlock previous = null; for (int i = 0; i < numBlocks; i++) { Block newBlock = addBlockToFile(true, cluster.getDataNodes(), dfs, ns, file.toString(), fileNode, dfs.getClient().getClientName(), previous, numStripesPerBlk, 0); previous = new ExtendedBlock(ns.getBlockPoolId(), newBlock); } dfs.getClient().namenode.complete(file.toString(), dfs.getClient().getClientName(), previous, fileNode.getId()); } /** * Adds a block or a striped block group to a file. * This method only manipulates NameNode * states of the file and the block without injecting data to DataNode. * It does mimic block reports. * You should disable periodical heartbeat before use this. * @param isStripedBlock a boolean tell if the block added a striped block * @param dataNodes List DataNodes to host the striped block group * @param previous Previous block in the file * @param numStripes Number of stripes in each block group * @param len block size for a non striped block added * @return The added block or block group */ public static Block addBlockToFile(boolean isStripedBlock, List<DataNode> dataNodes, DistributedFileSystem fs, FSNamesystem ns, String file, INodeFile fileNode, String clientName, ExtendedBlock previous, int numStripes, int len) throws Exception { fs.getClient().namenode.addBlock(file, clientName, previous, null, fileNode.getId(), null, null); final BlockInfo lastBlock = fileNode.getLastBlock(); final int groupSize = fileNode.getPreferredBlockReplication(); assert dataNodes.size() >= groupSize; // 1. RECEIVING_BLOCK IBR for (int i = 0; i < groupSize; i++) { DataNode dn = dataNodes.get(i); final Block block = new Block(lastBlock.getBlockId() + i, 0, lastBlock.getGenerationStamp()); DatanodeStorage storage = new DatanodeStorage(UUID.randomUUID().toString()); StorageReceivedDeletedBlocks[] reports = DFSTestUtil.makeReportForReceivedBlock(block, ReceivedDeletedBlockInfo.BlockStatus.RECEIVING_BLOCK, storage); for (StorageReceivedDeletedBlocks report : reports) { ns.processIncrementalBlockReport(dn.getDatanodeId(), report); } } final ErasureCodingPolicy ecPolicy = fs.getErasureCodingPolicy(new Path(file)); // 2. RECEIVED_BLOCK IBR long blockSize = isStripedBlock ? numStripes * ecPolicy.getCellSize() : len; for (int i = 0; i < groupSize; i++) { DataNode dn = dataNodes.get(i); final Block block = new Block(lastBlock.getBlockId() + i, blockSize, lastBlock.getGenerationStamp()); DatanodeStorage storage = new DatanodeStorage(UUID.randomUUID().toString()); StorageReceivedDeletedBlocks[] reports = DFSTestUtil.makeReportForReceivedBlock(block, ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage); for (StorageReceivedDeletedBlocks report : reports) { ns.processIncrementalBlockReport(dn.getDatanodeId(), report); } } long bytes = isStripedBlock ? numStripes * ecPolicy.getCellSize() * ecPolicy.getNumDataUnits() : len; lastBlock.setNumBytes(bytes); return lastBlock; } /* * Copy a block from sourceProxy to destination. If the block becomes * over-replicated, preferably remove it from source. * Return true if a block is successfully copied; otherwise false. */ public static boolean replaceBlock(ExtendedBlock block, DatanodeInfo source, DatanodeInfo sourceProxy, DatanodeInfo destination) throws IOException { return replaceBlock(block, source, sourceProxy, destination, StorageType.DEFAULT, Status.SUCCESS); } /* * Replace block */ public static boolean replaceBlock(ExtendedBlock block, DatanodeInfo source, DatanodeInfo sourceProxy, DatanodeInfo destination, StorageType targetStorageType, Status opStatus) throws IOException, SocketException { Socket sock = new Socket(); try { sock.connect(NetUtils.createSocketAddr(destination.getXferAddr()), HdfsConstants.READ_TIMEOUT); sock.setKeepAlive(true); // sendRequest DataOutputStream out = new DataOutputStream(sock.getOutputStream()); new Sender(out).replaceBlock(block, targetStorageType, BlockTokenSecretManager.DUMMY_TOKEN, source.getDatanodeUuid(), sourceProxy, null); out.flush(); // receiveResponse DataInputStream reply = new DataInputStream(sock.getInputStream()); BlockOpResponseProto proto = BlockOpResponseProto.parseDelimitedFrom(reply); while (proto.getStatus() == Status.IN_PROGRESS) { proto = BlockOpResponseProto.parseDelimitedFrom(reply); } return proto.getStatus() == opStatus; } finally { sock.close(); } } /** * Because currently DFSStripedOutputStream does not support hflush/hsync, * tests can use this method to flush all the buffered data to DataNodes. */ public static ExtendedBlock flushInternal(DFSStripedOutputStream out) throws IOException { out.flushAllInternals(); return out.getBlock(); } public static ExtendedBlock flushBuffer(DFSStripedOutputStream out) throws IOException { out.flush(); return out.getBlock(); } public static void waitForMetric(final JMXGet jmx, final String metricName, final int expectedValue) throws TimeoutException, InterruptedException { GenericTestUtils.waitFor(new Supplier<Boolean>() { @Override public Boolean get() { try { final int currentValue = Integer.parseInt(jmx.getValue(metricName)); LOG.info("Waiting for " + metricName + " to reach value " + expectedValue + ", current value = " + currentValue); return currentValue == expectedValue; } catch (Exception e) { throw new RuntimeException("Test failed due to unexpected exception", e); } } }, 1000, 60000); } /** * Close current file system and create a new instance as given * {@link UserGroupInformation}. */ public static FileSystem login(final FileSystem fs, final Configuration conf, final UserGroupInformation ugi) throws IOException, InterruptedException { if (fs != null) { fs.close(); } return DFSTestUtil.getFileSystemAs(ugi, conf); } /** * Test if the given {@link FileStatus} user, group owner and its permission * are expected, throw {@link AssertionError} if any value is not expected. */ public static void verifyFilePermission(FileStatus stat, String owner, String group, FsAction u, FsAction g, FsAction o) { if (stat != null) { if (!Strings.isNullOrEmpty(owner)) { assertEquals(owner, stat.getOwner()); } if (!Strings.isNullOrEmpty(group)) { assertEquals(group, stat.getGroup()); } FsPermission permission = stat.getPermission(); if (u != null) { assertEquals(u, permission.getUserAction()); } if (g != null) { assertEquals(g, permission.getGroupAction()); } if (o != null) { assertEquals(o, permission.getOtherAction()); } } } public static void verifyDelete(FsShell shell, FileSystem fs, Path path, boolean shouldExistInTrash) throws Exception { Path trashPath = Path.mergePaths(shell.getCurrentTrashDir(path), path); verifyDelete(shell, fs, path, trashPath, shouldExistInTrash); } public static void verifyDelete(FsShell shell, FileSystem fs, Path path, Path trashPath, boolean shouldExistInTrash) throws Exception { assertTrue(path + " file does not exist", fs.exists(path)); // Verify that trashPath has a path component named ".Trash" Path checkTrash = trashPath; while (!checkTrash.isRoot() && !checkTrash.getName().equals(".Trash")) { checkTrash = checkTrash.getParent(); } assertEquals("No .Trash component found in trash path " + trashPath, ".Trash", checkTrash.getName()); String[] argv = new String[] { "-rm", "-r", path.toString() }; int res = ToolRunner.run(shell, argv); assertEquals("rm failed", 0, res); if (shouldExistInTrash) { assertTrue("File not in trash : " + trashPath, fs.exists(trashPath)); } else { assertFalse("File in trash : " + trashPath, fs.exists(trashPath)); } } public static Map<Path, FSDataOutputStream> createOpenFiles(FileSystem fs, String filePrefix, int numFilesToCreate) throws IOException { final Map<Path, FSDataOutputStream> filesCreated = new HashMap<>(); final byte[] buffer = new byte[(int) (1024 * 1.75)]; final Random rand = new Random(0xFEED0BACL); for (int i = 0; i < numFilesToCreate; i++) { Path file = new Path("/" + filePrefix + "-" + i); FSDataOutputStream stm = fs.create(file, true, 1024, (short) 1, 1024); rand.nextBytes(buffer); stm.write(buffer); filesCreated.put(file, stm); } return filesCreated; } public static HashSet<Path> closeOpenFiles(HashMap<Path, FSDataOutputStream> openFilesMap, int numFilesToClose) throws IOException { HashSet<Path> closedFiles = new HashSet<>(); for (Iterator<Entry<Path, FSDataOutputStream>> it = openFilesMap.entrySet().iterator(); it.hasNext();) { Entry<Path, FSDataOutputStream> entry = it.next(); LOG.info("Closing file: " + entry.getKey()); entry.getValue().close(); closedFiles.add(entry.getKey()); it.remove(); numFilesToClose--; if (numFilesToClose == 0) { break; } } return closedFiles; } /** * Check the correctness of the snapshotDiff report. * Make sure all items in the passed entries are in the snapshotDiff * report. */ public static void verifySnapshotDiffReport(DistributedFileSystem fs, Path dir, String from, String to, DiffReportEntry... entries) throws IOException { SnapshotDiffReport report = fs.getSnapshotDiffReport(dir, from, to); // reverse the order of from and to SnapshotDiffReport inverseReport = fs.getSnapshotDiffReport(dir, to, from); LOG.info(report.toString()); LOG.info(inverseReport.toString() + "\n"); assertEquals(entries.length, report.getDiffList().size()); assertEquals(entries.length, inverseReport.getDiffList().size()); for (DiffReportEntry entry : entries) { if (entry.getType() == DiffType.MODIFY) { assertTrue(report.getDiffList().contains(entry)); assertTrue(inverseReport.getDiffList().contains(entry)); } else if (entry.getType() == DiffType.DELETE) { assertTrue(report.getDiffList().contains(entry)); assertTrue(inverseReport.getDiffList() .contains(new DiffReportEntry(DiffType.CREATE, entry.getSourcePath()))); } else if (entry.getType() == DiffType.CREATE) { assertTrue(report.getDiffList().contains(entry)); assertTrue(inverseReport.getDiffList() .contains(new DiffReportEntry(DiffType.DELETE, entry.getSourcePath()))); } } } /** * Check whether the Block movement has been successfully * completed to satisfy the storage policy for the given file. * @param fileName file name. * @param expectedStorageType storage type. * @param expectedStorageCount expected storage type. * @param timeout timeout. * @param fs distributedFileSystem. * @throws Exception */ public static void waitExpectedStorageType(String fileName, final StorageType expectedStorageType, int expectedStorageCount, int timeout, DistributedFileSystem fs) throws Exception { GenericTestUtils.waitFor(new Supplier<Boolean>() { @Override public Boolean get() { final LocatedBlock lb; try { lb = fs.getClient().getLocatedBlocks(fileName, 0).get(0); } catch (IOException e) { LOG.error("Exception while getting located blocks", e); return false; } int actualStorageCount = 0; for (StorageType type : lb.getStorageTypes()) { if (expectedStorageType == type) { actualStorageCount++; } } LOG.info(expectedStorageType + " replica count, expected=" + expectedStorageCount + " and actual=" + actualStorageCount); return expectedStorageCount == actualStorageCount; } }, 500, timeout); } /** * Waits for removal of a specified Xattr on a specified file. * * @param srcPath * file name. * @param xattr * name of the extended attribute. * @param ns * Namesystem * @param timeout * max wait time * @throws Exception */ public static void waitForXattrRemoved(String srcPath, String xattr, Namesystem ns, int timeout) throws TimeoutException, InterruptedException, UnresolvedLinkException, AccessControlException, ParentNotDirectoryException { final INode inode = ns.getFSDirectory().getINode(srcPath); final XAttr satisfyXAttr = XAttrHelper.buildXAttr(xattr); GenericTestUtils.waitFor(new Supplier<Boolean>() { @Override public Boolean get() { List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode); return !existingXAttrs.contains(satisfyXAttr); } }, 100, timeout); } /** * Get namenode connector using the given configuration and file path. * * @param conf * hdfs configuration * @param filePath * file path * @param namenodeCount * number of namenodes * @param createMoverPath * create move path flag to skip the path creation * @return Namenode connector. * @throws IOException */ public static NameNodeConnector getNameNodeConnector(Configuration conf, Path filePath, int namenodeCount, boolean createMoverPath) throws IOException { final Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf); Assert.assertEquals(namenodeCount, namenodes.size()); NameNodeConnector.checkOtherInstanceRunning(createMoverPath); while (true) { try { final List<NameNodeConnector> nncs = NameNodeConnector.newNameNodeConnectors(namenodes, StoragePolicySatisfier.class.getSimpleName(), filePath, conf, NameNodeConnector.DEFAULT_MAX_IDLE_ITERATIONS); return nncs.get(0); } catch (IOException e) { LOG.warn("Failed to connect with namenode", e); // Ignore } } } }