org.apache.hadoop.hdfs.server.datanode.TestFsDatasetCache.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hdfs.server.datanode.TestFsDatasetCache.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hdfs.server.datanode;

import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Mockito.doReturn;

import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.HashSet;
import java.util.Set;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.HdfsBlockLocation;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.LogVerificationAppender;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetCache;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetCache.PageRounder;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.BlockIdCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIO.POSIX.CacheManipulator;
import org.apache.hadoop.io.nativeio.NativeIO.POSIX.NoMlockCacheManipulator;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.MetricsAsserts;
import org.apache.log4j.Logger;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;

import com.google.common.base.Supplier;
import com.google.common.primitives.Ints;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hdfs.BlockReaderTestUtil;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
import org.apache.hadoop.io.IOUtils;

public class TestFsDatasetCache {
    private static final Log LOG = LogFactory.getLog(TestFsDatasetCache.class);

    // Most Linux installs allow a default of 64KB locked memory
    static final long CACHE_CAPACITY = 64 * 1024;
    // mlock always locks the entire page. So we don't need to deal with this
    // rounding, use the OS page size for the block size.
    private static final long PAGE_SIZE = NativeIO.POSIX.getCacheManipulator().getOperatingSystemPageSize();
    private static final long BLOCK_SIZE = PAGE_SIZE;

    private static Configuration conf;
    private static MiniDFSCluster cluster = null;
    private static FileSystem fs;
    private static NameNode nn;
    private static DataNode dn;
    private static FsDatasetSpi<?> fsd;
    private static DatanodeProtocolClientSideTranslatorPB spyNN;
    private static PageRounder rounder = new PageRounder();
    private static CacheManipulator prevCacheManipulator;

    static {
        LogManager.getLogger(FsDatasetCache.class).setLevel(Level.DEBUG);
    }

    @Before
    public void setUp() throws Exception {
        conf = new HdfsConfiguration();
        conf.setLong(DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS, 100);
        conf.setLong(DFSConfigKeys.DFS_CACHEREPORT_INTERVAL_MSEC_KEY, 500);
        conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
        conf.setLong(DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY, CACHE_CAPACITY);
        conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);

        prevCacheManipulator = NativeIO.POSIX.getCacheManipulator();
        NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator());

        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
        cluster.waitActive();

        fs = cluster.getFileSystem();
        nn = cluster.getNameNode();
        dn = cluster.getDataNodes().get(0);
        fsd = dn.getFSDataset();

        spyNN = DataNodeTestUtils.spyOnBposToNN(dn, nn);

    }

    @After
    public void tearDown() throws Exception {
        // Verify that each test uncached whatever it cached.  This cleanup is
        // required so that file descriptors are not leaked across tests.
        DFSTestUtil.verifyExpectedCacheUsage(0, 0, fsd);
        if (fs != null) {
            fs.close();
        }
        if (cluster != null) {
            cluster.shutdown();
        }
        // Restore the original CacheManipulator
        NativeIO.POSIX.setCacheManipulator(prevCacheManipulator);
    }

    private static void setHeartbeatResponse(DatanodeCommand[] cmds) throws IOException {
        HeartbeatResponse response = new HeartbeatResponse(cmds, null);
        doReturn(response).when(spyNN).sendHeartbeat((DatanodeRegistration) any(), (StorageReport[]) any(),
                anyLong(), anyLong(), anyInt(), anyInt(), anyInt(), (VolumeFailureSummary) any());
    }

    private static DatanodeCommand[] cacheBlock(HdfsBlockLocation loc) {
        return cacheBlocks(new HdfsBlockLocation[] { loc });
    }

    private static DatanodeCommand[] cacheBlocks(HdfsBlockLocation[] locs) {
        return new DatanodeCommand[] { getResponse(locs, DatanodeProtocol.DNA_CACHE) };
    }

    private static DatanodeCommand[] uncacheBlock(HdfsBlockLocation loc) {
        return uncacheBlocks(new HdfsBlockLocation[] { loc });
    }

    private static DatanodeCommand[] uncacheBlocks(HdfsBlockLocation[] locs) {
        return new DatanodeCommand[] { getResponse(locs, DatanodeProtocol.DNA_UNCACHE) };
    }

    /**
     * Creates a cache or uncache DatanodeCommand from an array of locations
     */
    private static DatanodeCommand getResponse(HdfsBlockLocation[] locs, int action) {
        String bpid = locs[0].getLocatedBlock().getBlock().getBlockPoolId();
        long[] blocks = new long[locs.length];
        for (int i = 0; i < locs.length; i++) {
            blocks[i] = locs[i].getLocatedBlock().getBlock().getBlockId();
        }
        return new BlockIdCommand(action, bpid, blocks);
    }

    private static long[] getBlockSizes(HdfsBlockLocation[] locs) throws Exception {
        long[] sizes = new long[locs.length];
        for (int i = 0; i < locs.length; i++) {
            HdfsBlockLocation loc = locs[i];
            String bpid = loc.getLocatedBlock().getBlock().getBlockPoolId();
            Block block = loc.getLocatedBlock().getBlock().getLocalBlock();
            ExtendedBlock extBlock = new ExtendedBlock(bpid, block);
            FileInputStream blockInputStream = null;
            FileChannel blockChannel = null;
            try {
                blockInputStream = (FileInputStream) fsd.getBlockInputStream(extBlock, 0);
                blockChannel = blockInputStream.getChannel();
                sizes[i] = blockChannel.size();
            } finally {
                IOUtils.cleanup(LOG, blockChannel, blockInputStream);
            }
        }
        return sizes;
    }

    private void testCacheAndUncacheBlock() throws Exception {
        LOG.info("beginning testCacheAndUncacheBlock");
        final int NUM_BLOCKS = 5;

        DFSTestUtil.verifyExpectedCacheUsage(0, 0, fsd);
        assertEquals(0, fsd.getNumBlocksCached());

        // Write a test file
        final Path testFile = new Path("/testCacheBlock");
        final long testFileLen = BLOCK_SIZE * NUM_BLOCKS;
        DFSTestUtil.createFile(fs, testFile, testFileLen, (short) 1, 0xABBAl);

        // Get the details of the written file
        HdfsBlockLocation[] locs = (HdfsBlockLocation[]) fs.getFileBlockLocations(testFile, 0, testFileLen);
        assertEquals("Unexpected number of blocks", NUM_BLOCKS, locs.length);
        final long[] blockSizes = getBlockSizes(locs);

        // Check initial state
        final long cacheCapacity = fsd.getCacheCapacity();
        long cacheUsed = fsd.getCacheUsed();
        long current = 0;
        assertEquals("Unexpected cache capacity", CACHE_CAPACITY, cacheCapacity);
        assertEquals("Unexpected amount of cache used", current, cacheUsed);

        MetricsRecordBuilder dnMetrics;
        long numCacheCommands = 0;
        long numUncacheCommands = 0;

        // Cache each block in succession, checking each time
        for (int i = 0; i < NUM_BLOCKS; i++) {
            setHeartbeatResponse(cacheBlock(locs[i]));
            current = DFSTestUtil.verifyExpectedCacheUsage(current + blockSizes[i], i + 1, fsd);
            dnMetrics = getMetrics(dn.getMetrics().name());
            long cmds = MetricsAsserts.getLongCounter("BlocksCached", dnMetrics);
            assertTrue("Expected more cache requests from the NN (" + cmds + " <= " + numCacheCommands + ")",
                    cmds > numCacheCommands);
            numCacheCommands = cmds;
        }

        // Uncache each block in succession, again checking each time
        for (int i = 0; i < NUM_BLOCKS; i++) {
            setHeartbeatResponse(uncacheBlock(locs[i]));
            current = DFSTestUtil.verifyExpectedCacheUsage(current - blockSizes[i], NUM_BLOCKS - 1 - i, fsd);
            dnMetrics = getMetrics(dn.getMetrics().name());
            long cmds = MetricsAsserts.getLongCounter("BlocksUncached", dnMetrics);
            assertTrue("Expected more uncache requests from the NN", cmds > numUncacheCommands);
            numUncacheCommands = cmds;
        }
        LOG.info("finishing testCacheAndUncacheBlock");
    }

    @Test(timeout = 600000)
    public void testCacheAndUncacheBlockSimple() throws Exception {
        testCacheAndUncacheBlock();
    }

    /**
     * Run testCacheAndUncacheBlock with some failures injected into the mlock
     * call.  This tests the ability of the NameNode to resend commands.
     */
    @Test(timeout = 600000)
    public void testCacheAndUncacheBlockWithRetries() throws Exception {
        // We don't have to save the previous cacheManipulator
        // because it will be reinstalled by the @After function.
        NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator() {
            private final Set<String> seenIdentifiers = new HashSet<String>();

            @Override
            public void mlock(String identifier, ByteBuffer mmap, long length) throws IOException {
                if (seenIdentifiers.contains(identifier)) {
                    // mlock succeeds the second time.
                    LOG.info("mlocking " + identifier);
                    return;
                }
                seenIdentifiers.add(identifier);
                throw new IOException("injecting IOException during mlock of " + identifier);
            }
        });
        testCacheAndUncacheBlock();
    }

    @Test(timeout = 600000)
    public void testFilesExceedMaxLockedMemory() throws Exception {
        LOG.info("beginning testFilesExceedMaxLockedMemory");

        // Create some test files that will exceed total cache capacity
        final int numFiles = 5;
        final long fileSize = CACHE_CAPACITY / (numFiles - 1);

        final Path[] testFiles = new Path[numFiles];
        final HdfsBlockLocation[][] fileLocs = new HdfsBlockLocation[numFiles][];
        final long[] fileSizes = new long[numFiles];
        for (int i = 0; i < numFiles; i++) {
            testFiles[i] = new Path("/testFilesExceedMaxLockedMemory-" + i);
            DFSTestUtil.createFile(fs, testFiles[i], fileSize, (short) 1, 0xDFAl);
            fileLocs[i] = (HdfsBlockLocation[]) fs.getFileBlockLocations(testFiles[i], 0, fileSize);
            // Get the file size (sum of blocks)
            long[] sizes = getBlockSizes(fileLocs[i]);
            for (int j = 0; j < sizes.length; j++) {
                fileSizes[i] += sizes[j];
            }
        }

        // Cache the first n-1 files
        long total = 0;
        DFSTestUtil.verifyExpectedCacheUsage(0, 0, fsd);
        for (int i = 0; i < numFiles - 1; i++) {
            setHeartbeatResponse(cacheBlocks(fileLocs[i]));
            total = DFSTestUtil.verifyExpectedCacheUsage(rounder.round(total + fileSizes[i]), 4 * (i + 1), fsd);
        }

        // nth file should hit a capacity exception
        final LogVerificationAppender appender = new LogVerificationAppender();
        final Logger logger = Logger.getRootLogger();
        logger.addAppender(appender);
        setHeartbeatResponse(cacheBlocks(fileLocs[numFiles - 1]));

        GenericTestUtils.waitFor(new Supplier<Boolean>() {
            @Override
            public Boolean get() {
                int lines = appender.countLinesWithMessage(
                        "more bytes in the cache: " + DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY);
                return lines > 0;
            }
        }, 500, 30000);
        // Also check the metrics for the failure
        assertTrue("Expected more than 0 failed cache attempts", fsd.getNumBlocksFailedToCache() > 0);

        // Uncache the n-1 files
        int curCachedBlocks = 16;
        for (int i = 0; i < numFiles - 1; i++) {
            setHeartbeatResponse(uncacheBlocks(fileLocs[i]));
            long uncachedBytes = rounder.round(fileSizes[i]);
            total -= uncachedBytes;
            curCachedBlocks -= uncachedBytes / BLOCK_SIZE;
            DFSTestUtil.verifyExpectedCacheUsage(total, curCachedBlocks, fsd);
        }
        LOG.info("finishing testFilesExceedMaxLockedMemory");
    }

    @Test(timeout = 600000)
    public void testUncachingBlocksBeforeCachingFinishes() throws Exception {
        LOG.info("beginning testUncachingBlocksBeforeCachingFinishes");
        final int NUM_BLOCKS = 5;

        DFSTestUtil.verifyExpectedCacheUsage(0, 0, fsd);

        // Write a test file
        final Path testFile = new Path("/testCacheBlock");
        final long testFileLen = BLOCK_SIZE * NUM_BLOCKS;
        DFSTestUtil.createFile(fs, testFile, testFileLen, (short) 1, 0xABBAl);

        // Get the details of the written file
        HdfsBlockLocation[] locs = (HdfsBlockLocation[]) fs.getFileBlockLocations(testFile, 0, testFileLen);
        assertEquals("Unexpected number of blocks", NUM_BLOCKS, locs.length);
        final long[] blockSizes = getBlockSizes(locs);

        // Check initial state
        final long cacheCapacity = fsd.getCacheCapacity();
        long cacheUsed = fsd.getCacheUsed();
        long current = 0;
        assertEquals("Unexpected cache capacity", CACHE_CAPACITY, cacheCapacity);
        assertEquals("Unexpected amount of cache used", current, cacheUsed);

        NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator() {
            @Override
            public void mlock(String identifier, ByteBuffer mmap, long length) throws IOException {
                LOG.info("An mlock operation is starting on " + identifier);
                try {
                    Thread.sleep(3000);
                } catch (InterruptedException e) {
                    Assert.fail();
                }
            }
        });
        // Starting caching each block in succession.  The usedBytes amount
        // should increase, even though caching doesn't complete on any of them.
        for (int i = 0; i < NUM_BLOCKS; i++) {
            setHeartbeatResponse(cacheBlock(locs[i]));
            current = DFSTestUtil.verifyExpectedCacheUsage(current + blockSizes[i], i + 1, fsd);
        }

        setHeartbeatResponse(new DatanodeCommand[] { getResponse(locs, DatanodeProtocol.DNA_UNCACHE) });

        // wait until all caching jobs are finished cancelling.
        current = DFSTestUtil.verifyExpectedCacheUsage(0, 0, fsd);
        LOG.info("finishing testUncachingBlocksBeforeCachingFinishes");
    }

    @Test(timeout = 60000)
    public void testUncacheUnknownBlock() throws Exception {
        // Create a file
        Path fileName = new Path("/testUncacheUnknownBlock");
        int fileLen = 4096;
        DFSTestUtil.createFile(fs, fileName, fileLen, (short) 1, 0xFDFD);
        HdfsBlockLocation[] locs = (HdfsBlockLocation[]) fs.getFileBlockLocations(fileName, 0, fileLen);

        // Try to uncache it without caching it first
        setHeartbeatResponse(uncacheBlocks(locs));

        GenericTestUtils.waitFor(new Supplier<Boolean>() {
            @Override
            public Boolean get() {
                return fsd.getNumBlocksFailedToUncache() > 0;
            }
        }, 100, 10000);
    }

    @Test(timeout = 60000)
    public void testPageRounder() throws Exception {
        // Write a small file
        Path fileName = new Path("/testPageRounder");
        final int smallBlocks = 512; // This should be smaller than the page size
        assertTrue("Page size should be greater than smallBlocks!", PAGE_SIZE > smallBlocks);
        final int numBlocks = 5;
        final int fileLen = smallBlocks * numBlocks;
        FSDataOutputStream out = fs.create(fileName, false, 4096, (short) 1, smallBlocks);
        out.write(new byte[fileLen]);
        out.close();
        HdfsBlockLocation[] locs = (HdfsBlockLocation[]) fs.getFileBlockLocations(fileName, 0, fileLen);
        // Cache the file and check the sizes match the page size
        setHeartbeatResponse(cacheBlocks(locs));
        DFSTestUtil.verifyExpectedCacheUsage(PAGE_SIZE * numBlocks, numBlocks, fsd);
        // Uncache and check that it decrements by the page size too
        setHeartbeatResponse(uncacheBlocks(locs));
        DFSTestUtil.verifyExpectedCacheUsage(0, 0, fsd);
    }

    @Test(timeout = 60000)
    public void testUncacheQuiesces() throws Exception {
        // Create a file
        Path fileName = new Path("/testUncacheQuiesces");
        int fileLen = 4096;
        DFSTestUtil.createFile(fs, fileName, fileLen, (short) 1, 0xFDFD);
        // Cache it
        DistributedFileSystem dfs = cluster.getFileSystem();
        dfs.addCachePool(new CachePoolInfo("pool"));
        long directiveId = dfs.addCacheDirective(new CacheDirectiveInfo.Builder().setPool("pool").setPath(fileName)
                .setReplication((short) 3).build());
        GenericTestUtils.waitFor(new Supplier<Boolean>() {
            @Override
            public Boolean get() {
                MetricsRecordBuilder dnMetrics = getMetrics(dn.getMetrics().name());
                long blocksCached = MetricsAsserts.getLongCounter("BlocksCached", dnMetrics);
                return blocksCached > 0;
            }
        }, 1000, 30000);
        // Uncache it
        dfs.removeCacheDirective(directiveId);
        GenericTestUtils.waitFor(new Supplier<Boolean>() {
            @Override
            public Boolean get() {
                MetricsRecordBuilder dnMetrics = getMetrics(dn.getMetrics().name());
                long blocksUncached = MetricsAsserts.getLongCounter("BlocksUncached", dnMetrics);
                return blocksUncached > 0;
            }
        }, 1000, 30000);
        // Make sure that no additional messages were sent
        Thread.sleep(10000);
        MetricsRecordBuilder dnMetrics = getMetrics(dn.getMetrics().name());
        MetricsAsserts.assertCounter("BlocksCached", 1l, dnMetrics);
        MetricsAsserts.assertCounter("BlocksUncached", 1l, dnMetrics);
    }

    @Test(timeout = 60000)
    public void testReCacheAfterUncache() throws Exception {
        final int TOTAL_BLOCKS_PER_CACHE = Ints.checkedCast(CACHE_CAPACITY / BLOCK_SIZE);
        BlockReaderTestUtil.enableHdfsCachingTracing();
        Assert.assertEquals(0, CACHE_CAPACITY % BLOCK_SIZE);

        // Create a small file
        final Path SMALL_FILE = new Path("/smallFile");
        DFSTestUtil.createFile(fs, SMALL_FILE, BLOCK_SIZE, (short) 1, 0xcafe);
        // Create a file that will take up the whole cache
        final Path BIG_FILE = new Path("/bigFile");
        DFSTestUtil.createFile(fs, BIG_FILE, TOTAL_BLOCKS_PER_CACHE * BLOCK_SIZE, (short) 1, 0xbeef);
        final DistributedFileSystem dfs = cluster.getFileSystem();
        dfs.addCachePool(new CachePoolInfo("pool"));
        final long bigCacheDirectiveId = dfs.addCacheDirective(new CacheDirectiveInfo.Builder().setPool("pool")
                .setPath(BIG_FILE).setReplication((short) 1).build());
        GenericTestUtils.waitFor(new Supplier<Boolean>() {
            @Override
            public Boolean get() {
                MetricsRecordBuilder dnMetrics = getMetrics(dn.getMetrics().name());
                long blocksCached = MetricsAsserts.getLongCounter("BlocksCached", dnMetrics);
                if (blocksCached != TOTAL_BLOCKS_PER_CACHE) {
                    LOG.info("waiting for " + TOTAL_BLOCKS_PER_CACHE + " to " + "be cached.   Right now only "
                            + blocksCached + " blocks are cached.");
                    return false;
                }
                LOG.info(TOTAL_BLOCKS_PER_CACHE + " blocks are now cached.");
                return true;
            }
        }, 1000, 30000);

        // Try to cache a smaller file.  It should fail.
        final long shortCacheDirectiveId = dfs.addCacheDirective(new CacheDirectiveInfo.Builder().setPool("pool")
                .setPath(SMALL_FILE).setReplication((short) 1).build());
        Thread.sleep(10000);
        MetricsRecordBuilder dnMetrics = getMetrics(dn.getMetrics().name());
        Assert.assertEquals(TOTAL_BLOCKS_PER_CACHE, MetricsAsserts.getLongCounter("BlocksCached", dnMetrics));

        // Uncache the big file and verify that the small file can now be
        // cached (regression test for HDFS-6107)
        dfs.removeCacheDirective(bigCacheDirectiveId);
        GenericTestUtils.waitFor(new Supplier<Boolean>() {
            @Override
            public Boolean get() {
                RemoteIterator<CacheDirectiveEntry> iter;
                try {
                    iter = dfs.listCacheDirectives(new CacheDirectiveInfo.Builder().build());
                    CacheDirectiveEntry entry;
                    do {
                        entry = iter.next();
                    } while (entry.getInfo().getId() != shortCacheDirectiveId);
                    if (entry.getStats().getFilesCached() != 1) {
                        LOG.info("waiting for directive " + shortCacheDirectiveId + " to be cached.  stats = "
                                + entry.getStats());
                        return false;
                    }
                    LOG.info("directive " + shortCacheDirectiveId + " has been cached.");
                } catch (IOException e) {
                    Assert.fail("unexpected exception" + e.toString());
                }
                return true;
            }
        }, 1000, 30000);

        dfs.removeCacheDirective(shortCacheDirectiveId);
    }
}