org.apache.accumulo.gc.GarbageCollectWriteAheadLogsTest.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.accumulo.gc.GarbageCollectWriteAheadLogsTest.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.accumulo.gc;

import com.google.common.net.HostAndPort;

import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;

import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.ReplicationSection;
import org.apache.accumulo.core.protobuf.ProtobufUtil;
import org.apache.accumulo.core.replication.ReplicationSchema.StatusSection;
import org.apache.accumulo.core.replication.ReplicationTable;
import org.apache.accumulo.server.AccumuloServerContext;
import org.apache.accumulo.server.conf.ServerConfigurationFactory;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.replication.StatusUtil;
import org.apache.accumulo.server.replication.proto.Replication.Status;
import org.apache.hadoop.io.Text;
import org.easymock.EasyMock;
import org.easymock.IAnswer;
import org.junit.Assert;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.gc.thrift.GCStatus;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;

import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;

import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;

import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.gc.thrift.GcCycleStats;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.zookeeper.KeeperException;

import java.io.File;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.Map.Entry;

import static org.easymock.EasyMock.createMock;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static java.lang.Thread.sleep;

import java.io.FileOutputStream;

import org.apache.commons.io.FileUtils;

import java.util.concurrent.TimeUnit;

public class GarbageCollectWriteAheadLogsTest {
    private static final long BLOCK_SIZE = 64000000L;

    private static final Path DIR_1_PATH = new Path("/dir1");
    private static final Path DIR_2_PATH = new Path("/dir2");
    private static final Path DIR_3_PATH = new Path("/dir3");
    private static final String UUID1 = UUID.randomUUID().toString();
    private static final String UUID2 = UUID.randomUUID().toString();
    private static final String UUID3 = UUID.randomUUID().toString();

    private Instance instance;
    private AccumuloConfiguration systemConfig;
    private VolumeManager volMgr;
    private GarbageCollectWriteAheadLogs gcwal;
    private long modTime;

    @Rule
    public TestName testName = new TestName();

    @Before
    public void setUp() throws Exception {
        SiteConfiguration siteConfig = EasyMock.createMock(SiteConfiguration.class);
        instance = createMock(Instance.class);
        expect(instance.getInstanceID()).andReturn("mock").anyTimes();
        expect(instance.getZooKeepers()).andReturn("localhost").anyTimes();
        expect(instance.getZooKeepersSessionTimeOut()).andReturn(30000).anyTimes();
        systemConfig = new ConfigurationCopy(new HashMap<String, String>());
        volMgr = createMock(VolumeManager.class);
        ServerConfigurationFactory factory = createMock(ServerConfigurationFactory.class);
        expect(factory.getConfiguration()).andReturn(systemConfig).anyTimes();
        expect(factory.getInstance()).andReturn(instance).anyTimes();
        expect(factory.getSiteConfiguration()).andReturn(siteConfig).anyTimes();

        // Just make the SiteConfiguration delegate to our AccumuloConfiguration
        // Presently, we only need get(Property) and iterator().
        EasyMock.expect(siteConfig.get(EasyMock.anyObject(Property.class))).andAnswer(new IAnswer<String>() {
            @Override
            public String answer() {
                Object[] args = EasyMock.getCurrentArguments();
                return systemConfig.get((Property) args[0]);
            }
        }).anyTimes();
        EasyMock.expect(siteConfig.getBoolean(EasyMock.anyObject(Property.class)))
                .andAnswer(new IAnswer<Boolean>() {
                    @Override
                    public Boolean answer() {
                        Object[] args = EasyMock.getCurrentArguments();
                        return systemConfig.getBoolean((Property) args[0]);
                    }
                }).anyTimes();

        EasyMock.expect(siteConfig.iterator()).andAnswer(new IAnswer<Iterator<Entry<String, String>>>() {
            @Override
            public Iterator<Entry<String, String>> answer() {
                return systemConfig.iterator();
            }
        }).anyTimes();

        replay(instance, factory, siteConfig);
        AccumuloServerContext context = new AccumuloServerContext(factory);
        gcwal = new GarbageCollectWriteAheadLogs(context, volMgr, false);
        modTime = System.currentTimeMillis();
    }

    @Test
    public void testGetters() {
        assertSame(instance, gcwal.getInstance());
        assertSame(volMgr, gcwal.getVolumeManager());
        assertFalse(gcwal.isUsingTrash());
    }

    @Test
    public void testPathsToStrings() {
        ArrayList<Path> paths = new ArrayList<Path>();
        paths.add(new Path(DIR_1_PATH, "file1"));
        paths.add(DIR_2_PATH);
        paths.add(new Path(DIR_3_PATH, "file3"));
        List<String> strings = GarbageCollectWriteAheadLogs.paths2strings(paths);
        int len = 3;
        assertEquals(len, strings.size());
        for (int i = 0; i < len; i++) {
            assertEquals(paths.get(i).toString(), strings.get(i));
        }
    }

    @Test
    public void testMapServersToFiles() {
        // @formatter:off
        /*
         * Test fileToServerMap:
         * /dir1/server1/uuid1 -> server1 (new-style)
         * /dir1/uuid2 -> "" (old-style)
         * /dir3/server3/uuid3 -> server3 (new-style)
         */
        // @formatter:on
        Map<Path, String> fileToServerMap = new java.util.HashMap<Path, String>();
        Path path1 = new Path(new Path(DIR_1_PATH, "server1"), UUID1);
        fileToServerMap.put(path1, "server1"); // new-style
        Path path2 = new Path(DIR_1_PATH, UUID2);
        fileToServerMap.put(path2, ""); // old-style
        Path path3 = new Path(new Path(DIR_3_PATH, "server3"), UUID3);
        fileToServerMap.put(path3, "server3"); // old-style
        // @formatter:off
        /*
         * Test nameToFileMap:
         * uuid1 -> /dir1/server1/uuid1
         * uuid3 -> /dir3/server3/uuid3
         */
        // @formatter:on
        Map<String, Path> nameToFileMap = new java.util.HashMap<String, Path>();
        nameToFileMap.put(UUID1, path1);
        nameToFileMap.put(UUID3, path3);

        // @formatter:off
        /*
         * Expected map:
         * server1 -> [ /dir1/server1/uuid1 ]
         * server3 -> [ /dir3/server3/uuid3 ]
         */
        // @formatter:on
        Map<String, ArrayList<Path>> result = GarbageCollectWriteAheadLogs.mapServersToFiles(fileToServerMap,
                nameToFileMap);
        assertEquals(2, result.size());
        ArrayList<Path> list1 = result.get("server1");
        assertEquals(1, list1.size());
        assertTrue(list1.contains(path1));
        ArrayList<Path> list3 = result.get("server3");
        assertEquals(1, list3.size());
        assertTrue(list3.contains(path3));
    }

    private FileStatus makeFileStatus(int size, Path path) {
        boolean isDir = (size == 0);
        return new FileStatus(size, isDir, 3, BLOCK_SIZE, modTime, path);
    }

    private void mockListStatus(Path dir, FileStatus... fileStatuses) throws Exception {
        expect(volMgr.listStatus(dir)).andReturn(fileStatuses);
    }

    @Test
    public void testScanServers_NewStyle() throws Exception {
        String[] walDirs = new String[] { "/dir1", "/dir2", "/dir3" };
        // @formatter:off
        /*
         * Test directory layout:
         * /dir1/
         *   server1/
         *     uuid1
         *     file2
         *   subdir2/
         * /dir2/ missing
         * /dir3/
         *   server3/
         *     uuid3
         */
        // @formatter:on
        Path serverDir1Path = new Path(DIR_1_PATH, "server1");
        FileStatus serverDir1 = makeFileStatus(0, serverDir1Path);
        Path subDir2Path = new Path(DIR_1_PATH, "subdir2");
        FileStatus serverDir2 = makeFileStatus(0, subDir2Path);
        mockListStatus(DIR_1_PATH, serverDir1, serverDir2);
        Path path1 = new Path(serverDir1Path, UUID1);
        FileStatus file1 = makeFileStatus(100, path1);
        FileStatus file2 = makeFileStatus(200, new Path(serverDir1Path, "file2"));
        mockListStatus(serverDir1Path, file1, file2);
        mockListStatus(subDir2Path);
        expect(volMgr.listStatus(DIR_2_PATH)).andThrow(new FileNotFoundException());
        Path serverDir3Path = new Path(DIR_3_PATH, "server3");
        FileStatus serverDir3 = makeFileStatus(0, serverDir3Path);
        mockListStatus(DIR_3_PATH, serverDir3);
        Path path3 = new Path(serverDir3Path, UUID3);
        FileStatus file3 = makeFileStatus(300, path3);
        mockListStatus(serverDir3Path, file3);
        replay(volMgr);

        Map<Path, String> fileToServerMap = new java.util.HashMap<Path, String>();
        Map<String, Path> nameToFileMap = new java.util.HashMap<String, Path>();
        int count = gcwal.scanServers(walDirs, fileToServerMap, nameToFileMap);
        assertEquals(3, count);
        // @formatter:off
        /*
         * Expected fileToServerMap:
         * /dir1/server1/uuid1 -> server1
         * /dir3/server3/uuid3 -> server3
         */
        // @formatter:on
        assertEquals(2, fileToServerMap.size());
        assertEquals("server1", fileToServerMap.get(path1));
        assertEquals("server3", fileToServerMap.get(path3));
        // @formatter:off
        /*
         * Expected nameToFileMap:
         * uuid1 -> /dir1/server1/uuid1
         * uuid3 -> /dir3/server3/uuid3
         */
        // @formatter:on
        assertEquals(2, nameToFileMap.size());
        assertEquals(path1, nameToFileMap.get(UUID1));
        assertEquals(path3, nameToFileMap.get(UUID3));
    }

    @Test
    public void testScanServers_OldStyle() throws Exception {
        // @formatter:off
        /*
         * Test directory layout:
         * /dir1/
         *   uuid1
         * /dir3/
         *   uuid3
         */
        // @formatter:on
        String[] walDirs = new String[] { "/dir1", "/dir3" };
        Path serverFile1Path = new Path(DIR_1_PATH, UUID1);
        FileStatus serverFile1 = makeFileStatus(100, serverFile1Path);
        mockListStatus(DIR_1_PATH, serverFile1);
        Path serverFile3Path = new Path(DIR_3_PATH, UUID3);
        FileStatus serverFile3 = makeFileStatus(300, serverFile3Path);
        mockListStatus(DIR_3_PATH, serverFile3);
        replay(volMgr);

        Map<Path, String> fileToServerMap = new java.util.HashMap<Path, String>();
        Map<String, Path> nameToFileMap = new java.util.HashMap<String, Path>();
        int count = gcwal.scanServers(walDirs, fileToServerMap, nameToFileMap);
        /*
         * Expect only a single server, the non-server entry for upgrade WALs
         */
        assertEquals(1, count);
        // @formatter:off
        /*
         * Expected fileToServerMap:
         * /dir1/uuid1 -> ""
         * /dir3/uuid3 -> ""
         */
        // @formatter:on
        assertEquals(2, fileToServerMap.size());
        assertEquals("", fileToServerMap.get(serverFile1Path));
        assertEquals("", fileToServerMap.get(serverFile3Path));
        // @formatter:off
        /*
         * Expected nameToFileMap:
         * uuid1 -> /dir1/uuid1
         * uuid3 -> /dir3/uuid3
         */
        // @formatter:on
        assertEquals(2, nameToFileMap.size());
        assertEquals(serverFile1Path, nameToFileMap.get(UUID1));
        assertEquals(serverFile3Path, nameToFileMap.get(UUID3));
    }

    @Test
    public void testGetSortedWALogs() throws Exception {
        String[] recoveryDirs = new String[] { "/dir1", "/dir2", "/dir3" };
        // @formatter:off
        /*
         * Test directory layout:
         * /dir1/
         *   uuid1
         *   file2
         * /dir2/ missing
         * /dir3/
         *   uuid3
         */
        // @formatter:on
        expect(volMgr.exists(DIR_1_PATH)).andReturn(true);
        expect(volMgr.exists(DIR_2_PATH)).andReturn(false);
        expect(volMgr.exists(DIR_3_PATH)).andReturn(true);
        Path path1 = new Path(DIR_1_PATH, UUID1);
        FileStatus file1 = makeFileStatus(100, path1);
        FileStatus file2 = makeFileStatus(200, new Path(DIR_1_PATH, "file2"));
        mockListStatus(DIR_1_PATH, file1, file2);
        Path path3 = new Path(DIR_3_PATH, UUID3);
        FileStatus file3 = makeFileStatus(300, path3);
        mockListStatus(DIR_3_PATH, file3);
        replay(volMgr);

        Map<String, Path> sortedWalogs = gcwal.getSortedWALogs(recoveryDirs);
        // @formatter:off
        /*
         * Expected map:
         * uuid1 -> /dir1/uuid1
         * uuid3 -> /dir3/uuid3
         */
        // @formatter:on
        assertEquals(2, sortedWalogs.size());
        assertEquals(path1, sortedWalogs.get(UUID1));
        assertEquals(path3, sortedWalogs.get(UUID3));
    }

    @Test
    public void testIsUUID() {
        assertTrue(GarbageCollectWriteAheadLogs.isUUID(UUID.randomUUID().toString()));
        assertFalse(GarbageCollectWriteAheadLogs.isUUID("foo"));
        assertFalse(GarbageCollectWriteAheadLogs.isUUID("0" + UUID.randomUUID().toString()));
        assertFalse(GarbageCollectWriteAheadLogs.isUUID(null));
    }

    // It was easier to do this than get the mocking working for me
    private static class ReplicationGCWAL extends GarbageCollectWriteAheadLogs {

        private List<Entry<Key, Value>> replData;

        ReplicationGCWAL(AccumuloServerContext context, VolumeManager fs, boolean useTrash,
                List<Entry<Key, Value>> replData) throws IOException {
            super(context, fs, useTrash);
            this.replData = replData;
        }

        @Override
        protected Iterable<Entry<Key, Value>> getReplicationStatusForFile(Connector conn, String wal) {
            return this.replData;
        }
    }

    @Test
    public void replicationEntriesAffectGC() throws Exception {
        String file1 = UUID.randomUUID().toString(), file2 = UUID.randomUUID().toString();
        Connector conn = createMock(Connector.class);

        // Write a Status record which should prevent file1 from being deleted
        LinkedList<Entry<Key, Value>> replData = new LinkedList<>();
        replData.add(Maps.immutableEntry(new Key("/wals/" + file1, StatusSection.NAME.toString(), "1"),
                StatusUtil.fileCreatedValue(System.currentTimeMillis())));

        ReplicationGCWAL replGC = new ReplicationGCWAL(null, volMgr, false, replData);

        replay(conn);

        // Open (not-closed) file must be retained
        assertTrue(replGC.neededByReplication(conn, "/wals/" + file1));

        // No replication data, not needed
        replData.clear();
        assertFalse(replGC.neededByReplication(conn, "/wals/" + file2));

        // The file is closed but not replicated, must be retained
        replData.add(Maps.immutableEntry(new Key("/wals/" + file1, StatusSection.NAME.toString(), "1"),
                StatusUtil.fileClosedValue()));
        assertTrue(replGC.neededByReplication(conn, "/wals/" + file1));

        // File is closed and fully replicated, can be deleted
        replData.clear();
        replData.add(Maps.immutableEntry(new Key("/wals/" + file1, StatusSection.NAME.toString(), "1"),
                ProtobufUtil.toValue(Status.newBuilder().setInfiniteEnd(true).setBegin(Long.MAX_VALUE)
                        .setClosed(true).build())));
        assertFalse(replGC.neededByReplication(conn, "/wals/" + file1));
    }

    @Test
    public void removeReplicationEntries() throws Exception {
        String file1 = UUID.randomUUID().toString(), file2 = UUID.randomUUID().toString();

        Instance inst = new MockInstance(testName.getMethodName());
        AccumuloServerContext context = new AccumuloServerContext(new ServerConfigurationFactory(inst));

        GarbageCollectWriteAheadLogs gcWALs = new GarbageCollectWriteAheadLogs(context, volMgr, false);

        long file1CreateTime = System.currentTimeMillis();
        long file2CreateTime = file1CreateTime + 50;
        BatchWriter bw = ReplicationTable.getBatchWriter(context.getConnector());
        Mutation m = new Mutation("/wals/" + file1);
        StatusSection.add(m, new Text("1"), StatusUtil.fileCreatedValue(file1CreateTime));
        bw.addMutation(m);
        m = new Mutation("/wals/" + file2);
        StatusSection.add(m, new Text("1"), StatusUtil.fileCreatedValue(file2CreateTime));
        bw.addMutation(m);

        // These WALs are potential candidates for deletion from fs
        Map<String, Path> nameToFileMap = new HashMap<>();
        nameToFileMap.put(file1, new Path("/wals/" + file1));
        nameToFileMap.put(file2, new Path("/wals/" + file2));

        Map<String, Path> sortedWALogs = Collections.emptyMap();

        // Make the GCStatus and GcCycleStats
        GCStatus status = new GCStatus();
        GcCycleStats cycleStats = new GcCycleStats();
        status.currentLog = cycleStats;

        // We should iterate over two entries
        Assert.assertEquals(2, gcWALs.removeReplicationEntries(nameToFileMap, sortedWALogs, status));

        // We should have noted that two files were still in use
        Assert.assertEquals(2l, cycleStats.inUse);

        // Both should have been deleted
        Assert.assertEquals(0, nameToFileMap.size());
    }

    @Test
    public void replicationEntriesOnlyInMetaPreventGC() throws Exception {
        String file1 = UUID.randomUUID().toString(), file2 = UUID.randomUUID().toString();

        Instance inst = new MockInstance(testName.getMethodName());
        AccumuloServerContext context = new AccumuloServerContext(new ServerConfigurationFactory(inst));

        Connector conn = context.getConnector();

        GarbageCollectWriteAheadLogs gcWALs = new GarbageCollectWriteAheadLogs(context, volMgr, false);

        long file1CreateTime = System.currentTimeMillis();
        long file2CreateTime = file1CreateTime + 50;
        // Write some records to the metadata table, we haven't yet written status records to the replication table
        BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
        Mutation m = new Mutation(ReplicationSection.getRowPrefix() + "/wals/" + file1);
        m.put(ReplicationSection.COLF, new Text("1"), StatusUtil.fileCreatedValue(file1CreateTime));
        bw.addMutation(m);

        m = new Mutation(ReplicationSection.getRowPrefix() + "/wals/" + file2);
        m.put(ReplicationSection.COLF, new Text("1"), StatusUtil.fileCreatedValue(file2CreateTime));
        bw.addMutation(m);

        // These WALs are potential candidates for deletion from fs
        Map<String, Path> nameToFileMap = new HashMap<>();
        nameToFileMap.put(file1, new Path("/wals/" + file1));
        nameToFileMap.put(file2, new Path("/wals/" + file2));

        Map<String, Path> sortedWALogs = Collections.emptyMap();

        // Make the GCStatus and GcCycleStats objects
        GCStatus status = new GCStatus();
        GcCycleStats cycleStats = new GcCycleStats();
        status.currentLog = cycleStats;

        // We should iterate over two entries
        Assert.assertEquals(2, gcWALs.removeReplicationEntries(nameToFileMap, sortedWALogs, status));

        // We should have noted that two files were still in use
        Assert.assertEquals(2l, cycleStats.inUse);

        // Both should have been deleted
        Assert.assertEquals(0, nameToFileMap.size());
    }

    @Test
    public void noReplicationTableDoesntLimitMetatdataResults() throws Exception {
        Instance inst = new MockInstance(testName.getMethodName());
        AccumuloServerContext context = new AccumuloServerContext(new ServerConfigurationFactory(inst));
        Connector conn = context.getConnector();

        String wal = "hdfs://localhost:8020/accumulo/wal/tserver+port/123456-1234-1234-12345678";
        BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
        Mutation m = new Mutation(ReplicationSection.getRowPrefix() + wal);
        m.put(ReplicationSection.COLF, new Text("1"), StatusUtil.fileCreatedValue(System.currentTimeMillis()));
        bw.addMutation(m);
        bw.close();

        GarbageCollectWriteAheadLogs gcWALs = new GarbageCollectWriteAheadLogs(context, volMgr, false);

        Iterable<Entry<Key, Value>> data = gcWALs.getReplicationStatusForFile(conn, wal);
        Entry<Key, Value> entry = Iterables.getOnlyElement(data);

        Assert.assertEquals(ReplicationSection.getRowPrefix() + wal, entry.getKey().getRow().toString());
    }

    @Test
    public void fetchesReplicationEntriesFromMetadataAndReplicationTables() throws Exception {
        Instance inst = new MockInstance(testName.getMethodName());
        AccumuloServerContext context = new AccumuloServerContext(new ServerConfigurationFactory(inst));
        Connector conn = context.getConnector();

        long walCreateTime = System.currentTimeMillis();
        String wal = "hdfs://localhost:8020/accumulo/wal/tserver+port/123456-1234-1234-12345678";
        BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
        Mutation m = new Mutation(ReplicationSection.getRowPrefix() + wal);
        m.put(ReplicationSection.COLF, new Text("1"), StatusUtil.fileCreatedValue(walCreateTime));
        bw.addMutation(m);
        bw.close();

        bw = ReplicationTable.getBatchWriter(conn);
        m = new Mutation(wal);
        StatusSection.add(m, new Text("1"), StatusUtil.fileCreatedValue(walCreateTime));
        bw.addMutation(m);
        bw.close();

        GarbageCollectWriteAheadLogs gcWALs = new GarbageCollectWriteAheadLogs(context, volMgr, false);

        Iterable<Entry<Key, Value>> iter = gcWALs.getReplicationStatusForFile(conn, wal);
        Map<Key, Value> data = new HashMap<>();
        for (Entry<Key, Value> e : iter) {
            data.put(e.getKey(), e.getValue());
        }

        Assert.assertEquals(2, data.size());

        // Should get one element from each table (metadata and replication)
        for (Key k : data.keySet()) {
            String row = k.getRow().toString();
            if (row.startsWith(ReplicationSection.getRowPrefix())) {
                Assert.assertTrue(row.endsWith(wal));
            } else {
                Assert.assertEquals(wal, row);
            }
        }
    }

    @Test
    public void testTimeToDeleteTrue() throws InterruptedException {
        HostAndPort address = HostAndPort.fromString("tserver1:9998");
        long wait = AccumuloConfiguration.getTimeInMillis("1s");
        gcwal.clearFirstSeenDead();
        assertFalse("First call should be false and should store the first seen time",
                gcwal.timeToDelete(address, wait));
        sleep(wait * 2);
        assertTrue(gcwal.timeToDelete(address, wait));
    }

    @Test
    public void testTimeToDeleteFalse() {
        HostAndPort address = HostAndPort.fromString("tserver1:9998");
        long wait = AccumuloConfiguration.getTimeInMillis("1h");
        long t1, t2;
        boolean ttd;
        do {
            t1 = System.nanoTime();
            gcwal.clearFirstSeenDead();
            assertFalse("First call should be false and should store the first seen time",
                    gcwal.timeToDelete(address, wait));
            ttd = gcwal.timeToDelete(address, wait);
            t2 = System.nanoTime();
        } while (TimeUnit.NANOSECONDS.toMillis(t2 - t1) > (wait / 2)); // as long as it took less than half of the configured wait

        assertFalse(ttd);
    }

    @Test
    public void testTimeToDeleteWithNullAddress() {
        assertFalse(gcwal.timeToDelete(null, 123l));
    }

    /**
     * Wrapper class with some helper methods
     * <p>
     * Just a wrapper around a LinkedHashMap that store method name and argument information. Also includes some convenience methods to make usage cleaner.
     */
    class MethodCalls {

        private LinkedHashMap<String, List<Object>> mapWrapper;

        public MethodCalls() {
            mapWrapper = new LinkedHashMap<String, List<Object>>();
        }

        public void put(String methodName, Object... args) {
            mapWrapper.put(methodName, Arrays.asList(args));
        }

        public int size() {
            return mapWrapper.size();
        }

        public boolean hasOneEntry() {
            return size() == 1;
        }

        public Map.Entry<String, List<Object>> getFirstEntry() {
            return mapWrapper.entrySet().iterator().next();
        }

        public String getFirstEntryMethod() {
            return getFirstEntry().getKey();
        }

        public List<Object> getFirstEntryArgs() {
            return getFirstEntry().getValue();
        }

        public Object getFirstEntryArg(int number) {
            return getFirstEntryArgs().get(number);
        }
    }

    /**
     * Partial mock of the GarbageCollectWriteAheadLogs for testing the removeFile method
     * <p>
     * There is a map named methodCalls that can be used to assert parameters on methods called inside the removeFile method
     */
    class GCWALPartialMock extends GarbageCollectWriteAheadLogs {

        private boolean holdsLockBool = false;

        public GCWALPartialMock(AccumuloServerContext ctx, VolumeManager vm, boolean useTrash, boolean holdLock)
                throws IOException {
            super(ctx, vm, useTrash);
            this.holdsLockBool = holdLock;
        }

        public MethodCalls methodCalls = new MethodCalls();

        @Override
        boolean holdsLock(HostAndPort addr) {
            return holdsLockBool;
        }

        @Override
        void removeWALfromDownTserver(HostAndPort address, AccumuloConfiguration conf,
                Entry<String, ArrayList<Path>> entry, final GCStatus status) {
            methodCalls.put("removeWALFromDownTserver", address, conf, entry, status);
        }

        @Override
        void askTserverToRemoveWAL(HostAndPort address, AccumuloConfiguration conf,
                Entry<String, ArrayList<Path>> entry, final GCStatus status) {
            methodCalls.put("askTserverToRemoveWAL", address, conf, entry, status);
        }

        @Override
        void removeOldStyleWAL(Entry<String, ArrayList<Path>> entry, final GCStatus status) {
            methodCalls.put("removeOldStyleWAL", entry, status);
        }

        @Override
        void removeSortedWAL(Path swalog) {
            methodCalls.put("removeSortedWAL", swalog);
        }
    }

    private GCWALPartialMock getGCWALForRemoveFileTest(GCStatus s, final boolean locked) throws IOException {
        AccumuloServerContext ctx = new AccumuloServerContext(
                new ServerConfigurationFactory(new MockInstance("accumulo")));
        return new GCWALPartialMock(ctx, VolumeManagerImpl.get(), false, locked);
    }

    private Map<String, Path> getEmptyMap() {
        return new HashMap<String, Path>();
    }

    private Map<String, ArrayList<Path>> getServerToFileMap1(String key, Path singlePath) {
        Map<String, ArrayList<Path>> serverToFileMap = new HashMap<String, ArrayList<Path>>();
        serverToFileMap.put(key, new ArrayList<Path>(Arrays.asList(singlePath)));
        return serverToFileMap;
    }

    @Test
    public void testRemoveFilesWithOldStyle() throws IOException {
        GCStatus status = new GCStatus();
        GarbageCollectWriteAheadLogs realGCWAL = getGCWALForRemoveFileTest(status, true);
        Path p1 = new Path("hdfs://localhost:9000/accumulo/wal/tserver1+9997/" + UUID.randomUUID().toString());
        Map<String, ArrayList<Path>> serverToFileMap = getServerToFileMap1("", p1);

        realGCWAL.removeFiles(getEmptyMap(), serverToFileMap, getEmptyMap(), status);

        MethodCalls calls = ((GCWALPartialMock) realGCWAL).methodCalls;
        assertEquals("Only one method should have been called", 1, calls.size());
        assertEquals("Method should be removeOldStyleWAL", "removeOldStyleWAL", calls.getFirstEntryMethod());
        Entry<String, ArrayList<Path>> firstServerToFileMap = serverToFileMap.entrySet().iterator().next();
        assertEquals("First param should be empty", firstServerToFileMap, calls.getFirstEntryArg(0));
        assertEquals("Second param should be the status", status, calls.getFirstEntryArg(1));
    }

    @Test
    public void testRemoveFilesWithDeadTservers() throws IOException {
        GCStatus status = new GCStatus();
        GarbageCollectWriteAheadLogs realGCWAL = getGCWALForRemoveFileTest(status, false);
        String server = "tserver1+9997";
        Path p1 = new Path("hdfs://localhost:9000/accumulo/wal/" + server + "/" + UUID.randomUUID().toString());
        Map<String, ArrayList<Path>> serverToFileMap = getServerToFileMap1(server, p1);

        realGCWAL.removeFiles(getEmptyMap(), serverToFileMap, getEmptyMap(), status);

        MethodCalls calls = ((GCWALPartialMock) realGCWAL).methodCalls;
        assertEquals("Only one method should have been called", 1, calls.size());
        assertEquals("Method should be removeWALfromDownTserver", "removeWALFromDownTserver",
                calls.getFirstEntryMethod());
        assertEquals("First param should be address", HostAndPort.fromString(server.replaceAll("[+]", ":")),
                calls.getFirstEntryArg(0));
        assertTrue("Second param should be an AccumuloConfiguration",
                calls.getFirstEntryArg(1) instanceof AccumuloConfiguration);
        Entry<String, ArrayList<Path>> firstServerToFileMap = serverToFileMap.entrySet().iterator().next();
        assertEquals("Third param should be the entry", firstServerToFileMap, calls.getFirstEntryArg(2));
        assertEquals("Forth param should be the status", status, calls.getFirstEntryArg(3));
    }

    @Test
    public void testRemoveFilesWithLiveTservers() throws IOException {
        GCStatus status = new GCStatus();
        GarbageCollectWriteAheadLogs realGCWAL = getGCWALForRemoveFileTest(status, true);
        String server = "tserver1+9997";
        Path p1 = new Path("hdfs://localhost:9000/accumulo/wal/" + server + "/" + UUID.randomUUID().toString());
        Map<String, ArrayList<Path>> serverToFileMap = getServerToFileMap1(server, p1);

        realGCWAL.removeFiles(getEmptyMap(), serverToFileMap, getEmptyMap(), status);

        MethodCalls calls = ((GCWALPartialMock) realGCWAL).methodCalls;
        assertEquals("Only one method should have been called", 1, calls.size());
        assertEquals("Method should be askTserverToRemoveWAL", "askTserverToRemoveWAL",
                calls.getFirstEntryMethod());
        assertEquals("First param should be address", HostAndPort.fromString(server.replaceAll("[+]", ":")),
                calls.getFirstEntryArg(0));
        assertTrue("Second param should be an AccumuloConfiguration",
                calls.getFirstEntryArg(1) instanceof AccumuloConfiguration);
        Entry<String, ArrayList<Path>> firstServerToFileMap = serverToFileMap.entrySet().iterator().next();
        assertEquals("Third param should be the entry", firstServerToFileMap, calls.getFirstEntryArg(2));
        assertEquals("Forth param should be the status", status, calls.getFirstEntryArg(3));
    }

    @Test
    public void testRemoveFilesRemovesSortedWALs() throws IOException {
        GCStatus status = new GCStatus();
        GarbageCollectWriteAheadLogs realGCWAL = getGCWALForRemoveFileTest(status, true);
        Map<String, ArrayList<Path>> serverToFileMap = new HashMap<String, ArrayList<Path>>();
        Map<String, Path> sortedWALogs = new HashMap<String, Path>();
        Path p1 = new Path("hdfs://localhost:9000/accumulo/wal/tserver1+9997/" + UUID.randomUUID().toString());
        sortedWALogs.put("junk", p1); // TODO: see if this key is actually used here, maybe can be removed

        realGCWAL.removeFiles(getEmptyMap(), serverToFileMap, sortedWALogs, status);
        MethodCalls calls = ((GCWALPartialMock) realGCWAL).methodCalls;
        assertEquals("Only one method should have been called", 1, calls.size());
        assertEquals("Method should be removeSortedWAL", "removeSortedWAL", calls.getFirstEntryMethod());
        assertEquals("First param should be the Path", p1, calls.getFirstEntryArg(0));

    }

    static String GCWAL_DEAD_DIR = "gcwal-collect-deadtserver";
    static String GCWAL_DEAD_TSERVER = "tserver1";
    static String GCWAL_DEAD_TSERVER_PORT = "9995";
    static String GCWAL_DEAD_TSERVER_COLLECT_FILE = UUID.randomUUID().toString();

    class GCWALDeadTserverCollectMock extends GarbageCollectWriteAheadLogs {

        public GCWALDeadTserverCollectMock(AccumuloServerContext ctx, VolumeManager vm, boolean useTrash)
                throws IOException {
            super(ctx, vm, useTrash);
        }

        @Override
        boolean holdsLock(HostAndPort addr) {
            // tries use zookeeper
            return false;
        }

        @Override
        Map<String, Path> getSortedWALogs() {
            return new HashMap<String, Path>();
        }

        @Override
        int scanServers(Map<Path, String> fileToServerMap, Map<String, Path> nameToFileMap) throws Exception {
            String sep = File.separator;
            Path p = new Path(System.getProperty("user.dir") + sep + "target" + sep + GCWAL_DEAD_DIR + sep
                    + GCWAL_DEAD_TSERVER + "+" + GCWAL_DEAD_TSERVER_PORT + sep + GCWAL_DEAD_TSERVER_COLLECT_FILE);
            fileToServerMap.put(p, GCWAL_DEAD_TSERVER + ":" + GCWAL_DEAD_TSERVER_PORT);
            nameToFileMap.put(GCWAL_DEAD_TSERVER_COLLECT_FILE, p);
            return 1;
        }

        @Override
        int removeMetadataEntries(Map<String, Path> nameToFileMap, Map<String, Path> sortedWALogs, GCStatus status)
                throws IOException, KeeperException, InterruptedException {
            return 0;
        }

        long getGCWALDeadServerWaitTime(AccumuloConfiguration conf) {
            // tries to use zookeeper
            return 1000l;
        }
    }

    @Test
    public void testCollectWithDeadTserver() throws IOException, InterruptedException {
        Instance i = new MockInstance();
        AccumuloServerContext ctx = new AccumuloServerContext(new ServerConfigurationFactory(i));
        File walDir = new File(
                System.getProperty("user.dir") + File.separator + "target" + File.separator + GCWAL_DEAD_DIR);
        File walFileDir = new File(walDir + File.separator + GCWAL_DEAD_TSERVER + "+" + GCWAL_DEAD_TSERVER_PORT);
        File walFile = new File(walFileDir + File.separator + GCWAL_DEAD_TSERVER_COLLECT_FILE);
        if (!walFileDir.exists()) {
            assertTrue("Directory was made", walFileDir.mkdirs());
            new FileOutputStream(walFile).close();
        }

        try {
            VolumeManager vm = VolumeManagerImpl.getLocal(walDir.toString());
            GarbageCollectWriteAheadLogs gcwal2 = new GCWALDeadTserverCollectMock(ctx, vm, false);
            GCStatus status = new GCStatus(new GcCycleStats(), new GcCycleStats(), new GcCycleStats(),
                    new GcCycleStats());

            gcwal2.collect(status);

            assertTrue("File should not be deleted", walFile.exists());
            assertEquals("Should have one candidate", 1, status.lastLog.getCandidates());
            assertEquals("Should not have deleted that file", 0, status.lastLog.getDeleted());

            sleep(2000);
            gcwal2.collect(status);

            assertFalse("File should be gone", walFile.exists());
            assertEquals("Should have one candidate", 1, status.lastLog.getCandidates());
            assertEquals("Should have deleted that file", 1, status.lastLog.getDeleted());

        } finally {
            if (walDir.exists()) {
                FileUtils.deleteDirectory(walDir);
            }
        }
    }
}