org.apache.hadoop.hdfs.TestDFSOpsCountStatistics.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hdfs.TestDFSOpsCountStatistics.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * <p>
 * http://www.apache.org/licenses/LICENSE-2.0
 * <p>
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hdfs;

import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.fs.StorageStatistics.LongStatistic;

import org.apache.hadoop.hdfs.DFSOpsCountStatistics.OpType;

import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;

import org.junit.rules.ExpectedException;
import org.junit.rules.Timeout;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;

import static java.util.concurrent.Executors.newFixedThreadPool;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

/**
 * This tests basic operations of {@link DFSOpsCountStatistics} class.
 */
public class TestDFSOpsCountStatistics {

    private static final Logger LOG = LoggerFactory.getLogger(TestDFSOpsCountStatistics.class);
    private static final String NO_SUCH_OP = "no-such-dfs-operation-dude";

    private final DFSOpsCountStatistics statistics = new DFSOpsCountStatistics();
    private final Map<OpType, AtomicLong> expectedOpsCountMap = new HashMap<>();

    @Rule
    public final Timeout globalTimeout = new Timeout(10 * 1000);
    @Rule
    public final ExpectedException exception = ExpectedException.none();

    @Before
    public void setup() {
        for (OpType opType : OpType.values()) {
            expectedOpsCountMap.put(opType, new AtomicLong());
        }
        incrementOpsCountByRandomNumbers();
    }

    /**
     * This is to test the the {@link OpType} symbols are unique.
     */
    @Test
    public void testOpTypeSymbolsAreUnique() {
        final Set<String> opTypeSymbols = new HashSet<>();
        for (OpType opType : OpType.values()) {
            assertFalse(opTypeSymbols.contains(opType.getSymbol()));
            opTypeSymbols.add(opType.getSymbol());
        }
        assertEquals(OpType.values().length, opTypeSymbols.size());
    }

    @Test
    public void testGetLongStatistics() {
        short iterations = 0; // number of the iter.hasNext()
        final Iterator<LongStatistic> iter = statistics.getLongStatistics();

        while (iter.hasNext()) {
            final LongStatistic longStat = iter.next();
            assertNotNull(longStat);
            final OpType opType = OpType.fromSymbol(longStat.getName());
            assertNotNull(opType);
            assertTrue(expectedOpsCountMap.containsKey(opType));
            assertEquals(expectedOpsCountMap.get(opType).longValue(), longStat.getValue());
            iterations++;
        }

        // check that all the OpType enum entries are iterated via iter
        assertEquals(OpType.values().length, iterations);
    }

    @Test
    public void testGetLong() {
        assertNull(statistics.getLong(null));
        assertNull(statistics.getLong(NO_SUCH_OP));
        verifyStatistics();
    }

    @Test
    public void testIsTracked() {
        assertFalse(statistics.isTracked(null));
        assertFalse(statistics.isTracked(NO_SUCH_OP));

        final Iterator<LongStatistic> iter = statistics.getLongStatistics();
        while (iter.hasNext()) {
            final LongStatistic longStatistic = iter.next();
            assertTrue(statistics.isTracked(longStatistic.getName()));
        }
    }

    @Test
    public void testReset() {
        statistics.reset();
        for (OpType opType : OpType.values()) {
            expectedOpsCountMap.get(opType).set(0);
        }

        final Iterator<LongStatistic> iter = statistics.getLongStatistics();
        while (iter.hasNext()) {
            final LongStatistic longStat = iter.next();
            assertEquals(0, longStat.getValue());
        }

        incrementOpsCountByRandomNumbers();
        verifyStatistics();
    }

    @Test
    public void testCurrentAccess() throws InterruptedException {
        final int numThreads = 10;
        final ExecutorService threadPool = newFixedThreadPool(numThreads);

        try {
            final CountDownLatch allReady = new CountDownLatch(numThreads);
            final CountDownLatch startBlocker = new CountDownLatch(1);
            final CountDownLatch allDone = new CountDownLatch(numThreads);
            final AtomicReference<Throwable> childError = new AtomicReference<>();

            for (int i = 0; i < numThreads; i++) {
                threadPool.submit(new Runnable() {
                    @Override
                    public void run() {
                        allReady.countDown();
                        try {
                            startBlocker.await();
                            incrementOpsCountByRandomNumbers();
                        } catch (Throwable t) {
                            LOG.error("Child failed when calling mkdir", t);
                            childError.compareAndSet(null, t);
                        } finally {
                            allDone.countDown();
                        }
                    }
                });
            }

            allReady.await(); // wait until all threads are ready
            startBlocker.countDown(); // all threads start making directories
            allDone.await(); // wait until all threads are done

            assertNull("Child failed with exception.", childError.get());
            verifyStatistics();
        } finally {
            threadPool.shutdownNow();
        }
    }

    /**
     * This is helper method to increment the statistics by random data.
     */
    private void incrementOpsCountByRandomNumbers() {
        for (OpType opType : OpType.values()) {
            final Long randomCount = RandomUtils.nextLong(0, 100);
            expectedOpsCountMap.get(opType).addAndGet(randomCount);
            for (long i = 0; i < randomCount; i++) {
                statistics.incrementOpCounter(opType);
            }
        }
    }

    /**
     * We have the expected ops count in {@link #expectedOpsCountMap}, and this
     * method is to verify that its ops count is the same as the one in
     * {@link #statistics}.
     */
    private void verifyStatistics() {
        for (OpType opType : OpType.values()) {
            assertNotNull(expectedOpsCountMap.get(opType));
            assertNotNull(statistics.getLong(opType.getSymbol()));
            assertEquals("Not expected count for operation " + opType.getSymbol(),
                    expectedOpsCountMap.get(opType).longValue(),
                    statistics.getLong(opType.getSymbol()).longValue());
        }
    }

}