org.apache.accumulo.test.functional.FunctionalTestUtils.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.accumulo.test.functional.FunctionalTestUtils.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.accumulo.test.functional;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;

import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

import org.apache.accumulo.cluster.AccumuloCluster;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.fate.AdminUtil;
import org.apache.accumulo.fate.AdminUtil.FateStatus;
import org.apache.accumulo.fate.ZooStore;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.server.zookeeper.ZooReaderWriterFactory;
import org.apache.accumulo.test.TestIngest;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.zookeeper.KeeperException;

import com.google.common.collect.Iterators;

public class FunctionalTestUtils {

    public static int countRFiles(AccumuloClient c, String tableName) throws Exception {
        try (Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            TableId tableId = TableId.of(c.tableOperations().tableIdMap().get(tableName));
            scanner.setRange(MetadataSchema.TabletsSection.getRange(tableId));
            scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
            return Iterators.size(scanner.iterator());
        }
    }

    static void checkRFiles(AccumuloClient c, String tableName, int minTablets, int maxTablets, int minRFiles,
            int maxRFiles) throws Exception {
        try (Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
            String tableId = c.tableOperations().tableIdMap().get(tableName);
            scanner.setRange(new Range(new Text(tableId + ";"), true, new Text(tableId + "<"), true));
            scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
            MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);

            HashMap<Text, Integer> tabletFileCounts = new HashMap<>();

            for (Entry<Key, Value> entry : scanner) {

                Text row = entry.getKey().getRow();

                Integer count = tabletFileCounts.get(row);
                if (count == null)
                    count = 0;
                if (entry.getKey().getColumnFamily()
                        .equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME)) {
                    count = count + 1;
                }

                tabletFileCounts.put(row, count);
            }

            if (tabletFileCounts.size() < minTablets || tabletFileCounts.size() > maxTablets) {
                throw new Exception("Did not find expected number of tablets " + tabletFileCounts.size());
            }

            Set<Entry<Text, Integer>> es = tabletFileCounts.entrySet();
            for (Entry<Text, Integer> entry : es) {
                if (entry.getValue() > maxRFiles || entry.getValue() < minRFiles) {
                    throw new Exception("tablet " + entry.getKey() + " has " + entry.getValue() + " map files");
                }
            }
        }
    }

    public static void checkSplits(AccumuloClient c, String table, int min, int max) throws Exception {
        Collection<Text> splits = c.tableOperations().listSplits(table);
        if (splits.size() < min || splits.size() > max) {
            throw new Exception("# of table splits points out of range, #splits=" + splits.size() + " table="
                    + table + " min=" + min + " max=" + max);
        }
    }

    public static void createRFiles(final AccumuloClient c, final FileSystem fs, String path, int rows, int splits,
            int threads) throws Exception {
        fs.delete(new Path(path), true);
        ExecutorService threadPool = Executors.newFixedThreadPool(threads);
        final AtomicBoolean fail = new AtomicBoolean(false);
        for (int i = 0; i < rows; i += rows / splits) {
            TestIngest.IngestParams params = new TestIngest.IngestParams(c.properties());
            params.outputFile = String.format("%s/mf%s", path, i);
            params.random = 56;
            params.timestamp = 1;
            params.dataSize = 50;
            params.rows = rows / splits;
            params.startRow = i;
            params.cols = 1;
            threadPool.execute(() -> {
                try {
                    TestIngest.ingest(c, fs, params);
                } catch (Exception e) {
                    fail.set(true);
                }
            });
        }
        threadPool.shutdown();
        threadPool.awaitTermination(1, TimeUnit.HOURS);
        assertFalse(fail.get());
    }

    public static String readAll(InputStream is) throws IOException {
        return IOUtils.toString(is, UTF_8);
    }

    static Mutation nm(String row, String cf, String cq, Value value) {
        Mutation m = new Mutation(new Text(row));
        m.put(new Text(cf), new Text(cq), value);
        return m;
    }

    static Mutation nm(String row, String cf, String cq, String value) {
        return nm(row, cf, cq, new Value(value.getBytes()));
    }

    public static SortedSet<Text> splits(String[] splits) {
        SortedSet<Text> result = new TreeSet<>();
        for (String split : splits)
            result.add(new Text(split));
        return result;
    }

    public static void assertNoDanglingFateLocks(ClientContext context, AccumuloCluster cluster) {
        FateStatus fateStatus = getFateStatus(context, cluster);
        assertEquals("Dangling FATE locks : " + fateStatus.getDanglingHeldLocks(), 0,
                fateStatus.getDanglingHeldLocks().size());
        assertEquals("Dangling FATE locks : " + fateStatus.getDanglingWaitingLocks(), 0,
                fateStatus.getDanglingWaitingLocks().size());
    }

    private static FateStatus getFateStatus(ClientContext context, AccumuloCluster cluster) {
        try {
            AdminUtil<String> admin = new AdminUtil<>(false);
            String secret = cluster.getSiteConfiguration().get(Property.INSTANCE_SECRET);
            IZooReaderWriter zk = new ZooReaderWriterFactory().getZooReaderWriter(context.getZooKeepers(),
                    context.getZooKeepersSessionTimeOut(), secret);
            ZooStore<String> zs = new ZooStore<>(context.getZooKeeperRoot() + Constants.ZFATE, zk);
            return admin.getStatus(zs, zk, context.getZooKeeperRoot() + Constants.ZTABLE_LOCKS, null, null);
        } catch (KeeperException | InterruptedException e) {
            throw new RuntimeException(e);
        }
    }
}