com.twitter.distributedlog.DLMTestUtil.java Source code

Java tutorial

Introduction

Here is the source code for com.twitter.distributedlog.DLMTestUtil.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.twitter.distributedlog;

import com.twitter.distributedlog.impl.BKLogSegmentEntryWriter;
import com.twitter.distributedlog.metadata.BKDLConfig;
import com.twitter.distributedlog.metadata.DLMetadata;
import com.twitter.distributedlog.namespace.DistributedLogNamespace;
import com.twitter.distributedlog.namespace.DistributedLogNamespaceBuilder;
import com.twitter.distributedlog.util.ConfUtils;
import com.twitter.distributedlog.util.FutureUtils;
import com.twitter.distributedlog.util.PermitLimiter;
import com.twitter.distributedlog.util.RetryPolicyUtils;
import com.twitter.distributedlog.util.Utils;
import com.twitter.util.Await;
import com.twitter.util.Duration;
import com.twitter.util.Future;
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.feature.SettableFeatureProvider;
import org.apache.bookkeeper.stats.NullStatsLogger;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static com.google.common.base.Charsets.UTF_8;
import static org.junit.Assert.assertNotNull;

/**
 * Utility class for setting up bookkeeper ensembles
 * and bringing individual bookies up and down
 */
public class DLMTestUtil {
    protected static final Logger LOG = LoggerFactory.getLogger(DLMTestUtil.class);
    private final static byte[] payloadStatic = repeatString("abc", 512).getBytes();

    static String repeatString(String s, int n) {
        String ret = s;
        for (int i = 1; i < n; i++) {
            ret += s;
        }
        return ret;
    }

    public static Map<Long, LogSegmentMetadata> readLogSegments(ZooKeeperClient zkc, String ledgerPath)
            throws Exception {
        List<String> children = zkc.get().getChildren(ledgerPath, false);
        LOG.info("Children under {} : {}", ledgerPath, children);
        Map<Long, LogSegmentMetadata> segments = new HashMap<Long, LogSegmentMetadata>(children.size());
        for (String child : children) {
            LogSegmentMetadata segment = FutureUtils.result(LogSegmentMetadata.read(zkc, ledgerPath + "/" + child));
            LOG.info("Read segment {} : {}", child, segment);
            segments.put(segment.getLogSegmentSequenceNumber(), segment);
        }
        return segments;
    }

    static void updateBKDLConfig(URI uri, String zkServers, String ledgersPath, boolean sanityCheckTxnID)
            throws Exception {
        BKDLConfig bkdlConfig = new BKDLConfig(zkServers, ledgersPath).setSanityCheckTxnID(sanityCheckTxnID);
        DLMetadata.create(bkdlConfig).update(uri);
    }

    public static URI createDLMURI(int port, String path) throws Exception {
        return LocalDLMEmulator.createDLMURI("127.0.0.1:" + port, path);
    }

    public static DistributedLogManager createNewDLM(String name, DistributedLogConfiguration conf, URI uri)
            throws Exception {
        DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
        return namespace.openLog(name);
    }

    static MetadataAccessor createNewMetadataAccessor(DistributedLogConfiguration conf, String name, URI uri)
            throws Exception {
        // TODO: Metadata Accessor seems to be a legacy object which only used by kestrel
        //       (we might consider deprecating this)
        BKDistributedLogNamespace namespace = BKDistributedLogNamespace.newBuilder().conf(conf).uri(uri).build();
        return namespace.createMetadataAccessor(name);
    }

    public static class BKLogPartitionWriteHandlerAndClients {
        private BKLogWriteHandler writeHandler;
        private ZooKeeperClient zooKeeperClient;
        private BookKeeperClient bookKeeperClient;

        public BKLogPartitionWriteHandlerAndClients(BKLogWriteHandler writeHandler, ZooKeeperClient zooKeeperClient,
                BookKeeperClient bookKeeperClient) {
            this.writeHandler = writeHandler;
            this.zooKeeperClient = zooKeeperClient;
            this.bookKeeperClient = bookKeeperClient;
        }

        public void close() {
            bookKeeperClient.close();
            zooKeeperClient.close();
            Utils.closeQuietly(writeHandler);
        }

        public BKLogWriteHandler getWriteHandler() {
            return writeHandler;
        }
    }

    static BKLogPartitionWriteHandlerAndClients createNewBKDLM(DistributedLogConfiguration conf, String logName,
            int zkPort) throws Exception {
        URI uri = createDLMURI(zkPort, "/" + logName);

        ZooKeeperClientBuilder zkcBuilder = TestZooKeeperClientBuilder.newBuilder(conf)
                .name(String.format("dlzk:%s:handler_dedicated", logName)).uri(uri);

        ZooKeeperClient zkClient = zkcBuilder.build();

        try {
            zkClient.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
        } catch (KeeperException.NodeExistsException nee) {
            // ignore
        }

        // resolve uri
        BKDLConfig bkdlConfig = BKDLConfig.resolveDLConfig(zkClient, uri);
        BKDLConfig.propagateConfiguration(bkdlConfig, conf);
        BookKeeperClientBuilder bkcBuilder = BookKeeperClientBuilder.newBuilder().dlConfig(conf)
                .name(String.format("bk:%s:handler_dedicated", logName))
                .zkServers(bkdlConfig.getBkZkServersForWriter()).ledgersPath(bkdlConfig.getBkLedgersPath())
                .statsLogger(NullStatsLogger.INSTANCE);

        BKDistributedLogManager bkdlm = new BKDistributedLogManager(logName, conf, uri, zkcBuilder, zkcBuilder,
                zkClient, zkClient, bkcBuilder, bkcBuilder, new SettableFeatureProvider("", 0),
                PermitLimiter.NULL_PERMIT_LIMITER, NullStatsLogger.INSTANCE);

        BKLogWriteHandler writeHandler = bkdlm.createWriteHandler(true);
        return new BKLogPartitionWriteHandlerAndClients(writeHandler, zkClient, bkcBuilder.build());
    }

    public static void fenceStream(DistributedLogConfiguration conf, URI uri, String name) throws Exception {
        BKDistributedLogManager dlm = (BKDistributedLogManager) createNewDLM(name, conf, uri);
        try {
            BKLogReadHandler readHandler = dlm.createReadHandler();
            List<LogSegmentMetadata> ledgerList = readHandler.getFullLedgerList(true, true);
            LogSegmentMetadata lastSegment = ledgerList.get(ledgerList.size() - 1);
            BookKeeperClient bkc = dlm.getWriterBKC();
            LedgerHandle lh = bkc.get().openLedger(lastSegment.getLedgerId(), BookKeeper.DigestType.CRC32,
                    conf.getBKDigestPW().getBytes(UTF_8));
            lh.close();
        } finally {
            dlm.close();
        }
    }

    static long getNumberofLogRecords(DistributedLogManager bkdlm, long startTxId) throws IOException {
        long numLogRecs = 0;
        LogReader reader = bkdlm.getInputStream(startTxId);
        LogRecord record = reader.readNext(false);
        while (null != record) {
            numLogRecs++;
            verifyLogRecord(record);
            record = reader.readNext(false);
        }
        reader.close();
        return numLogRecs;
    }

    public static LogRecord getLogRecordInstance(long txId) {
        return new LogRecord(txId, generatePayload(txId));
    }

    public static LogRecord getLogRecordInstance(long txId, int size) {
        ByteBuffer buf = ByteBuffer.allocate(size);
        return new LogRecord(txId, buf.array());
    }

    public static void verifyLogRecord(LogRecord record) {
        assertEquals(generatePayload(record.getTransactionId()).length, record.getPayload().length);
        assertArrayEquals(generatePayload(record.getTransactionId()), record.getPayload());
        assert (!record.isControl());
        verifyPayload(record.getTransactionId(), record.getPayload());
    }

    static byte[] generatePayload(long txId) {
        return String.format("%d;%d", txId, txId).getBytes();
    }

    static void verifyPayload(long txId, byte[] payload) {
        String[] txIds = new String(payload).split(";");
        assertEquals(Long.valueOf(txIds[0]), Long.valueOf(txIds[0]));
    }

    static LogRecord getLargeLogRecordInstance(long txId, boolean control) {
        LogRecord record = new LogRecord(txId, payloadStatic);
        if (control) {
            record.setControl();
        }
        return record;
    }

    static LogRecord getLargeLogRecordInstance(long txId) {
        return new LogRecord(txId, payloadStatic);
    }

    static List<LogRecord> getLargeLogRecordInstanceList(long firstTxId, int count) {
        List<LogRecord> logrecs = new ArrayList<LogRecord>(count);
        for (long i = 0; i < count; i++) {
            logrecs.add(getLargeLogRecordInstance(firstTxId + i));
        }
        return logrecs;
    }

    static List<LogRecord> getLogRecordInstanceList(long firstTxId, int count, int size) {
        List<LogRecord> logrecs = new ArrayList<LogRecord>(count);
        for (long i = 0; i < count; i++) {
            logrecs.add(getLogRecordInstance(firstTxId + i, size));
        }
        return logrecs;
    }

    static void verifyLargeLogRecord(LogRecord record) {
        verifyLargeLogRecord(record.getPayload());
    }

    static void verifyLargeLogRecord(byte[] payload) {
        assertArrayEquals(payloadStatic, payload);
    }

    static LogRecord getEmptyLogRecordInstance(long txId) {
        return new LogRecord(txId, new byte[0]);
    }

    static void verifyEmptyLogRecord(LogRecord record) {
        assert (record.getPayload().length == 0);
    }

    public static LogRecordWithDLSN getLogRecordWithDLSNInstance(DLSN dlsn, long txId) {
        return getLogRecordWithDLSNInstance(dlsn, txId, false);
    }

    public static LogRecordWithDLSN getLogRecordWithDLSNInstance(DLSN dlsn, long txId, boolean isControlRecord) {
        LogRecordWithDLSN record = new LogRecordWithDLSN(dlsn, txId, generatePayload(txId), 1L);
        record.setPositionWithinLogSegment((int) txId + 1);
        if (isControlRecord) {
            record.setControl();
        }
        return record;
    }

    public static String inprogressZNodeName(long logSegmentSeqNo) {
        return String.format("%s_%018d", DistributedLogConstants.INPROGRESS_LOGSEGMENT_PREFIX, logSegmentSeqNo);
    }

    public static String completedLedgerZNodeNameWithVersion(long ledgerId, long firstTxId, long lastTxId,
            long logSegmentSeqNo) {
        return String.format("%s_%018d_%018d_%018d_v%dl%d_%04d",
                DistributedLogConstants.COMPLETED_LOGSEGMENT_PREFIX, firstTxId, lastTxId, logSegmentSeqNo,
                DistributedLogConstants.LOGSEGMENT_NAME_VERSION, ledgerId, DistributedLogConstants.LOCAL_REGION_ID);
    }

    public static String completedLedgerZNodeNameWithTxID(long firstTxId, long lastTxId) {
        return String.format("%s_%018d_%018d", DistributedLogConstants.COMPLETED_LOGSEGMENT_PREFIX, firstTxId,
                lastTxId);
    }

    public static String completedLedgerZNodeNameWithLogSegmentSequenceNumber(long logSegmentSeqNo) {
        return String.format("%s_%018d", DistributedLogConstants.COMPLETED_LOGSEGMENT_PREFIX, logSegmentSeqNo);
    }

    public static LogSegmentMetadata inprogressLogSegment(String ledgerPath, long ledgerId, long firstTxId,
            long logSegmentSeqNo) {
        return inprogressLogSegment(ledgerPath, ledgerId, firstTxId, logSegmentSeqNo,
                LogSegmentMetadata.LEDGER_METADATA_CURRENT_LAYOUT_VERSION);
    }

    public static LogSegmentMetadata inprogressLogSegment(String ledgerPath, long ledgerId, long firstTxId,
            long logSegmentSeqNo, int version) {
        return new LogSegmentMetadata.LogSegmentMetadataBuilder(
                ledgerPath + "/" + inprogressZNodeName(logSegmentSeqNo), version, ledgerId, firstTxId)
                        .setLogSegmentSequenceNo(logSegmentSeqNo).build();
    }

    public static LogSegmentMetadata completedLogSegment(String ledgerPath, long ledgerId, long firstTxId,
            long lastTxId, int recordCount, long logSegmentSeqNo, long lastEntryId, long lastSlotId) {
        return completedLogSegment(ledgerPath, ledgerId, firstTxId, lastTxId, recordCount, logSegmentSeqNo,
                lastEntryId, lastSlotId, LogSegmentMetadata.LEDGER_METADATA_CURRENT_LAYOUT_VERSION);
    }

    public static LogSegmentMetadata completedLogSegment(String ledgerPath, long ledgerId, long firstTxId,
            long lastTxId, int recordCount, long logSegmentSeqNo, long lastEntryId, long lastSlotId, int version) {
        LogSegmentMetadata metadata = new LogSegmentMetadata.LogSegmentMetadataBuilder(
                ledgerPath + "/" + inprogressZNodeName(logSegmentSeqNo), version, ledgerId, firstTxId)
                        .setInprogress(false).setLogSegmentSequenceNo(logSegmentSeqNo).build();
        return metadata.completeLogSegment(
                ledgerPath + "/" + completedLedgerZNodeNameWithLogSegmentSequenceNumber(logSegmentSeqNo), lastTxId,
                recordCount, lastEntryId, lastSlotId, firstTxId);
    }

    public static void generateCompletedLogSegments(DistributedLogManager manager, DistributedLogConfiguration conf,
            long numCompletedSegments, long segmentSize) throws Exception {
        BKDistributedLogManager dlm = (BKDistributedLogManager) manager;
        long txid = 1L;
        for (long i = 0; i < numCompletedSegments; i++) {
            BKSyncLogWriter writer = dlm.startLogSegmentNonPartitioned();
            for (long j = 1; j <= segmentSize; j++) {
                writer.write(DLMTestUtil.getLogRecordInstance(txid++));
            }
            writer.closeAndComplete();
        }
    }

    public static long generateLogSegmentNonPartitioned(DistributedLogManager dlm, int controlEntries,
            int userEntries, long startTxid) throws Exception {
        return generateLogSegmentNonPartitioned(dlm, controlEntries, userEntries, startTxid, 1L);
    }

    public static long generateLogSegmentNonPartitioned(DistributedLogManager dlm, int controlEntries,
            int userEntries, long startTxid, long txidStep) throws Exception {
        AsyncLogWriter out = dlm.startAsyncLogSegmentNonPartitioned();
        long txid = startTxid;
        for (int i = 0; i < controlEntries; ++i) {
            LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid);
            record.setControl();
            Await.result(out.write(record));
            txid += txidStep;
        }
        for (int i = 0; i < userEntries; ++i) {
            LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid);
            Await.result(out.write(record));
            txid += txidStep;
        }
        Utils.close(out);
        return txid - startTxid;
    }

    public static void injectLogSegmentWithGivenLogSegmentSeqNo(DistributedLogManager manager,
            DistributedLogConfiguration conf, long logSegmentSeqNo, long startTxID, boolean writeEntries,
            long segmentSize, boolean completeLogSegment) throws Exception {
        BKDistributedLogManager dlm = (BKDistributedLogManager) manager;
        BKLogWriteHandler writeHandler = dlm.createWriteHandler(false);
        FutureUtils.result(writeHandler.lockHandler());
        // Start a log segment with a given ledger seq number.
        BookKeeperClient bkc = dlm.getWriterBKC();
        LedgerHandle lh = bkc.get().createLedger(conf.getEnsembleSize(), conf.getWriteQuorumSize(),
                conf.getAckQuorumSize(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes());
        String inprogressZnodeName = writeHandler.inprogressZNodeName(lh.getId(), startTxID, logSegmentSeqNo);
        String znodePath = writeHandler.inprogressZNode(lh.getId(), startTxID, logSegmentSeqNo);
        LogSegmentMetadata l = new LogSegmentMetadata.LogSegmentMetadataBuilder(znodePath,
                conf.getDLLedgerMetadataLayoutVersion(), lh.getId(), startTxID)
                        .setLogSegmentSequenceNo(logSegmentSeqNo).build();
        l.write(dlm.writerZKC);
        writeHandler.maxTxId.store(startTxID);
        writeHandler.addLogSegmentToCache(inprogressZnodeName, l);
        BKLogSegmentWriter writer = new BKLogSegmentWriter(writeHandler.getFullyQualifiedName(),
                inprogressZnodeName, conf, conf.getDLLedgerMetadataLayoutVersion(), new BKLogSegmentEntryWriter(lh),
                writeHandler.lock, startTxID, logSegmentSeqNo, writeHandler.scheduler, writeHandler.statsLogger,
                writeHandler.statsLogger, writeHandler.alertStatsLogger, PermitLimiter.NULL_PERMIT_LIMITER,
                new SettableFeatureProvider("", 0), ConfUtils.getConstDynConf(conf));
        if (writeEntries) {
            long txid = startTxID;
            for (long j = 1; j <= segmentSize; j++) {
                writer.write(DLMTestUtil.getLogRecordInstance(txid++));
            }
            FutureUtils.result(writer.flushAndCommit());
        }
        if (completeLogSegment) {
            FutureUtils.result(writeHandler.completeAndCloseLogSegment(writer));
        }
        FutureUtils.result(writeHandler.unlockHandler());
    }

    public static void injectLogSegmentWithLastDLSN(DistributedLogManager manager, DistributedLogConfiguration conf,
            long logSegmentSeqNo, long startTxID, long segmentSize, boolean recordWrongLastDLSN) throws Exception {
        BKDistributedLogManager dlm = (BKDistributedLogManager) manager;
        BKLogWriteHandler writeHandler = dlm.createWriteHandler(false);
        FutureUtils.result(writeHandler.lockHandler());
        // Start a log segment with a given ledger seq number.
        BookKeeperClient bkc = dlm.getReaderBKC();
        LedgerHandle lh = bkc.get().createLedger(conf.getEnsembleSize(), conf.getWriteQuorumSize(),
                conf.getAckQuorumSize(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes());
        String inprogressZnodeName = writeHandler.inprogressZNodeName(lh.getId(), startTxID, logSegmentSeqNo);
        String znodePath = writeHandler.inprogressZNode(lh.getId(), startTxID, logSegmentSeqNo);
        LogSegmentMetadata l = new LogSegmentMetadata.LogSegmentMetadataBuilder(znodePath,
                conf.getDLLedgerMetadataLayoutVersion(), lh.getId(), startTxID)
                        .setLogSegmentSequenceNo(logSegmentSeqNo).setInprogress(false).build();
        l.write(dlm.writerZKC);
        writeHandler.maxTxId.store(startTxID);
        writeHandler.addLogSegmentToCache(inprogressZnodeName, l);
        BKLogSegmentWriter writer = new BKLogSegmentWriter(writeHandler.getFullyQualifiedName(),
                inprogressZnodeName, conf, conf.getDLLedgerMetadataLayoutVersion(), new BKLogSegmentEntryWriter(lh),
                writeHandler.lock, startTxID, logSegmentSeqNo, writeHandler.scheduler, writeHandler.statsLogger,
                writeHandler.statsLogger, writeHandler.alertStatsLogger, PermitLimiter.NULL_PERMIT_LIMITER,
                new SettableFeatureProvider("", 0), ConfUtils.getConstDynConf(conf));
        long txid = startTxID;
        DLSN wrongDLSN = null;
        for (long j = 1; j <= segmentSize; j++) {
            DLSN dlsn = Await.result(writer.asyncWrite(DLMTestUtil.getLogRecordInstance(txid++)));
            if (j == (segmentSize - 1)) {
                wrongDLSN = dlsn;
            }
        }
        assertNotNull(wrongDLSN);
        if (recordWrongLastDLSN) {
            FutureUtils.result(writer.asyncClose());
            writeHandler.completeAndCloseLogSegment(
                    writeHandler.inprogressZNodeName(writer.getLogSegmentId(), writer.getStartTxId(),
                            writer.getLogSegmentSequenceNumber()),
                    writer.getLogSegmentSequenceNumber(), writer.getLogSegmentId(), writer.getStartTxId(),
                    startTxID + segmentSize - 2, writer.getPositionWithinLogSegment() - 1, wrongDLSN.getEntryId(),
                    wrongDLSN.getSlotId());
        } else {
            FutureUtils.result(writeHandler.completeAndCloseLogSegment(writer));
        }
        FutureUtils.result(writeHandler.unlockHandler());
    }

    public static void updateSegmentMetadata(ZooKeeperClient zkc, LogSegmentMetadata segment) throws Exception {
        byte[] finalisedData = segment.getFinalisedData().getBytes(UTF_8);
        zkc.get().setData(segment.getZkPath(), finalisedData, -1);
    }

    public static ServerConfiguration loadTestBkConf() {
        ServerConfiguration conf = new ServerConfiguration();
        ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
        URL confUrl = classLoader.getResource("bk_server.conf");
        try {
            if (null != confUrl) {
                conf.loadConf(confUrl);
                LOG.info("loaded bk_server.conf from resources");
            }
        } catch (org.apache.commons.configuration.ConfigurationException ex) {
            LOG.warn("loading conf failed", ex);
        }
        conf.setAllowLoopback(true);
        return conf;
    }

    public static <T> void validateFutureFailed(Future<T> future, Class exClass) {
        try {
            Await.result(future);
        } catch (Exception ex) {
            LOG.info("Expected: {} Actual: {}", exClass.getName(), ex.getClass().getName());
            assertTrue("exceptions types equal", exClass.isInstance(ex));
        }
    }

    public static <T> T validateFutureSucceededAndGetResult(Future<T> future) throws Exception {
        try {
            return Await.result(future, Duration.fromSeconds(10));
        } catch (Exception ex) {
            fail("unexpected exception " + ex.getClass().getName());
            throw ex;
        }
    }
}