io.prestosql.plugin.kafka.util.EmbeddedKafka.java Source code

Introduction

Here is the source code for io.prestosql.plugin.kafka.util.EmbeddedKafka.java

Source

/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.prestosql.plugin.kafka.util;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.common.io.Files;
import kafka.admin.AdminUtils;
import kafka.admin.RackAwareMode;
import kafka.javaapi.producer.Producer;
import kafka.producer.ProducerConfig;
import kafka.server.KafkaConfig;
import kafka.server.KafkaServer;
import kafka.utils.ZKStringSerializer$;
import kafka.utils.ZkUtils;
import org.I0Itec.zkclient.ZkClient;
import org.I0Itec.zkclient.ZkConnection;
import org.apache.kafka.common.network.ListenerName;
import org.apache.kafka.common.protocol.SecurityProtocol;
import org.apache.kafka.common.utils.Time;
import scala.Option;
import scala.collection.immutable.List;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicBoolean;

import static com.google.common.base.Preconditions.checkState;
import static com.google.common.io.MoreFiles.deleteRecursively;
import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE;
import static io.prestosql.plugin.kafka.util.TestUtils.toProperties;
import static java.util.Objects.requireNonNull;

public class EmbeddedKafka implements Closeable {
    private final EmbeddedZookeeper zookeeper;
    private final File kafkaDataDir;
    private final KafkaServer kafkaServer;

    private final AtomicBoolean started = new AtomicBoolean();
    private final AtomicBoolean stopped = new AtomicBoolean();

    public static EmbeddedKafka createEmbeddedKafka() throws IOException {
        return new EmbeddedKafka(new EmbeddedZookeeper(), new Properties());
    }

    public static EmbeddedKafka createEmbeddedKafka(Properties overrideProperties) throws IOException {
        return new EmbeddedKafka(new EmbeddedZookeeper(), overrideProperties);
    }

    EmbeddedKafka(EmbeddedZookeeper zookeeper, Properties overrideProperties) {
        this.zookeeper = requireNonNull(zookeeper, "zookeeper is null");
        requireNonNull(overrideProperties, "overrideProperties is null");

        this.kafkaDataDir = Files.createTempDir();

        Map<String, String> properties = ImmutableMap.<String, String>builder().put("broker.id", "0")
                .put("host.name", "localhost").put("num.partitions", "2")
                .put("log.flush.interval.messages", "10000").put("log.flush.interval.ms", "1000")
                .put("log.retention.minutes", "60").put("log.segment.bytes", "1048576")
                .put("auto.create.topics.enable", "false").put("zookeeper.connection.timeout.ms", "1000000")
                .put("port", "0").put("log.dirs", kafkaDataDir.getAbsolutePath())
                .put("zookeeper.connect", zookeeper.getConnectString())
                .putAll(Maps.fromProperties(overrideProperties)).build();

        KafkaConfig config = new KafkaConfig(properties);
        this.kafkaServer = new KafkaServer(config, Time.SYSTEM, Option.empty(), List.make(0, null));
    }

    public void start() throws InterruptedException, IOException {
        if (!started.getAndSet(true)) {
            zookeeper.start();
            kafkaServer.startup();
        }
    }

    @Override
    public void close() throws IOException {
        if (started.get() && !stopped.getAndSet(true)) {
            kafkaServer.shutdown();
            kafkaServer.awaitShutdown();
            zookeeper.close();
            deleteRecursively(kafkaDataDir.toPath(), ALLOW_INSECURE);
        }
    }

    public void createTopics(String... topics) {
        createTopics(2, 1, new Properties(), topics);
    }

    public void createTopics(int partitions, int replication, Properties topicProperties, String... topics) {
        checkState(started.get() && !stopped.get(), "not started!");

        ZkConnection zkConnection = new ZkConnection(getZookeeperConnectString(), 30_000);
        ZkClient zkClient = new ZkClient(zkConnection, 30_000, ZKStringSerializer$.MODULE$);
        try {
            for (String topic : topics) {
                AdminUtils.createTopic(new ZkUtils(zkClient, zkConnection, false), topic, partitions, replication,
                        topicProperties, RackAwareMode.Disabled$.MODULE$);
            }
        } finally {
            zkClient.close();
        }
    }

    public CloseableProducer<Long, Object> createProducer() {
        Map<String, String> properties = ImmutableMap.<String, String>builder()
                .put("metadata.broker.list", getConnectString())
                .put("serializer.class", JsonEncoder.class.getName())
                .put("key.serializer.class", NumberEncoder.class.getName())
                .put("partitioner.class", NumberPartitioner.class.getName()).put("request.required.acks", "1")
                .build();

        ProducerConfig producerConfig = new ProducerConfig(toProperties(properties));
        return new CloseableProducer<>(producerConfig);
    }

    public static class CloseableProducer<K, V> extends Producer<K, V> implements AutoCloseable {
        public CloseableProducer(ProducerConfig config) {
            super(config);
        }
    }

    public String getConnectString() {
        return "localhost:" + kafkaServer.boundPort(ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT));
    }

    public String getZookeeperConnectString() {
        return zookeeper.getConnectString();
    }
}