Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.pulsar.io; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.gson.Gson; import lombok.ToString; import org.apache.bookkeeper.test.PortManager; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.ServiceConfigurationUtils; import org.apache.pulsar.broker.authentication.AuthenticationProviderBasic; import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; import org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl; import org.apache.pulsar.client.admin.BrokerStats; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.Utils; import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.FunctionStats; import org.apache.pulsar.common.policies.data.FunctionStatus; import org.apache.pulsar.common.policies.data.SubscriptionStats; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.functions.instance.InstanceUtils; import org.apache.pulsar.functions.utils.FunctionDetailsUtils; import org.apache.pulsar.functions.worker.FunctionRuntimeManager; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerService; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.lang.reflect.Method; import java.net.HttpURLConnection; import java.net.InetAddress; import java.net.URL; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; import static com.google.common.base.Preconditions.checkArgument; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically; import static org.apache.pulsar.functions.utils.functioncache.FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertTrue; /** * Test Pulsar sink on function * */ public class PulsarFunctionE2ETest { LocalBookkeeperEnsemble bkEnsemble; ServiceConfiguration config; WorkerConfig workerConfig; URL urlTls; PulsarService pulsar; PulsarAdmin admin; PulsarClient pulsarClient; BrokerStats brokerStatsClient; WorkerService functionsWorkerService; final String tenant = "external-repl-prop"; String pulsarFunctionsNamespace = tenant + "/use/pulsar-function-admin"; String primaryHost; String workerId; private final int ZOOKEEPER_PORT = PortManager.nextFreePort(); private final int brokerWebServicePort = PortManager.nextFreePort(); private final int brokerWebServiceTlsPort = PortManager.nextFreePort(); private final int brokerServicePort = PortManager.nextFreePort(); private final int brokerServiceTlsPort = PortManager.nextFreePort(); private final int workerServicePort = PortManager.nextFreePort(); private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem"; private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem"; private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; private static final Logger log = LoggerFactory.getLogger(PulsarFunctionE2ETest.class); @DataProvider(name = "validRoleName") public Object[][] validRoleName() { return new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } }; } @BeforeMethod void setup(Method method) throws Exception { log.info("--- Setting up method {} ---", method.getName()); // Start local bookkeeper ensemble bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, () -> PortManager.nextFreePort()); bkEnsemble.start(); String brokerServiceUrl = "https://127.0.0.1:" + brokerWebServiceTlsPort; config = spy(new ServiceConfiguration()); config.setClusterName("use"); Set<String> superUsers = Sets.newHashSet("superUser"); config.setSuperUserRoles(superUsers); config.setWebServicePort(brokerWebServicePort); config.setWebServicePortTls(brokerWebServiceTlsPort); config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT); config.setBrokerServicePort(brokerServicePort); config.setBrokerServicePortTls(brokerServiceTlsPort); config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config.setTlsAllowInsecureConnection(true); Set<String> providers = new HashSet<>(); providers.add(AuthenticationProviderTls.class.getName()); config.setAuthenticationEnabled(true); config.setAuthenticationProviders(providers); config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); config.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); config.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); config.setBrokerClientAuthenticationParameters( "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH); config.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); config.setBrokerClientTlsEnabled(true); functionsWorkerService = createPulsarFunctionWorker(config); urlTls = new URL(brokerServiceUrl); Optional<WorkerService> functionWorkerService = Optional.of(functionsWorkerService); pulsar = new PulsarService(config, functionWorkerService); pulsar.start(); Map<String, String> authParams = new HashMap<>(); authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); Authentication authTls = new AuthenticationTls(); authTls.configure(authParams); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl) .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true) .authentication(authTls).build()); brokerStatsClient = admin.brokerStats(); primaryHost = String.format("http://%s:%d", InetAddress.getLocalHost().getHostName(), brokerWebServicePort); // update cluster metadata ClusterData clusterData = new ClusterData(urlTls.toString()); admin.clusters().updateCluster(config.getClusterName(), clusterData); ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(this.workerConfig.getPulsarServiceUrl()); if (isNotBlank(workerConfig.getClientAuthenticationPlugin()) && isNotBlank(workerConfig.getClientAuthenticationParameters())) { clientBuilder.enableTls(workerConfig.isUseTls()); clientBuilder.allowTlsInsecureConnection(workerConfig.isTlsAllowInsecureConnection()); clientBuilder.authentication(workerConfig.getClientAuthenticationPlugin(), workerConfig.getClientAuthenticationParameters()); } pulsarClient = clientBuilder.build(); TenantInfo propAdmin = new TenantInfo(); propAdmin.getAdminRoles().add("superUser"); propAdmin.setAllowedClusters(Sets.newHashSet(Lists.newArrayList("use"))); admin.tenants().updateTenant(tenant, propAdmin); System.setProperty(JAVA_INSTANCE_JAR_PROPERTY, ""); Thread.sleep(100); } @AfterMethod void shutdown() throws Exception { log.info("--- Shutting down ---"); pulsarClient.close(); admin.close(); functionsWorkerService.stop(); pulsar.close(); bkEnsemble.stop(); } private WorkerService createPulsarFunctionWorker(ServiceConfiguration config) { workerConfig = new WorkerConfig(); workerConfig.setPulsarFunctionsNamespace(pulsarFunctionsNamespace); workerConfig.setSchedulerClassName( org.apache.pulsar.functions.worker.scheduler.RoundRobinScheduler.class.getName()); workerConfig.setThreadContainerFactory(new WorkerConfig.ThreadContainerFactory().setThreadGroupName("use")); // worker talks to local broker workerConfig.setPulsarServiceUrl("pulsar://127.0.0.1:" + config.getBrokerServicePortTls().get()); workerConfig.setPulsarWebServiceUrl("https://127.0.0.1:" + config.getWebServicePortTls().get()); workerConfig.setFailureCheckFreqMs(100); workerConfig.setNumFunctionPackageReplicas(1); workerConfig.setClusterCoordinationTopicName("coordinate"); workerConfig.setFunctionAssignmentTopicName("assignment"); workerConfig.setFunctionMetadataTopicName("metadata"); workerConfig.setInstanceLivenessCheckFreqMs(100); workerConfig.setWorkerPort(workerServicePort); workerConfig.setPulsarFunctionsCluster(config.getClusterName()); String hostname = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(config.getAdvertisedAddress()); this.workerId = "c-" + config.getClusterName() + "-fw-" + hostname + "-" + workerConfig.getWorkerPort(); workerConfig.setWorkerHostname(hostname); workerConfig.setWorkerId(workerId); workerConfig.setClientAuthenticationPlugin(AuthenticationTls.class.getName()); workerConfig.setClientAuthenticationParameters( String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH)); workerConfig.setUseTls(true); workerConfig.setTlsAllowInsecureConnection(true); workerConfig.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); workerConfig.setAuthenticationEnabled(true); workerConfig.setAuthorizationEnabled(true); return new WorkerService(workerConfig); } protected static FunctionConfig createFunctionConfig(String tenant, String namespace, String functionName, String sourceTopic, String sinkTopic, String subscriptionName) { String sourceTopicPattern = String.format("persistent://%s/%s/%s", tenant, namespace, sourceTopic); FunctionConfig functionConfig = new FunctionConfig(); functionConfig.setTenant(tenant); functionConfig.setNamespace(namespace); functionConfig.setName(functionName); functionConfig.setParallelism(1); functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE); functionConfig.setSubName(subscriptionName); functionConfig.setTopicsPattern(sourceTopicPattern); functionConfig.setAutoAck(true); functionConfig.setClassName("org.apache.pulsar.functions.api.examples.ExclamationFunction"); functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); functionConfig.setOutput(sinkTopic); functionConfig.setCleanupSubscription(true); return functionConfig; } private static SourceConfig createSourceConfig(String tenant, String namespace, String functionName, String sinkTopic) { SourceConfig sourceConfig = new SourceConfig(); sourceConfig.setTenant(tenant); sourceConfig.setNamespace(namespace); sourceConfig.setName(functionName); sourceConfig.setParallelism(1); sourceConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE); sourceConfig.setTopicName(sinkTopic); return sourceConfig; } private static SinkConfig createSinkConfig(String tenant, String namespace, String functionName, String sourceTopic, String subName) { SinkConfig sinkConfig = new SinkConfig(); sinkConfig.setTenant(tenant); sinkConfig.setNamespace(namespace); sinkConfig.setName(functionName); sinkConfig.setParallelism(1); sinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE); sinkConfig.setInputs(Collections.singleton(sourceTopic)); sinkConfig.setSourceSubscriptionName(subName); sinkConfig.setCleanupSubscription(true); return sinkConfig; } /** * Validates pulsar sink e2e functionality on functions. * * @throws Exception */ @Test(timeOut = 20000) public void testE2EPulsarFunction() throws Exception { final String namespacePortion = "io"; final String replNamespace = tenant + "/" + namespacePortion; final String sourceTopic = "persistent://" + replNamespace + "/my-topic1"; final String sinkTopic = "persistent://" + replNamespace + "/output"; final String propertyKey = "key"; final String propertyValue = "value"; final String functionName = "PulsarSink-test"; final String subscriptionName = "test-sub"; admin.namespaces().createNamespace(replNamespace); Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use")); admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters); // create a producer that creates a topic at broker Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create(); Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(sinkTopic).subscriptionName("sub") .subscribe(); String jarFilePathUrl = Utils.FILE + ":" + getClass().getClassLoader().getResource("pulsar-functions-api-examples.jar").getFile(); FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, "my.*", sinkTopic, subscriptionName); admin.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl); // try to update function to test: update-function functionality admin.functions().updateFunctionWithUrl(functionConfig, jarFilePathUrl); retryStrategically((test) -> { try { return admin.topics().getStats(sourceTopic).subscriptions.size() == 1; } catch (PulsarAdminException e) { return false; } }, 5, 150); // validate pulsar sink consumer has started on the topic assertEquals(admin.topics().getStats(sourceTopic).subscriptions.size(), 1); int totalMsgs = 5; for (int i = 0; i < totalMsgs; i++) { String data = "my-message-" + i; producer.newMessage().property(propertyKey, propertyValue).value(data).send(); } retryStrategically((test) -> { try { SubscriptionStats subStats = admin.topics().getStats(sourceTopic).subscriptions .get(subscriptionName); return subStats.unackedMessages == 0; } catch (PulsarAdminException e) { return false; } }, 5, 150); Message<String> msg = consumer.receive(5, TimeUnit.SECONDS); String receivedPropertyValue = msg.getProperty(propertyKey); assertEquals(propertyValue, receivedPropertyValue); // validate pulsar-sink consumer has consumed all messages and delivered to Pulsar sink but unacked messages // due to publish failure assertNotEquals( admin.topics().getStats(sourceTopic).subscriptions.values().iterator().next().unackedMessages, totalMsgs); // delete functions admin.functions().deleteFunction(tenant, namespacePortion, functionName); retryStrategically((test) -> { try { return admin.topics().getStats(sourceTopic).subscriptions.size() == 0; } catch (PulsarAdminException e) { return false; } }, 5, 150); // make sure subscriptions are cleanup assertEquals(admin.topics().getStats(sourceTopic).subscriptions.size(), 0); } @Test(timeOut = 20000) public void testPulsarSinkStats() throws Exception { final String namespacePortion = "io"; final String replNamespace = tenant + "/" + namespacePortion; final String sourceTopic = "persistent://" + replNamespace + "/input"; final String functionName = "PulsarSink-test"; final String propertyKey = "key"; final String propertyValue = "value"; final String subscriptionName = "test-sub"; admin.namespaces().createNamespace(replNamespace); Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use")); admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters); // create a producer that creates a topic at broker Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create(); String jarFilePathUrl = Utils.FILE + ":" + getClass().getClassLoader().getResource("pulsar-io-data-generator.nar").getFile(); SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, functionName, sourceTopic, subscriptionName); admin.sink().createSinkWithUrl(sinkConfig, jarFilePathUrl); admin.sink().updateSinkWithUrl(sinkConfig, jarFilePathUrl); retryStrategically((test) -> { try { return admin.topics().getStats(sourceTopic).subscriptions.size() == 1; } catch (PulsarAdminException e) { return false; } }, 5, 150); // validate pulsar sink consumer has started on the topic assertEquals(admin.topics().getStats(sourceTopic).subscriptions.size(), 1); // validate prometheus metrics empty String prometheusMetrics = getPrometheusMetrics(brokerWebServicePort); log.info("prometheus metrics: {}", prometheusMetrics); Map<String, Metric> metrics = parseMetrics(prometheusMetrics); Metric m = metrics.get("pulsar_sink_received_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_sink_received_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_sink_written_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_sink_written_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_sink_sink_exceptions_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_sink_sink_exceptions_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_sink_system_exceptions_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_sink_system_exceptions_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_sink_last_invocation"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); int totalMsgs = 10; for (int i = 0; i < totalMsgs; i++) { String data = "my-message-" + i; producer.newMessage().property(propertyKey, propertyValue).value(data).send(); } retryStrategically((test) -> { try { SubscriptionStats subStats = admin.topics().getStats(sourceTopic).subscriptions .get(subscriptionName); return subStats.unackedMessages == 0 && subStats.msgThroughputOut == totalMsgs; } catch (PulsarAdminException e) { return false; } }, 5, 200); // get stats after producing prometheusMetrics = getPrometheusMetrics(brokerWebServicePort); log.info("prometheusMetrics: {}", prometheusMetrics); metrics = parseMetrics(prometheusMetrics); m = metrics.get("pulsar_sink_received_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, (double) totalMsgs); m = metrics.get("pulsar_sink_received_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, (double) totalMsgs); m = metrics.get("pulsar_sink_written_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, (double) totalMsgs); m = metrics.get("pulsar_sink_written_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, (double) totalMsgs); m = metrics.get("pulsar_sink_sink_exceptions_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_sink_sink_exceptions_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_sink_system_exceptions_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_sink_system_exceptions_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_sink_last_invocation"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertTrue(m.value > 0.0); // delete functions admin.sink().deleteSink(tenant, namespacePortion, functionName); retryStrategically((test) -> { try { return admin.topics().getStats(sourceTopic).subscriptions.size() == 0; } catch (PulsarAdminException e) { return false; } }, 5, 150); // make sure subscriptions are cleanup assertEquals(admin.topics().getStats(sourceTopic).subscriptions.size(), 0); } @Test(timeOut = 20000) public void testPulsarSourceStats() throws Exception { final String namespacePortion = "io"; final String replNamespace = tenant + "/" + namespacePortion; final String sinkTopic = "persistent://" + replNamespace + "/output"; final String functionName = "PulsarSource-test"; admin.namespaces().createNamespace(replNamespace); Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use")); admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters); String jarFilePathUrl = Utils.FILE + ":" + getClass().getClassLoader().getResource("pulsar-io-data-generator.nar").getFile(); SourceConfig sourceConfig = createSourceConfig(tenant, namespacePortion, functionName, sinkTopic); admin.source().createSourceWithUrl(sourceConfig, jarFilePathUrl); admin.source().updateSourceWithUrl(sourceConfig, jarFilePathUrl); retryStrategically((test) -> { try { return (admin.topics().getStats(sinkTopic).publishers.size() == 1) && (admin.topics().getInternalStats(sinkTopic).numberOfEntries > 4); } catch (PulsarAdminException e) { return false; } }, 10, 150); assertEquals(admin.topics().getStats(sinkTopic).publishers.size(), 1); String prometheusMetrics = getPrometheusMetrics(brokerWebServicePort); log.info("prometheusMetrics: {}", prometheusMetrics); Map<String, Metric> metrics = parseMetrics(prometheusMetrics); Metric m = metrics.get("pulsar_source_received_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertTrue(m.value > 0.0); m = metrics.get("pulsar_source_received_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertTrue(m.value > 0.0); m = metrics.get("pulsar_source_written_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertTrue(m.value > 0.0); m = metrics.get("pulsar_source_written_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertTrue(m.value > 0.0); m = metrics.get("pulsar_source_source_exceptions_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_source_source_exceptions_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_source_system_exceptions_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_source_system_exceptions_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_source_last_invocation"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertTrue(m.value > 0.0); } @Test(timeOut = 20000) public void testPulsarFunctionStats() throws Exception { final String namespacePortion = "io"; final String replNamespace = tenant + "/" + namespacePortion; final String sourceTopic = "persistent://" + replNamespace + "/my-topic1"; final String sinkTopic = "persistent://" + replNamespace + "/output"; final String propertyKey = "key"; final String propertyValue = "value"; final String functionName = "PulsarSink-test"; final String subscriptionName = "test-sub"; admin.namespaces().createNamespace(replNamespace); Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use")); admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters); // create a producer that creates a topic at broker Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create(); String jarFilePathUrl = Utils.FILE + ":" + getClass().getClassLoader().getResource("pulsar-functions-api-examples.jar").getFile(); FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, "my.*", sinkTopic, subscriptionName); admin.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl); // try to update function to test: update-function functionality admin.functions().updateFunctionWithUrl(functionConfig, jarFilePathUrl); retryStrategically((test) -> { try { return admin.topics().getStats(sourceTopic).subscriptions.size() == 1; } catch (PulsarAdminException e) { return false; } }, 5, 150); // validate pulsar sink consumer has started on the topic assertEquals(admin.topics().getStats(sourceTopic).subscriptions.size(), 1); // validate stats are empty FunctionRuntimeManager functionRuntimeManager = functionsWorkerService.getFunctionRuntimeManager(); FunctionStats functionStats = functionRuntimeManager.getFunctionStats(tenant, namespacePortion, functionName, null); FunctionStats functionStatsFromAdmin = admin.functions().getFunctionStats(tenant, namespacePortion, functionName); assertEquals(functionStats, functionStatsFromAdmin); assertEquals(functionStats.getReceivedTotal(), 0); assertEquals(functionStats.getProcessedSuccessfullyTotal(), 0); assertEquals(functionStats.getSystemExceptionsTotal(), 0); assertEquals(functionStats.getUserExceptionsTotal(), 0); assertEquals(functionStats.avgProcessLatency, null); assertEquals(functionStats.oneMin.getReceivedTotal(), 0); assertEquals(functionStats.oneMin.getProcessedSuccessfullyTotal(), 0); assertEquals(functionStats.oneMin.getSystemExceptionsTotal(), 0); assertEquals(functionStats.oneMin.getUserExceptionsTotal(), 0); assertEquals(functionStats.oneMin.getAvgProcessLatency(), null); assertEquals(functionStats.getAvgProcessLatency(), functionStats.oneMin.getAvgProcessLatency()); assertEquals(functionStats.getLastInvocation(), null); assertEquals(functionStats.instances.size(), 1); assertEquals(functionStats.instances.get(0).getInstanceId(), 0); assertEquals(functionStats.instances.get(0).getMetrics().getReceivedTotal(), 0); assertEquals(functionStats.instances.get(0).getMetrics().getProcessedSuccessfullyTotal(), 0); assertEquals(functionStats.instances.get(0).getMetrics().getSystemExceptionsTotal(), 0); assertEquals(functionStats.instances.get(0).getMetrics().getUserExceptionsTotal(), 0); assertEquals(functionStats.instances.get(0).getMetrics().avgProcessLatency, null); assertEquals(functionStats.instances.get(0).getMetrics().oneMin.getReceivedTotal(), 0); assertEquals(functionStats.instances.get(0).getMetrics().oneMin.getProcessedSuccessfullyTotal(), 0); assertEquals(functionStats.instances.get(0).getMetrics().oneMin.getSystemExceptionsTotal(), 0); assertEquals(functionStats.instances.get(0).getMetrics().oneMin.getUserExceptionsTotal(), 0); assertEquals(functionStats.instances.get(0).getMetrics().oneMin.getAvgProcessLatency(), null); assertEquals(functionStats.instances.get(0).getMetrics().getAvgProcessLatency(), functionStats.instances.get(0).getMetrics().oneMin.getAvgProcessLatency()); assertEquals(functionStats.instances.get(0).getMetrics().getAvgProcessLatency(), functionStats.getAvgProcessLatency()); // validate prometheus metrics empty String prometheusMetrics = getPrometheusMetrics(brokerWebServicePort); log.info("prometheus metrics: {}", prometheusMetrics); Map<String, Metric> metrics = parseMetrics(prometheusMetrics); Metric m = metrics.get("pulsar_function_received_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_function_received_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_function_user_exceptions_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_function_user_exceptions_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_function_process_latency_ms"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, Double.NaN); m = metrics.get("pulsar_function_process_latency_ms_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, Double.NaN); m = metrics.get("pulsar_function_system_exceptions_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_function_system_exceptions_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_function_last_invocation"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_function_processed_successfully_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_function_processed_successfully_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); // validate function instance stats empty FunctionStats.FunctionInstanceStats.FunctionInstanceStatsData functionInstanceStats = functionRuntimeManager .getFunctionInstanceStats(tenant, namespacePortion, functionName, 0, null); FunctionStats.FunctionInstanceStats.FunctionInstanceStatsData functionInstanceStatsAdmin = admin.functions() .getFunctionStats(tenant, namespacePortion, functionName, 0); assertEquals(functionInstanceStats, functionInstanceStatsAdmin); assertEquals(functionInstanceStats, functionStats.instances.get(0).getMetrics()); int totalMsgs = 10; for (int i = 0; i < totalMsgs; i++) { String data = "my-message-" + i; producer.newMessage().property(propertyKey, propertyValue).value(data).send(); } retryStrategically((test) -> { try { SubscriptionStats subStats = admin.topics().getStats(sourceTopic).subscriptions .get(subscriptionName); return subStats.unackedMessages == 0 && subStats.msgThroughputOut == totalMsgs; } catch (PulsarAdminException e) { return false; } }, 5, 200); // get stats after producing functionStats = functionRuntimeManager.getFunctionStats(tenant, namespacePortion, functionName, null); functionStatsFromAdmin = admin.functions().getFunctionStats(tenant, namespacePortion, functionName); assertEquals(functionStats, functionStatsFromAdmin); assertEquals(functionStats.getReceivedTotal(), totalMsgs); assertEquals(functionStats.getProcessedSuccessfullyTotal(), totalMsgs); assertEquals(functionStats.getSystemExceptionsTotal(), 0); assertEquals(functionStats.getUserExceptionsTotal(), 0); assertTrue(functionStats.avgProcessLatency > 0); assertEquals(functionStats.oneMin.getReceivedTotal(), totalMsgs); assertEquals(functionStats.oneMin.getProcessedSuccessfullyTotal(), totalMsgs); assertEquals(functionStats.oneMin.getSystemExceptionsTotal(), 0); assertEquals(functionStats.oneMin.getUserExceptionsTotal(), 0); assertTrue(functionStats.oneMin.getAvgProcessLatency() > 0); assertEquals(functionStats.getAvgProcessLatency(), functionStats.oneMin.getAvgProcessLatency()); assertTrue(functionStats.getLastInvocation() > 0); assertEquals(functionStats.instances.size(), 1); assertEquals(functionStats.instances.get(0).getInstanceId(), 0); assertEquals(functionStats.instances.get(0).getMetrics().getReceivedTotal(), totalMsgs); assertEquals(functionStats.instances.get(0).getMetrics().getProcessedSuccessfullyTotal(), totalMsgs); assertEquals(functionStats.instances.get(0).getMetrics().getSystemExceptionsTotal(), 0); assertEquals(functionStats.instances.get(0).getMetrics().getUserExceptionsTotal(), 0); assertTrue(functionStats.instances.get(0).getMetrics().avgProcessLatency > 0); assertEquals(functionStats.instances.get(0).getMetrics().oneMin.getReceivedTotal(), totalMsgs); assertEquals(functionStats.instances.get(0).getMetrics().oneMin.getProcessedSuccessfullyTotal(), totalMsgs); assertEquals(functionStats.instances.get(0).getMetrics().oneMin.getSystemExceptionsTotal(), 0); assertEquals(functionStats.instances.get(0).getMetrics().oneMin.getUserExceptionsTotal(), 0); assertTrue(functionStats.instances.get(0).getMetrics().oneMin.getAvgProcessLatency() > 0); assertEquals(functionStats.instances.get(0).getMetrics().getAvgProcessLatency(), functionStats.instances.get(0).getMetrics().oneMin.getAvgProcessLatency()); assertEquals(functionStats.instances.get(0).getMetrics().getAvgProcessLatency(), functionStats.getAvgProcessLatency()); // validate function instance stats functionInstanceStats = functionRuntimeManager.getFunctionInstanceStats(tenant, namespacePortion, functionName, 0, null); functionInstanceStatsAdmin = admin.functions().getFunctionStats(tenant, namespacePortion, functionName, 0); assertEquals(functionInstanceStats, functionInstanceStatsAdmin); assertEquals(functionInstanceStats, functionStats.instances.get(0).getMetrics()); // validate prometheus metrics prometheusMetrics = getPrometheusMetrics(brokerWebServicePort); log.info("prometheus metrics: {}", prometheusMetrics); metrics = parseMetrics(prometheusMetrics); m = metrics.get("pulsar_function_received_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, (double) totalMsgs); m = metrics.get("pulsar_function_received_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, (double) totalMsgs); m = metrics.get("pulsar_function_user_exceptions_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_function_user_exceptions_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_function_process_latency_ms"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertTrue(m.value > 0.0); m = metrics.get("pulsar_function_process_latency_ms_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertTrue(m.value > 0.0); m = metrics.get("pulsar_function_system_exceptions_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_function_system_exceptions_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, 0.0); m = metrics.get("pulsar_function_last_invocation"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertTrue(m.value > 0.0); m = metrics.get("pulsar_function_processed_successfully_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, (double) totalMsgs); m = metrics.get("pulsar_function_processed_successfully_total_1min"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); assertEquals(m.tags.get("namespace"), String.format("%s/%s", tenant, namespacePortion)); assertEquals(m.tags.get("fqfn"), FunctionDetailsUtils.getFullyQualifiedName(tenant, namespacePortion, functionName)); assertEquals(m.value, (double) totalMsgs); // delete functions admin.functions().deleteFunction(tenant, namespacePortion, functionName); retryStrategically((test) -> { try { return admin.topics().getStats(sourceTopic).subscriptions.size() == 0; } catch (PulsarAdminException e) { return false; } }, 5, 150); // make sure subscriptions are cleanup assertEquals(admin.topics().getStats(sourceTopic).subscriptions.size(), 0); } @Test(timeOut = 20000) public void testPulsarFunctionStatus() throws Exception { final String namespacePortion = "io"; final String replNamespace = tenant + "/" + namespacePortion; final String sourceTopic = "persistent://" + replNamespace + "/my-topic1"; final String sinkTopic = "persistent://" + replNamespace + "/output"; final String propertyKey = "key"; final String propertyValue = "value"; final String functionName = "PulsarSink-test"; final String subscriptionName = "test-sub"; admin.namespaces().createNamespace(replNamespace); Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use")); admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters); // create a producer that creates a topic at broker Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create(); String jarFilePathUrl = Utils.FILE + ":" + getClass().getClassLoader().getResource("pulsar-functions-api-examples.jar").getFile(); FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, "my.*", sinkTopic, subscriptionName); admin.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl); // try to update function to test: update-function functionality admin.functions().updateFunctionWithUrl(functionConfig, jarFilePathUrl); retryStrategically((test) -> { try { return admin.topics().getStats(sourceTopic).subscriptions.size() == 1; } catch (PulsarAdminException e) { return false; } }, 5, 150); // validate pulsar sink consumer has started on the topic assertEquals(admin.topics().getStats(sourceTopic).subscriptions.size(), 1); int totalMsgs = 10; for (int i = 0; i < totalMsgs; i++) { String data = "my-message-" + i; producer.newMessage().property(propertyKey, propertyValue).value(data).send(); } retryStrategically((test) -> { try { SubscriptionStats subStats = admin.topics().getStats(sourceTopic).subscriptions .get(subscriptionName); return subStats.unackedMessages == 0 && subStats.msgThroughputOut == totalMsgs; } catch (PulsarAdminException e) { return false; } }, 5, 200); FunctionStatus functionStatus = admin.functions().getFunctionStatus(tenant, namespacePortion, functionName); int numInstances = functionStatus.getNumInstances(); assertEquals(numInstances, 1); FunctionStatus.FunctionInstanceStatus.FunctionInstanceStatusData status = functionStatus.getInstances() .get(0).getStatus(); double count = status.getNumReceived(); double success = status.getNumSuccessfullyProcessed(); String ownerWorkerId = status.getWorkerId(); assertEquals((int) count, totalMsgs); assertEquals((int) success, totalMsgs); assertEquals(ownerWorkerId, workerId); // delete functions admin.functions().deleteFunction(tenant, namespacePortion, functionName); retryStrategically((test) -> { try { return admin.topics().getStats(sourceTopic).subscriptions.size() == 0; } catch (PulsarAdminException e) { return false; } }, 5, 150); // make sure subscriptions are cleanup assertEquals(admin.topics().getStats(sourceTopic).subscriptions.size(), 0); } @Test(dataProvider = "validRoleName") public void testAuthorization(boolean validRoleName) throws Exception { final String namespacePortion = "io"; final String replNamespace = tenant + "/" + namespacePortion; final String sinkTopic = "persistent://" + replNamespace + "/output"; final String functionName = "PulsarSink-test"; final String subscriptionName = "test-sub"; admin.namespaces().createNamespace(replNamespace); Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use")); admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters); String roleName = validRoleName ? "superUser" : "invalid"; TenantInfo propAdmin = new TenantInfo(); propAdmin.getAdminRoles().add(roleName); propAdmin.setAllowedClusters(Sets.newHashSet(Lists.newArrayList("use"))); admin.tenants().updateTenant(tenant, propAdmin); String jarFilePathUrl = Utils.FILE + ":" + getClass().getClassLoader().getResource("pulsar-functions-api-examples.jar").getFile(); FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, "my.*", sinkTopic, subscriptionName); try { admin.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl); assertTrue(validRoleName); } catch (org.apache.pulsar.client.admin.PulsarAdminException.NotAuthorizedException ne) { assertFalse(validRoleName); } } @Test(timeOut = 20000) public void testFunctionStopAndRestartApi() throws Exception { final String namespacePortion = "io"; final String replNamespace = tenant + "/" + namespacePortion; final String sourceTopicName = "restartFunction"; final String sourceTopic = "persistent://" + replNamespace + "/" + sourceTopicName; final String sinkTopic = "persistent://" + replNamespace + "/output"; final String functionName = "PulsarSink-test"; final String subscriptionName = "test-sub"; admin.namespaces().createNamespace(replNamespace); Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use")); admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters); // create source topic Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create(); String jarFilePathUrl = Utils.FILE + ":" + getClass().getClassLoader().getResource("pulsar-functions-api-examples.jar").getFile(); FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, sourceTopicName, sinkTopic, subscriptionName); admin.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl); retryStrategically((test) -> { try { SubscriptionStats subStats = admin.topics().getStats(sourceTopic).subscriptions .get(subscriptionName); return subStats != null && subStats.consumers.size() == 1; } catch (PulsarAdminException e) { return false; } }, 5, 150); SubscriptionStats subStats = admin.topics().getStats(sourceTopic).subscriptions.get(subscriptionName); assertEquals(subStats.consumers.size(), 1); // it should stop consumer : so, check none of the consumer connected on subscription admin.functions().stopFunction(tenant, namespacePortion, functionName); retryStrategically((test) -> { try { SubscriptionStats subStat = admin.topics().getStats(sourceTopic).subscriptions .get(subscriptionName); return subStat != null && subStat.consumers.size() == 0; } catch (PulsarAdminException e) { return false; } }, 5, 150); subStats = admin.topics().getStats(sourceTopic).subscriptions.get(subscriptionName); assertEquals(subStats.consumers.size(), 0); // it should restart consumer : so, check if consumer came up again after restarting function admin.functions().restartFunction(tenant, namespacePortion, functionName); retryStrategically((test) -> { try { SubscriptionStats subStat = admin.topics().getStats(sourceTopic).subscriptions .get(subscriptionName); return subStat != null && subStat.consumers.size() == 1; } catch (PulsarAdminException e) { return false; } }, 5, 150); subStats = admin.topics().getStats(sourceTopic).subscriptions.get(subscriptionName); assertEquals(subStats.consumers.size(), 1); producer.close(); } @Test(timeOut = 20000) public void testFunctionAutomaticSubCleanup() throws Exception { final String namespacePortion = "io"; final String replNamespace = tenant + "/" + namespacePortion; final String sourceTopic = "persistent://" + replNamespace + "/my-topic1"; final String sinkTopic = "persistent://" + replNamespace + "/output"; final String propertyKey = "key"; final String propertyValue = "value"; final String functionName = "PulsarFunction-test"; admin.namespaces().createNamespace(replNamespace); Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use")); admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters); // create a producer that creates a topic at broker Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create(); String jarFilePathUrl = Utils.FILE + ":" + getClass().getClassLoader().getResource("pulsar-functions-api-examples.jar").getFile(); FunctionConfig functionConfig = new FunctionConfig(); functionConfig.setTenant(tenant); functionConfig.setNamespace(namespacePortion); functionConfig.setName(functionName); functionConfig.setParallelism(1); functionConfig.setInputs(Collections.singleton(sourceTopic)); functionConfig.setClassName("org.apache.pulsar.functions.api.examples.ExclamationFunction"); functionConfig.setOutput(sinkTopic); functionConfig.setCleanupSubscription(true); functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); admin.functions().createFunctionWithUrl(functionConfig, jarFilePathUrl); retryStrategically((test) -> { try { return admin.topics().getStats(sourceTopic).subscriptions.size() == 1; } catch (PulsarAdminException e) { return false; } }, 5, 150); // validate pulsar source consumer has started on the topic assertEquals(admin.topics().getStats(sourceTopic).subscriptions.size(), 1); int totalMsgs = 10; for (int i = 0; i < totalMsgs; i++) { String data = "my-message-" + i; producer.newMessage().property(propertyKey, propertyValue).value(data).send(); } retryStrategically((test) -> { try { SubscriptionStats subStats = admin.topics().getStats(sourceTopic).subscriptions .get(InstanceUtils.getDefaultSubscriptionName(tenant, namespacePortion, functionName)); return subStats.unackedMessages == 0 && subStats.msgThroughputOut == totalMsgs; } catch (PulsarAdminException e) { return false; } }, 5, 200); FunctionStatus functionStatus = admin.functions().getFunctionStatus(tenant, namespacePortion, functionName); int numInstances = functionStatus.getNumInstances(); assertEquals(numInstances, 1); FunctionStatus.FunctionInstanceStatus.FunctionInstanceStatusData status = functionStatus.getInstances() .get(0).getStatus(); double count = status.getNumReceived(); double success = status.getNumSuccessfullyProcessed(); String ownerWorkerId = status.getWorkerId(); assertEquals((int) count, totalMsgs); assertEquals((int) success, totalMsgs); assertEquals(ownerWorkerId, workerId); // delete functions admin.functions().deleteFunction(tenant, namespacePortion, functionName); retryStrategically((test) -> { try { return admin.topics().getStats(sourceTopic).subscriptions.size() == 0; } catch (PulsarAdminException e) { return false; } }, 5, 150); // make sure subscriptions are cleanup assertEquals(admin.topics().getStats(sourceTopic).subscriptions.size(), 0); } public static String getPrometheusMetrics(int metricsPort) throws IOException { StringBuilder result = new StringBuilder(); URL url = new URL( String.format("http://%s:%s/metrics", InetAddress.getLocalHost().getHostAddress(), metricsPort)); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("GET"); BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream())); String line; while ((line = rd.readLine()) != null) { result.append(line + System.lineSeparator()); } rd.close(); return result.toString(); } /** * Hacky parsing of Prometheus text format. Sould be good enough for unit tests */ private static Map<String, Metric> parseMetrics(String metrics) { Map<String, Metric> parsed = new HashMap<>(); // Example of lines are // jvm_threads_current{cluster="standalone",} 203.0 // or // pulsar_subscriptions_count{cluster="standalone", namespace="sample/standalone/ns1", // topic="persistent://sample/standalone/ns1/test-2"} 0.0 1517945780897 Pattern pattern = Pattern.compile("^(\\w+)\\{([^\\}]+)\\}\\s(-?[\\d\\w\\.]+)(\\s(\\d+))?$"); Pattern tagsPattern = Pattern.compile("(\\w+)=\"([^\"]+)\"(,\\s?)?"); Arrays.asList(metrics.split("\n")).forEach(line -> { if (line.isEmpty() || line.startsWith("#")) { return; } Matcher matcher = pattern.matcher(line); checkArgument(matcher.matches()); String name = matcher.group(1); Metric m = new Metric(); m.value = Double.valueOf(matcher.group(3)); String tags = matcher.group(2); Matcher tagsMatcher = tagsPattern.matcher(tags); while (tagsMatcher.find()) { String tag = tagsMatcher.group(1); String value = tagsMatcher.group(2); m.tags.put(tag, value); } parsed.put(name, m); }); return parsed; } @ToString static class Metric { Map<String, String> tags = new TreeMap<>(); double value; } }