Java tutorial
/* * Copyright 2013 Jive Software, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package com.jivesoftware.os.miru.writer.deployable; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.guava.GuavaModule; import com.google.common.collect.Maps; import com.jivesoftware.os.amza.api.wal.WALKey; import com.jivesoftware.os.amza.embed.EmbedAmzaServiceInitializer.Lifecycle; import com.jivesoftware.os.amza.service.EmbeddedClientProvider; import com.jivesoftware.os.miru.amza.MiruAmzaServiceConfig; import com.jivesoftware.os.miru.amza.MiruAmzaServiceInitializer; import com.jivesoftware.os.miru.api.MiruStats; import com.jivesoftware.os.miru.api.base.MiruTenantId; import com.jivesoftware.os.miru.api.topology.MiruClusterClient; import com.jivesoftware.os.miru.api.wal.AmzaCursor; import com.jivesoftware.os.miru.api.wal.AmzaSipCursor; import com.jivesoftware.os.miru.api.wal.MiruWALClient; import com.jivesoftware.os.miru.api.wal.MiruWALConfig; import com.jivesoftware.os.miru.api.wal.RCVSCursor; import com.jivesoftware.os.miru.api.wal.RCVSSipCursor; import com.jivesoftware.os.miru.cluster.client.MiruClusterClientInitializer; import com.jivesoftware.os.miru.logappender.MiruLogAppenderInitializer; import com.jivesoftware.os.miru.logappender.MiruLogAppenderInitializer.MiruLogAppenderConfig; import com.jivesoftware.os.miru.metric.sampler.MiruMetricSamplerInitializer; import com.jivesoftware.os.miru.metric.sampler.MiruMetricSamplerInitializer.MiruMetricSamplerConfig; import com.jivesoftware.os.miru.ui.MiruSoyRenderer; import com.jivesoftware.os.miru.ui.MiruSoyRendererInitializer; import com.jivesoftware.os.miru.ui.MiruSoyRendererInitializer.MiruSoyRendererConfig; import com.jivesoftware.os.miru.wal.client.MiruWALClientInitializer; import com.jivesoftware.os.miru.wal.client.MiruWALClientInitializer.WALClientSickThreadsHealthCheckConfig; import com.jivesoftware.os.miru.writer.deployable.base.MiruActivityIngress; import com.jivesoftware.os.miru.writer.deployable.endpoints.MiruIngressEndpoints; import com.jivesoftware.os.miru.writer.partition.AmzaPartitionIdProvider; import com.jivesoftware.os.routing.bird.deployable.Deployable; import com.jivesoftware.os.routing.bird.deployable.DeployableHealthCheckRegistry; import com.jivesoftware.os.routing.bird.deployable.ErrorHealthCheckConfig; import com.jivesoftware.os.routing.bird.deployable.InstanceConfig; import com.jivesoftware.os.routing.bird.endpoints.base.FullyOnlineVersion; import com.jivesoftware.os.routing.bird.endpoints.base.HasUI; import com.jivesoftware.os.routing.bird.endpoints.base.HasUI.UI; import com.jivesoftware.os.routing.bird.endpoints.base.LoadBalancerHealthCheckEndpoints; import com.jivesoftware.os.routing.bird.health.api.HealthFactory; import com.jivesoftware.os.routing.bird.health.checkers.FileDescriptorCountHealthChecker; import com.jivesoftware.os.routing.bird.health.checkers.GCLoadHealthChecker; import com.jivesoftware.os.routing.bird.health.checkers.GCPauseHealthChecker; import com.jivesoftware.os.routing.bird.health.checkers.LoadAverageHealthChecker; import com.jivesoftware.os.routing.bird.health.checkers.ServiceStartupHealthCheck; import com.jivesoftware.os.routing.bird.health.checkers.SickThreads; import com.jivesoftware.os.routing.bird.health.checkers.SickThreadsHealthCheck; import com.jivesoftware.os.routing.bird.health.checkers.SystemCpuHealthChecker; import com.jivesoftware.os.routing.bird.http.client.HttpDeliveryClientHealthProvider; import com.jivesoftware.os.routing.bird.http.client.HttpRequestHelperUtils; import com.jivesoftware.os.routing.bird.http.client.TenantAwareHttpClient; import com.jivesoftware.os.routing.bird.http.client.TenantRoutingHttpClientInitializer; import com.jivesoftware.os.routing.bird.server.util.Resource; import com.jivesoftware.os.routing.bird.shared.TenantRoutingProvider; import java.io.File; import java.util.Arrays; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicReference; import org.merlin.config.defaults.LongDefault; import org.merlin.config.defaults.StringDefault; public class MiruWriterMain { public static void main(String[] args) throws Exception { new MiruWriterMain().run(args); } public interface WriterAmzaServiceConfig extends MiruAmzaServiceConfig { @StringDefault("./var/amza/writer/data/") @Override String getWorkingDirectories(); @LongDefault(60_000L) long getReplicateLatestPartitionTimeoutMillis(); @LongDefault(60_000L) long getReplicateCursorTimeoutMillis(); } void run(String[] args) throws Exception { ServiceStartupHealthCheck serviceStartupHealthCheck = new ServiceStartupHealthCheck(); try { final Deployable deployable = new Deployable(args); InstanceConfig instanceConfig = deployable.config(InstanceConfig.class); HealthFactory.initialize(deployable::config, new DeployableHealthCheckRegistry(deployable)); deployable.addManageInjectables(HasUI.class, new HasUI(Arrays .asList(new UI("Miru-Writer", "main", "/ui"), new UI("Miru-Writer-Amza", "main", "/amza/ui")))); deployable.addHealthCheck(new GCPauseHealthChecker( deployable.config(GCPauseHealthChecker.GCPauseHealthCheckerConfig.class))); deployable.addHealthCheck(new GCLoadHealthChecker( deployable.config(GCLoadHealthChecker.GCLoadHealthCheckerConfig.class))); deployable.addHealthCheck(new SystemCpuHealthChecker( deployable.config(SystemCpuHealthChecker.SystemCpuHealthCheckerConfig.class))); deployable.addHealthCheck(new LoadAverageHealthChecker( deployable.config(LoadAverageHealthChecker.LoadAverageHealthCheckerConfig.class))); deployable.addHealthCheck(new FileDescriptorCountHealthChecker(deployable .config(FileDescriptorCountHealthChecker.FileDescriptorCountHealthCheckerConfig.class))); deployable.addHealthCheck(serviceStartupHealthCheck); deployable.addErrorHealthChecks(deployable.config(ErrorHealthCheckConfig.class)); AtomicReference<Callable<Boolean>> isAmzaReady = new AtomicReference<>(() -> false); deployable.addManageInjectables(FullyOnlineVersion.class, (FullyOnlineVersion) () -> { if (serviceStartupHealthCheck.startupHasSucceeded() && isAmzaReady.get().call()) { return instanceConfig.getVersion(); } else { return null; } }); deployable.buildManageServer().start(); ObjectMapper mapper = new ObjectMapper(); mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); mapper.registerModule(new GuavaModule()); TenantRoutingProvider tenantRoutingProvider = deployable.getTenantRoutingProvider(); TenantRoutingHttpClientInitializer<String> tenantRoutingHttpClientInitializer = deployable .getTenantRoutingHttpClientInitializer(); HttpDeliveryClientHealthProvider clientHealthProvider = new HttpDeliveryClientHealthProvider( instanceConfig.getInstanceKey(), HttpRequestHelperUtils.buildRequestHelper(false, false, null, instanceConfig.getRoutesHost(), instanceConfig.getRoutesPort()), instanceConfig.getConnectionsHealth(), 5_000, 100); MiruLogAppenderConfig miruLogAppenderConfig = deployable.config(MiruLogAppenderConfig.class); @SuppressWarnings("unchecked") TenantAwareHttpClient<String> miruStumptownClient = tenantRoutingHttpClientInitializer .builder(tenantRoutingProvider.getConnections("miru-stumptown", "main", 10_000), clientHealthProvider) .deadAfterNErrors(10).checkDeadEveryNMillis(10_000).build(); new MiruLogAppenderInitializer().initialize(instanceConfig.getDatacenter(), instanceConfig.getClusterName(), instanceConfig.getHost(), instanceConfig.getServiceName(), String.valueOf(instanceConfig.getInstanceName()), instanceConfig.getVersion(), miruLogAppenderConfig, miruStumptownClient).install(); MiruMetricSamplerConfig metricSamplerConfig = deployable.config(MiruMetricSamplerConfig.class); @SuppressWarnings("unchecked") TenantAwareHttpClient<String> miruAnomalyClient = tenantRoutingHttpClientInitializer .builder(tenantRoutingProvider.getConnections("miru-anomaly", "main", 10_000), clientHealthProvider) .deadAfterNErrors(10).checkDeadEveryNMillis(10_000).build(); new MiruMetricSamplerInitializer().initialize(instanceConfig.getDatacenter(), instanceConfig.getClusterName(), instanceConfig.getHost(), instanceConfig.getServiceName(), String.valueOf(instanceConfig.getInstanceName()), instanceConfig.getVersion(), metricSamplerConfig, miruAnomalyClient).start(); MiruClientConfig clientConfig = deployable.config(MiruClientConfig.class); @SuppressWarnings("unchecked") TenantAwareHttpClient<String> manageHttpClient = tenantRoutingHttpClientInitializer .builder(tenantRoutingProvider.getConnections("miru-manage", "main", 10_000), // TODO config clientHealthProvider) .deadAfterNErrors(10).checkDeadEveryNMillis(10_000).build(); // TODO expose to conf @SuppressWarnings("unchecked") TenantAwareHttpClient<String> walHttpClient = tenantRoutingHttpClientInitializer .builder(tenantRoutingProvider.getConnections("miru-wal", "main", 10_000), // TODO config clientHealthProvider) .deadAfterNErrors(10).checkDeadEveryNMillis(10_000).build(); // TODO expose to conf final Map<MiruTenantId, Boolean> latestAlignmentCache = Maps.newConcurrentMap(); WriterAmzaServiceConfig miruAmzaServiceConfig = deployable.config(WriterAmzaServiceConfig.class); Lifecycle amzaLifecycle = new MiruAmzaServiceInitializer().initialize(deployable, clientHealthProvider, instanceConfig.getInstanceName(), instanceConfig.getInstanceKey(), instanceConfig.getServiceName(), instanceConfig.getDatacenter(), instanceConfig.getRack(), instanceConfig.getHost(), instanceConfig.getMainPort(), instanceConfig.getMainServiceAuthEnabled(), null, //"miru-writer-" + instanceConfig.getClusterName(), miruAmzaServiceConfig, true, -1, changes -> { if (changes.getVersionedPartitionName().getPartitionName() .equals(AmzaPartitionIdProvider.LATEST_PARTITIONS_PARTITION_NAME)) { for (WALKey key : changes.getApply().keySet()) { MiruTenantId tenantId = AmzaPartitionIdProvider .extractTenantForLatestPartition(key); latestAlignmentCache.remove(tenantId); } } }); SickThreads walClientSickThreads = new SickThreads(); deployable.addHealthCheck(new SickThreadsHealthCheck( deployable.config(WALClientSickThreadsHealthCheckConfig.class), walClientSickThreads)); MiruWALConfig walConfig = deployable.config(MiruWALConfig.class); MiruWALClient<?, ?> walClient; if (walConfig.getActivityWALType().equals("rcvs") || walConfig.getActivityWALType().equals("rcvs_amza")) { MiruWALClient<RCVSCursor, RCVSSipCursor> rcvsWALClient = new MiruWALClientInitializer().initialize( "", walHttpClient, mapper, walClientSickThreads, 10_000, "/miru/wal/rcvs", RCVSCursor.class, RCVSSipCursor.class); walClient = rcvsWALClient; } else if (walConfig.getActivityWALType().equals("amza") || walConfig.getActivityWALType().equals("amza_rcvs")) { MiruWALClient<AmzaCursor, AmzaSipCursor> amzaWALClient = new MiruWALClientInitializer().initialize( "", walHttpClient, mapper, walClientSickThreads, 10_000, "/miru/wal/amza", AmzaCursor.class, AmzaSipCursor.class); walClient = amzaWALClient; } else { throw new IllegalStateException("Invalid activity WAL type: " + walConfig.getActivityWALType()); } String indexClass = "lab"; EmbeddedClientProvider clientProvider = new EmbeddedClientProvider(amzaLifecycle.amzaService); AmzaPartitionIdProvider amzaPartitionIdProvider = new AmzaPartitionIdProvider(amzaLifecycle.amzaService, clientProvider, miruAmzaServiceConfig.getReplicateLatestPartitionTimeoutMillis(), miruAmzaServiceConfig.getReplicateCursorTimeoutMillis(), indexClass, clientConfig.getTotalCapacity(), walClient); MiruStats miruStats = new MiruStats(); MiruClusterClient clusterClient = new MiruClusterClientInitializer().initialize(miruStats, "", manageHttpClient, mapper); MiruPartitioner miruPartitioner = new MiruPartitioner(instanceConfig.getInstanceName(), amzaPartitionIdProvider, walClient, clusterClient, clientConfig.getPartitionMaximumAgeInMillis()); ExecutorService sendActivitiesExecutorService = Executors .newFixedThreadPool(clientConfig.getSendActivitiesThreadPoolSize()); MiruActivityIngress activityIngress = new MiruActivityIngress(miruPartitioner, latestAlignmentCache, sendActivitiesExecutorService); MiruSoyRendererConfig rendererConfig = deployable.config(MiruSoyRendererConfig.class); File staticResourceDir = new File(System.getProperty("user.dir")); System.out.println("Static resources rooted at " + staticResourceDir.getAbsolutePath()); Resource sourceTree = new Resource(staticResourceDir) //.addResourcePath("../../../../../src/main/resources") // fluff? .addResourcePath(rendererConfig.getPathToStaticResources()).setDirectoryListingAllowed(false) .setContext("/ui/static"); MiruSoyRenderer renderer = new MiruSoyRendererInitializer().initialize(rendererConfig); MiruWriterUIService miruWriterUIService = new MiruWriterUIServiceInitializer().initialize( instanceConfig.getClusterName(), instanceConfig.getInstanceName(), renderer, miruStats, tenantRoutingProvider); if (instanceConfig.getMainServiceAuthEnabled()) { deployable.addRouteOAuth("/miru/*"); deployable.addSessionAuth("/ui/*", "/miru/*"); } else { deployable.addNoAuth("/miru/*"); deployable.addSessionAuth("/ui/*"); } deployable.addEndpoints(MiruWriterEndpoints.class); deployable.addInjectables(MiruWriterUIService.class, miruWriterUIService); deployable.addEndpoints(MiruIngressEndpoints.class); deployable.addInjectables(MiruStats.class, miruStats); deployable.addInjectables(MiruActivityIngress.class, activityIngress); deployable.addResource(sourceTree); deployable.addEndpoints(LoadBalancerHealthCheckEndpoints.class); deployable.buildServer().start(); clientHealthProvider.start(); isAmzaReady.set(amzaLifecycle::isReady); serviceStartupHealthCheck.success(); } catch (Throwable t) { serviceStartupHealthCheck.info("Encountered the following failure during startup.", t); } } }