Java tutorial
package com.linkedin.databus.client.generic; /* * * Copyright 2013 LinkedIn Corp. All rights reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * */ import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Enumeration; import java.util.List; import java.util.Properties; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.log4j.Logger; import com.linkedin.databus.client.DatabusHttpClientImpl; import com.linkedin.databus.client.pub.DatabusRegistration; import com.linkedin.databus.client.pub.ServerInfo.ServerInfoBuilder; import com.linkedin.databus.client.registration.DatabusV2RegistrationImpl; import com.linkedin.databus.core.util.ConfigLoader; import com.linkedin.databus.core.util.InvalidConfigException; import com.linkedin.databus2.core.container.netty.ServerContainer; import com.linkedin.databus2.core.container.request.ContainerOperationProcessor; import com.linkedin.databus2.core.container.request.ProcessorRegistrationConflictException; import com.linkedin.databus2.core.filter.DbusKeyCompositeFilterConfig; /** * Base class for setting up consumer, process arguments * @author dzhang * */ public class SimpleFileLoggingConsumer { public static final String MODULE = SimpleFileLoggingConsumer.class.getName(); public static final Logger LOG = Logger.getLogger(MODULE); public static final String RELAY_HOST_OPT_NAME = "relay_host"; public static final String RELAY_PORT_OPT_NAME = "relay_port"; public static final String EVENT_DUMP_FILE_OPT_NAME = "f"; public static final String VALUE_DUMP_FILE_OPT_NAME = "value_file"; public static final String HTTP_PORT_OPT_NAME = "http_port"; public static final String JMX_SERVICE_PORT_OPT_NAME = "jmx_service_port"; public static final String CHECKPOINT_DIR = "checkpoint_dir"; public static final String BOOTSTRAP_HOST_OPT_NAME = "bootstrap_host"; public static final String BOOTSTRAP_PORT_OPT_NAME = "bootstrap_port"; public static final String EVENT_PATTERN_OPT_NAME = "event_pattern"; public static final String FILTER_CONF_FILE_OPT_NAME = "filter_conf_file"; public static final String SERVER_SIDE_FILTER_PREFIX = "serversidefilter."; public static final String SOURCES_OPT_NAME = "sources"; private static String _eventDumpFile = null; private static String _valueDumpFile = null; private static String _relayHost = null; private static String _relayPort = null; private static String _bootstrapHost = null; private static String _bootstrapPort = null; private static String _httpPort = null; private static String _jmxServicePort = null; private static String _checkpointFileRootDir = null; private static String _eventPattern = null; private static boolean _enableBootStrap = false; private static String _filterConfFile = null; private static String[] _sources = null; protected static Options constructCommandLineOptions() { Options options = new Options(); options.addOption(RELAY_HOST_OPT_NAME, true, "Relay Host Name"); options.addOption(RELAY_PORT_OPT_NAME, true, "Relay Port"); options.addOption(EVENT_DUMP_FILE_OPT_NAME, true, "File to dump event"); options.addOption(VALUE_DUMP_FILE_OPT_NAME, true, "File to dump deserilized values"); options.addOption(HTTP_PORT_OPT_NAME, true, "Consumer http port"); options.addOption(JMX_SERVICE_PORT_OPT_NAME, true, "Consumer jmx service port"); options.addOption(CHECKPOINT_DIR, true, "Checkpoint dir"); options.addOption(BOOTSTRAP_HOST_OPT_NAME, true, "Bootstrap server Name"); options.addOption(BOOTSTRAP_PORT_OPT_NAME, true, "Bootstrap server Port"); options.addOption(EVENT_PATTERN_OPT_NAME, true, "Event Pattern Name to Check"); options.addOption(FILTER_CONF_FILE_OPT_NAME, true, "Server Side Filter Config"); options.addOption(SOURCES_OPT_NAME, true, "Comma seperated source names"); return options; } protected static String[] processLocalArgs(String[] cliArgs) throws IOException, ParseException { CommandLineParser cliParser = new GnuParser(); Options cliOptions = constructCommandLineOptions(); CommandLine cmd = cliParser.parse(cliOptions, cliArgs, true); // Options here has to be up front if (cmd.hasOption(RELAY_HOST_OPT_NAME)) { _relayHost = cmd.getOptionValue(RELAY_HOST_OPT_NAME); LOG.info("Relay Host = " + _relayHost); } if (cmd.hasOption(RELAY_PORT_OPT_NAME)) { _relayPort = cmd.getOptionValue(RELAY_PORT_OPT_NAME); LOG.info("Relay Port = " + _relayPort); } if (cmd.hasOption(EVENT_DUMP_FILE_OPT_NAME)) { _eventDumpFile = cmd.getOptionValue(EVENT_DUMP_FILE_OPT_NAME); LOG.info("Saving event dump to file: " + _eventDumpFile); } if (cmd.hasOption(VALUE_DUMP_FILE_OPT_NAME)) { _valueDumpFile = cmd.getOptionValue(VALUE_DUMP_FILE_OPT_NAME); LOG.info("Saving event value dump to file: " + _valueDumpFile); } if (cmd.hasOption(HTTP_PORT_OPT_NAME)) { _httpPort = cmd.getOptionValue(HTTP_PORT_OPT_NAME); LOG.info("Consumer http port = " + _httpPort); } if (cmd.hasOption(JMX_SERVICE_PORT_OPT_NAME)) { _jmxServicePort = cmd.getOptionValue(JMX_SERVICE_PORT_OPT_NAME); LOG.info("Consumer JMX Service port = " + _jmxServicePort); } if (cmd.hasOption(CHECKPOINT_DIR)) { _checkpointFileRootDir = cmd.getOptionValue(CHECKPOINT_DIR); LOG.info("Checkpoint dir = " + _checkpointFileRootDir); } if (cmd.hasOption(BOOTSTRAP_HOST_OPT_NAME)) { _bootstrapHost = cmd.getOptionValue(BOOTSTRAP_HOST_OPT_NAME); LOG.info("Bootstrap Server = " + _bootstrapHost); } if (cmd.hasOption(BOOTSTRAP_PORT_OPT_NAME)) { _bootstrapPort = cmd.getOptionValue(BOOTSTRAP_PORT_OPT_NAME); LOG.info("Bootstrap Server Port = " + _bootstrapPort); } if (cmd.hasOption(EVENT_PATTERN_OPT_NAME)) { _eventPattern = cmd.getOptionValue(EVENT_PATTERN_OPT_NAME); LOG.info("Event pattern = " + _eventPattern); } if (_bootstrapHost != null || _bootstrapPort != null) { _enableBootStrap = true; } if (cmd.hasOption(FILTER_CONF_FILE_OPT_NAME)) { _filterConfFile = cmd.getOptionValue(FILTER_CONF_FILE_OPT_NAME); LOG.info("Server Side Filtering Config File =" + _filterConfFile); } if (cmd.hasOption(SOURCES_OPT_NAME)) { _sources = cmd.getOptionValue(SOURCES_OPT_NAME).split(","); LOG.info("Sources to be subscribed are =" + Arrays.toString(_sources)); } // return what left over args return cmd.getArgs(); } /** * @return the sources to be consumed by the consumer. All the sources should be co-located in the same Relay EVB. */ protected String[] addSources() { return _sources; } protected DatabusFileLoggingConsumer createTypedConsumer(String valueDumpFile) throws IOException { return createTypedConsumer(valueDumpFile, null); } protected DatabusFileLoggingConsumer createTypedConsumer(String valueDumpFile, String eventDumpFile) throws IOException { return new DatabusFileLoggingConsumer(valueDumpFile, null, eventDumpFile, false); } public void mainFunction(String args[]) throws Exception { String[] leftOverArgs = processLocalArgs(args); Properties startupProps = DatabusHttpClientImpl.processCommandLineArgs(leftOverArgs); DatabusHttpClientImpl.Config clientConfigBuilder = new DatabusHttpClientImpl.Config(); clientConfigBuilder.getContainer().setIdFromName(MODULE + ".localhost"); if (_enableBootStrap) { clientConfigBuilder.getRuntime().getBootstrap().setEnabled(true); } ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>( "databus.client.", clientConfigBuilder); String[] sources = addSources(); StringBuilder sourcesString = new StringBuilder(); boolean firstSrc = true; for (String source : sources) { if (!firstSrc) sourcesString.append(","); firstSrc = false; sourcesString.append(source); } if (_httpPort != null) { startupProps.put("databus.client.container.httpPort", _httpPort); } if (_jmxServicePort != null) { startupProps.put("databus.client.container.jmx.jmxServicePort", _jmxServicePort); } if (_checkpointFileRootDir != null) { startupProps.put("databus.client.checkpointPersistence.fileSystem.rootDirectory", _checkpointFileRootDir); } DatabusHttpClientImpl.StaticConfig clientConfig = configLoader.loadConfig(startupProps); // set up relay ServerInfoBuilder relayBuilder = clientConfig.getRuntime().getRelay("1"); relayBuilder.setName("DefaultRelay"); if (_relayHost != null) { relayBuilder.setHost(_relayHost); } if (_relayPort != null) { relayBuilder.setPort(Integer.parseInt(_relayPort)); } relayBuilder.setSources(sourcesString.toString()); // set up bootstrap if (_enableBootStrap) { ServerInfoBuilder bootstrapBuilder = clientConfig.getRuntime().getBootstrap().getService("2"); bootstrapBuilder.setName("DefaultBootstrapServices"); if (_bootstrapHost != null) { bootstrapBuilder.setHost(_bootstrapHost); } if (_bootstrapPort != null) { bootstrapBuilder.setPort(Integer.parseInt(_bootstrapPort)); } bootstrapBuilder.setSources(sourcesString.toString()); } // handler server side filtering, can either pass a config file or set from command line DbusKeyCompositeFilterConfig filterConfig = createServerSideFilterConfig(_filterConfFile, startupProps); // set up listeners DatabusHttpClientImpl client = new DatabusHttpClientImpl(clientConfig); // dump decoded payload values and raw (undecoded) events DatabusFileLoggingConsumer consumer = createTypedConsumer(_valueDumpFile, _eventDumpFile); if (_eventPattern != null) { consumer.setEventPattern(_eventPattern); } DatabusRegistration reg = client.register(consumer, sources); if (!(reg instanceof DatabusV2RegistrationImpl)) { throw new RuntimeException("Unexpected type for registration Object !!"); } if (null != filterConfig) reg.withServerSideFilter(filterConfig); // add pause processor try { client.getProcessorRegistry().register(ConsumerPauseRequestProcessor.COMMAND_NAME, new ConsumerPauseRequestProcessor(null, consumer)); client.getProcessorRegistry().register(ContainerOperationProcessor.COMMAND_NAME, new ContainerOperationProcessor(null, client)); } catch (ProcessorRegistrationConflictException e) { LOG.error("Failed to register " + ConsumerPauseRequestProcessor.COMMAND_NAME); } DatabusClientShutdownThread shutdownThread = new DatabusClientShutdownThread(client); Runtime.getRuntime().addShutdownHook(shutdownThread); client.startAndBlock(); //this should automatically start the registration } private DbusKeyCompositeFilterConfig createServerSideFilterConfig(String filterConfFile, Properties startupProps) throws IOException, InvalidConfigException { Boolean cmdLineHasFilterConfig = hasServerSideFilterConfig(startupProps); if (filterConfFile != null || cmdLineHasFilterConfig) { Properties props; // cmdline override the config file if (cmdLineHasFilterConfig) { props = startupProps; } else { LOG.info("filterConfFile = " + filterConfFile); props = new Properties(); FileInputStream filterConfStream = new FileInputStream(filterConfFile); try { props.load(filterConfStream); } finally { filterConfStream.close(); } } DbusKeyCompositeFilterConfig.Config conf = new DbusKeyCompositeFilterConfig.Config(); ConfigLoader<DbusKeyCompositeFilterConfig.StaticConfig> filterConfigLoader = new ConfigLoader<DbusKeyCompositeFilterConfig.StaticConfig>( SERVER_SIDE_FILTER_PREFIX, conf); DbusKeyCompositeFilterConfig.StaticConfig sConf = filterConfigLoader.loadConfig(props); return new DbusKeyCompositeFilterConfig(sConf); } else { return null; } } private Boolean hasServerSideFilterConfig(Properties startupProps) { Enumeration<Object> propKeys = startupProps.keys(); while (propKeys.hasMoreElements()) { String key = (String) propKeys.nextElement(); if (key.contains(SERVER_SIDE_FILTER_PREFIX)) { return true; } } return false; } static class DatabusClientShutdownThread extends Thread { public static final String MODULE = DatabusClientShutdownThread.class.getName(); public static final Logger LOG = Logger.getLogger(MODULE); private final ServerContainer _serverContainer; public DatabusClientShutdownThread(ServerContainer serverContainer) { super("ServerContainer Shutdown Thread"); _serverContainer = serverContainer; } @Override public void run() { LOG.info("Starting shutdown procedure for server container..."); try { if (null != _serverContainer && _serverContainer.isRunningStatus()) { _serverContainer.shutdownUninteruptibly(); } LOG.info("Server Container shutdown."); } catch (Exception e) { LOG.error("Error shutting down Server Container", e); } } } }