Java tutorial
package com.linkedin.databus2.client.util; /* * * Copyright 2013 LinkedIn Corp. All rights reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * */ import java.io.FileReader; import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Properties; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.log4j.ConsoleAppender; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.log4j.PatternLayout; import org.codehaus.jackson.JsonParseException; import org.codehaus.jackson.map.JsonMappingException; import com.linkedin.databus.client.DatabusHttpClientImpl; import com.linkedin.databus.client.DatabusHttpClientImpl.CheckpointPersistenceStaticConfig; import com.linkedin.databus.client.DatabusHttpClientImpl.CheckpointPersistenceStaticConfigBuilder; import com.linkedin.databus.client.pub.CheckpointPersistenceProvider; import com.linkedin.databus.client.pub.FileSystemCheckpointPersistenceProvider; import com.linkedin.databus.core.BootstrapCheckpointHandler; import com.linkedin.databus.core.Checkpoint; import com.linkedin.databus.core.DatabusRuntimeException; import com.linkedin.databus.core.DbusClientMode; import com.linkedin.databus.core.util.ConfigLoader; /** Utility that can be used to serialize a checkpoint to a file */ //TODO We have to rethink this class and see if it is needed anymore. We also have to figure out what is the right level //of abstraction as currently it is too low level and needs access to internal checkpoint methods. public class CheckpointSerializerMain { public static final String MODULE = CheckpointSerializerMain.class.getSimpleName(); public static final Logger LOG = Logger.getLogger(MODULE); public static final String HELP_OPT_NAME = "help"; public static final String HELP_OPT_DESCR = "prints this help screen"; public static final String ACTION_OPT_NAME = "action"; public static final String ACTION_OPT_DESCR = "action to run: PRINT, CHANGE, DELETE"; public static final String CLIENT_PROPS_FILE_OPT_NAME = "client_props"; public static final String CLIENT_PROPS_FILE_OPT_DESCR = "specifies a client configuration properties file"; public static final String CP3_PROPS_FILE_OPT_NAME = "cp3_props"; public static final String CP3_PROPS_FILE_OPT_DESCR = "specifies a checkpoint persistence provider (CP3) configuration properties file "; public static final String PROPS_PREFIX_OPT_NAME = "props_prefix"; public static final String PROPS_PREFIX_OPT_DESCR = "properties name prefix for the configuration file (client or CP3)"; public static final String SOURCES_OPT_NAME = "sources"; public static final String SOURCES_OPT_DESCR = "comma-separated source list for the checkpoint; -1 for flexible checkpoint"; public static final String SCN_OPT_NAME = "sequence_num"; public static final String SCN_OPT_DESCR = "new scn for the checkpoint to be saved"; public static final String TYPE_OPT_NAME = "type"; public static final String TYPE_OPT_DESCR = "the type of the checkpoint to be saved: BOOTSTRAP_SNAPSHOT, BOOTSTRAP_CATCHUP, ONLINE_CONSUMPTION"; public static final String SINCE_SCN_OPT_NAME = "since_sequence_num"; public static final String SINCE_SCN_OPT_DESCR = "sequence number for when the bootstrap started (snapshot and catchup)"; public static final String START_SCN_OPT_NAME = "start_sequence_num"; public static final String START_SCN_OPT_DESCR = "start sequence number for bootstrap checkpoints (snapshot and catchup)"; public static final String TARGET_SCN_OPT_NAME = "target_sequence_num"; public static final String TARGET_SCN_OPT_DESCR = "target sequence number for bootstrap checkpoints (catchup only)"; public static final String BOOTSTRAP_SOURCE_OPT_NAME = "bootstrap_source"; public static final String BOOTSTRAP_SOURCE_OPT_DESCR = "the current bootstrap source name"; public static final char ACTION_OPT_CHAR = 'a'; public static final char BOOTSTRAP_SOURCE_OPT_CHAR = 'b'; public static final char CLIENT_PROPS_FILE_OPT_CHAR = 'c'; public static final char CP3_PROPS_FILE_OPT_CHAR = 'C'; public static final char PROPS_PREFIX_OPT_CHAR = 'f'; public static final char HELP_OPT_CHAR = 'h'; public static final char SOURCES_OPT_CHAR = 'S'; public static final char SCN_OPT_CHAR = 's'; public static final char TYPE_OPT_CHAR = 't'; public static final char SINCE_SCN_OPT_CHAR = 'x'; public static final char START_SCN_OPT_CHAR = 'y'; public static final char TARGET_SCN_OPT_CHAR = 'z'; private static enum Action { PRINT, CHANGE, DELETE } private static Action _action; private static String[] _sources; private static Properties _clientProps; private static Properties _cp3Props; private static String _propPrefix; private static Long _scn; private static Long _sinceScn; private static Long _startScn; private static Long _targetScn; private static DbusClientMode _cpType; private static String _bootstrapSource; @SuppressWarnings("static-access") private static Options createOptions() { Option helpOption = OptionBuilder.withLongOpt(HELP_OPT_NAME).withDescription(HELP_OPT_DESCR) .create(HELP_OPT_CHAR); Option clientPropsOption = OptionBuilder.withLongOpt(CLIENT_PROPS_FILE_OPT_NAME) .withDescription(CLIENT_PROPS_FILE_OPT_DESCR).hasArg().withArgName("properties_file") .create(CLIENT_PROPS_FILE_OPT_CHAR); Option cp3PropsOption = OptionBuilder.withLongOpt(CP3_PROPS_FILE_OPT_NAME) .withDescription(CP3_PROPS_FILE_OPT_DESCR).hasArg().withArgName("properties_file") .create(CP3_PROPS_FILE_OPT_CHAR); Option propsPrefixOption = OptionBuilder.withLongOpt(PROPS_PREFIX_OPT_NAME) .withDescription(PROPS_PREFIX_OPT_DESCR).hasArg().withArgName("prefix_string") .create(PROPS_PREFIX_OPT_CHAR); Option sourcesOption = OptionBuilder.withLongOpt(SOURCES_OPT_NAME).withDescription(SOURCES_OPT_DESCR) .hasArg().withArgName("sources_list").create(SOURCES_OPT_CHAR); Option scnOptOption = OptionBuilder.withLongOpt(SCN_OPT_NAME).withDescription(SCN_OPT_DESCR).hasArg() .withArgName("sequence_number").create(SCN_OPT_CHAR); Option actionOption = OptionBuilder.withLongOpt(ACTION_OPT_NAME).withDescription(ACTION_OPT_DESCR).hasArg() .withArgName("action").create(ACTION_OPT_CHAR); Option sinceScnOptOption = OptionBuilder.withLongOpt(SINCE_SCN_OPT_NAME) .withDescription(SINCE_SCN_OPT_DESCR).hasArg().withArgName("sequence_number") .create(SINCE_SCN_OPT_CHAR); Option startScnOptOption = OptionBuilder.withLongOpt(START_SCN_OPT_NAME) .withDescription(START_SCN_OPT_DESCR).hasArg().withArgName("sequence_number") .create(START_SCN_OPT_CHAR); Option targetScnOptOption = OptionBuilder.withLongOpt(TARGET_SCN_OPT_NAME) .withDescription(TARGET_SCN_OPT_DESCR).hasArg().withArgName("sequence_number") .create(TARGET_SCN_OPT_CHAR); Option typeOption = OptionBuilder.withLongOpt(TYPE_OPT_NAME).withDescription(TYPE_OPT_DESCR).hasArg() .withArgName("checkpoint_type").create(TYPE_OPT_CHAR); Option bootstrapSourceOption = OptionBuilder.withLongOpt(BOOTSTRAP_SOURCE_OPT_NAME) .withDescription(BOOTSTRAP_SOURCE_OPT_DESCR).hasArg().withArgName("bootstrap_source_name") .create(BOOTSTRAP_SOURCE_OPT_CHAR); Options options = new Options(); options.addOption(helpOption); options.addOption(actionOption); options.addOption(clientPropsOption); options.addOption(cp3PropsOption); options.addOption(propsPrefixOption); options.addOption(sourcesOption); options.addOption(scnOptOption); options.addOption(sinceScnOptOption); options.addOption(startScnOptOption); options.addOption(targetScnOptOption); options.addOption(typeOption); options.addOption(bootstrapSourceOption); return options; } private static void parseArgs(String[] args) throws Exception { CommandLineParser cliParser = new GnuParser(); Options options = createOptions(); CommandLine cmd = null; try { cmd = cliParser.parse(options, args); } catch (ParseException pe) { throw new RuntimeException("failed to parse command-line options.", pe); } if (cmd.hasOption(HELP_OPT_CHAR) || 0 == cmd.getOptions().length) { printCliHelp(options); System.exit(0); } try { _action = Action.valueOf(cmd.getOptionValue(ACTION_OPT_CHAR).toUpperCase()); } catch (Exception e) { throw new RuntimeException("invalid action: " + cmd.getOptionValue(ACTION_OPT_CHAR), e); } if (!cmd.hasOption(SOURCES_OPT_CHAR)) { throw new RuntimeException("expected sources list; see --help for more info"); } String sourcesListStr = cmd.getOptionValue(SOURCES_OPT_CHAR); _sources = sourcesListStr.split(","); if (null == _sources || 0 == _sources.length) { throw new RuntimeException("empty sources list"); } for (int i = 0; i < _sources.length; ++i) _sources[i] = _sources[i].trim(); if (Action.PRINT != _action && !cmd.hasOption(CLIENT_PROPS_FILE_OPT_CHAR) && !cmd.hasOption(CP3_PROPS_FILE_OPT_CHAR)) { throw new RuntimeException("expected client or CP3 configuration; see --help for more info"); } String defaultPropPrefix = null; if (cmd.hasOption(CLIENT_PROPS_FILE_OPT_CHAR)) { try { _clientProps = loadProperties(cmd.getOptionValue(CLIENT_PROPS_FILE_OPT_CHAR)); defaultPropPrefix = "databus2.client"; } catch (Exception e) { throw new RuntimeException("unable to load client properties", e); } } else if (cmd.hasOption(CP3_PROPS_FILE_OPT_CHAR)) { try { _cp3Props = loadProperties(cmd.getOptionValue(CP3_PROPS_FILE_OPT_CHAR)); defaultPropPrefix = "databus2.client.checkpointPersistence"; } catch (Exception e) { throw new RuntimeException("unable to load CP3 properties", e); } } _propPrefix = cmd.hasOption(PROPS_PREFIX_OPT_CHAR) ? cmd.getOptionValue(PROPS_PREFIX_OPT_CHAR) : defaultPropPrefix; if (null != _propPrefix && !_propPrefix.endsWith(".")) { _propPrefix = _propPrefix + "."; } if (!cmd.hasOption(ACTION_OPT_CHAR)) { throw new RuntimeException("action expected; see --help for more info"); } _scn = parseLongOption(cmd, SCN_OPT_CHAR, "sequence number"); _sinceScn = parseLongOption(cmd, SINCE_SCN_OPT_CHAR, "last sequence number"); _startScn = parseLongOption(cmd, START_SCN_OPT_CHAR, "start sequence number"); _targetScn = parseLongOption(cmd, TARGET_SCN_OPT_CHAR, "target sequence number"); if (cmd.hasOption(TYPE_OPT_CHAR)) { try { _cpType = DbusClientMode.valueOf(cmd.getOptionValue(TYPE_OPT_CHAR).toUpperCase()); } catch (Exception e) { throw new RuntimeException("invalid checkpoint type:" + cmd.getOptionValue(TYPE_OPT_CHAR), e); } } if (cmd.hasOption(BOOTSTRAP_SOURCE_OPT_CHAR)) { _bootstrapSource = cmd.getOptionValue(BOOTSTRAP_SOURCE_OPT_CHAR); } } private static Long parseLongOption(CommandLine cmd, char optionChar, String argName) { Long result = null; if (cmd.hasOption(optionChar)) { try { result = Long.parseLong(cmd.getOptionValue(optionChar)); } catch (NumberFormatException nfe) { throw new RuntimeException("invalid " + argName + ": " + cmd.getOptionValue(optionChar), nfe); } } return result; } private static Properties loadProperties(String fileName) throws IOException { Properties result = new Properties(); FileReader freader = new FileReader(fileName); try { result.load(freader); } finally { freader.close(); } return result; } private static void printCliHelp(Options cliOptions) { HelpFormatter helpFormatter = new HelpFormatter(); helpFormatter.setWidth(120); helpFormatter.printHelp("java " + CheckpointSerializerMain.class.getName(), cliOptions); } private static Checkpoint updateCheckpoint(Checkpoint cpOld) throws JsonParseException, JsonMappingException, IOException { Checkpoint cpNew = null != cpOld ? new Checkpoint(cpOld.toString()) : new Checkpoint(); if (null != _scn) { if (-1L != _scn) { cpNew.setWindowScn(_scn); cpNew.setWindowOffset(0); } else { cpNew.setFlexible(); } } if (null != _startScn) { cpNew.setBootstrapStartScn(_startScn); } if (null != _targetScn) { cpNew.setBootstrapTargetScn(_targetScn); } if (null != _cpType) { cpNew.setConsumptionMode(_cpType); switch (_cpType) { case ONLINE_CONSUMPTION: cpNew.setWindowOffset(0); break; /* * TODO Disabling as the bootstrap checkpoint creation leaves out important * information (e.g. catchup/snashot source index) out of the checkpoint * and thus is incorrect. We have to figure out what types of bootstrap * checkpoints it makes sense to create. case BOOTSTRAP_CATCHUP: { if (null != _bootstrapSource) cpNew.setCatchupSource(_bootstrapSource); cpNew.setCatchupOffset(-1); break; }*/ case BOOTSTRAP_SNAPSHOT: { BootstrapCheckpointHandler handler = new BootstrapCheckpointHandler(_sources); cpNew = handler.createInitialBootstrapCheckpoint(cpNew, _sinceScn); //if (null != _bootstrapSource) cpNew.setSnapshotSource(_bootstrapSource); cpNew.setSnapshotOffset(-1); break; } default: throw new DatabusRuntimeException("unsupported checkpoint type: " + _cpType); } } return cpNew; } public static void main(String[] args) throws Exception { parseArgs(args); PatternLayout defaultLayout = new PatternLayout("%d{ISO8601} +%r [%t] (%p) {%c{1}} %m%n"); ConsoleAppender defaultAppender = new ConsoleAppender(defaultLayout); Logger.getRootLogger().removeAllAppenders(); Logger.getRootLogger().addAppender(defaultAppender); Logger.getRootLogger().setLevel(Level.INFO); Logger.getRootLogger().info("NOTE. This tool works only with V2/V1 checkpoints"); CheckpointPersistenceProvider cp3 = null; if (null != _cp3Props) { CheckpointPersistenceStaticConfigBuilder cp3ConfBuilder = new CheckpointPersistenceStaticConfigBuilder(); ConfigLoader<CheckpointPersistenceStaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.CheckpointPersistenceStaticConfig>( _propPrefix, cp3ConfBuilder); configLoader.loadConfig(_cp3Props); CheckpointPersistenceStaticConfig cp3Conf = cp3ConfBuilder.build(); if (cp3Conf.getType() != CheckpointPersistenceStaticConfig.ProviderType.FILE_SYSTEM) { throw new RuntimeException("don't know what to do with cp3 type:" + cp3Conf.getType()); } cp3 = new FileSystemCheckpointPersistenceProvider(cp3Conf.getFileSystem(), 2); } else if (null != _clientProps) { DatabusHttpClientImpl.Config clientConfBuilder = new DatabusHttpClientImpl.Config(); ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>( _propPrefix, clientConfBuilder); configLoader.loadConfig(_clientProps); DatabusHttpClientImpl.StaticConfig clientConf = clientConfBuilder.build(); if (clientConf.getCheckpointPersistence() .getType() != CheckpointPersistenceStaticConfig.ProviderType.FILE_SYSTEM) { throw new RuntimeException( "don't know what to do with cp3 type:" + clientConf.getCheckpointPersistence().getType()); } cp3 = new FileSystemCheckpointPersistenceProvider(clientConf.getCheckpointPersistence().getFileSystem(), 2); } List<String> sourceList = Arrays.asList(_sources); Checkpoint cpOld = null != cp3 ? cp3.loadCheckpoint(sourceList) : new Checkpoint(); Checkpoint cpNew; if (Action.PRINT == _action) { cpNew = updateCheckpoint(cpOld); } else if (Action.CHANGE == _action) { cpNew = updateCheckpoint(cpOld); cp3.storeCheckpoint(sourceList, cpNew); //reread as a sanity check cpNew = cp3.loadCheckpoint(sourceList); } else if (Action.DELETE == _action) { cp3.removeCheckpoint(sourceList); cpNew = cp3.loadCheckpoint(sourceList); } else { throw new RuntimeException("don't know what to do with action: " + _action); } if (null != cpOld) System.out.println("old: " + cpOld.toString()); else System.out.println("old: null"); if (null != cpNew) System.out.println("new: " + cpNew.toString()); else System.out.println("new: null"); } }