Java tutorial
/* * Copyright 2012-2015, the original author or authors. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.flipkart.aesop.runtime.producer; import java.net.URI; import java.net.URISyntaxException; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.avro.generic.GenericRecord; import org.springframework.beans.factory.InitializingBean; import org.springframework.util.Assert; import org.trpr.platform.core.impl.logging.LogFactory; import org.trpr.platform.core.spi.logging.Logger; import com.flipkart.aesop.runtime.producer.avro.MysqlAvroEventManager; import com.flipkart.aesop.runtime.producer.eventlistener.OpenReplicationListener; import com.flipkart.aesop.runtime.producer.eventprocessor.BinLogEventProcessor; import com.flipkart.aesop.runtime.producer.mapper.BinLogEventMapper; import com.flipkart.aesop.runtime.producer.schema.eventprocessor.SchemaChangeEventProcessor; import com.flipkart.aesop.runtime.producer.spi.SCNGenerator; import com.flipkart.aesop.runtime.producer.txnprocessor.MysqlTransactionManager; import com.flipkart.aesop.runtime.producer.txnprocessor.impl.MysqlTransactionManagerImpl; import com.flipkart.aesop.runtime.producer.txnprocessor.impl.NaiveSCNGenerator; import com.google.code.or.OpenReplicator; import com.linkedin.databus.core.UnsupportedKeyException; import com.linkedin.databus.core.util.InvalidConfigException; import com.linkedin.databus2.core.DatabusException; import com.linkedin.databus2.producers.EventCreationException; import com.linkedin.databus2.relay.config.LogicalSourceStaticConfig; import com.linkedin.databus2.relay.config.PhysicalSourceStaticConfig; /** * <code>MysqlEventProducer</code> kick starts bin log event listener to listen to Mysql events using open replicator * library and in * turn creates change events of {@link GenericRecord} * @author Shoury B * @version 1.0, 07 Mar 2014 */ public class MysqlEventProducer<T extends GenericRecord> extends AbstractEventProducer implements InitializingBean { /** Logger for this class */ private static final Logger LOGGER = LogFactory.getLogger(MysqlEventProducer.class); /** Default Mysql port */ private static final Integer DEFAULT_MYSQL_PORT = 3306; /** Pattern for extracting /3306/mysql-bin out of mysql://or_test%2For_test@localhost:3306/3306/mysql-bin */ private static final Pattern PATH_PATTERN = Pattern.compile("/([0-9]+)/[a-z|A-Z|0-9|-]+"); /** Index of server id in pattern match group */ private static final int SERVER_ID = 1; /** Index of bin log prefix in pattern match group */ private static final int BIN_LOG_PREFIX = 2; /** Event mapper which maps bin log events to one of schemas registered */ protected Map<Integer, BinLogEventMapper<T>> binLogEventMappers; /** Open Replicator which listens and parses bin log events from Mysql */ protected OpenReplicator openReplicator; /** Mysql transaction manager */ protected MysqlTransactionManager mysqlTxnManager; /** Event Id to Event Processor map */ protected Map<Integer, BinLogEventProcessor> eventsMap; /** Schema Change Event Processor */ protected SchemaChangeEventProcessor schemaChangeEventProcessor; /** The SCN generator implementation, initialized to the default simple implementation*/ protected SCNGenerator scnGenerator = new NaiveSCNGenerator(); /** * Interface method implementation. Checks for mandatory dependencies and creates the Open Replicator * @see org.springframework.beans.factory.InitializingBean#afterPropertiesSet() */ @Override public void afterPropertiesSet() throws Exception { Assert.notNull(this.binLogEventMappers, "'binLogEventMapper' cannot be null. No bin log event mapper found. This Mysql Events producer will not be initialized"); Assert.notNull(this.eventsMap, "'eventsMap' cannot be null. eventsMap is not initialized properly.This Mysql Events producer will not be initialized"); } /** * Starting point for this event producer. Starts Open Replicator listener * @param sinceSCN starting SCN * @see com.linkedin.databus2.producers.EventProducer#start(long) */ public void start(long sinceSCN) { this.sinceSCN.set(sinceSCN); openReplicator = new OpenReplicator(); String binlogFile; try { String binlogFilePrefix = processUri(new URI(physicalSourceStaticConfig.getUri())); int offset = offset(sinceSCN); int logid = logid(sinceSCN); LOGGER.debug("SCN : " + sinceSCN + " logid : " + logid); binlogFile = String.format("%s.%06d", binlogFilePrefix, logid); LOGGER.debug("Bin Log File Name : " + binlogFile); Map<String, Short> tableUriToSrcIdMap = new HashMap<String, Short>(); Map<String, String> tableUriToSrcNameMap = new HashMap<String, String>(); Map<Integer, MysqlAvroEventManager<T>> eventManagersMap = new HashMap<Integer, MysqlAvroEventManager<T>>(); for (LogicalSourceStaticConfig sourceConfig : physicalSourceStaticConfig.getSources()) { tableUriToSrcIdMap.put(sourceConfig.getUri().toLowerCase(), sourceConfig.getId()); tableUriToSrcNameMap.put(sourceConfig.getUri().toLowerCase(), sourceConfig.getName()); MysqlAvroEventManager<T> manager = null; try { manager = buildEventManagers(sourceConfig, physicalSourceStaticConfig); } catch (Exception ex) { LOGGER.error("Got exception while building monitored sources for config :" + sourceConfig, ex); throw new InvalidConfigException(ex); } eventManagersMap.put(Integer.valueOf(sourceConfig.getId()), manager); } schemaChangeEventProcessor.setSchemaRegistryService(schemaRegistryService); schemaChangeEventProcessor.setTableUriToSrcNameMap(tableUriToSrcNameMap); /** updating schemas for registered logical sources */ for (LogicalSourceStaticConfig sourceConfig : physicalSourceStaticConfig.getSources()) { String[] parts = sourceConfig.getUri().split("\\."); schemaChangeEventProcessor.process(parts[0], parts[1]); } mysqlTxnManager = new MysqlTransactionManagerImpl<T>(eventBuffer, maxScnReaderWriter, dbusEventsStatisticsCollector, eventManagersMap, logid, tableUriToSrcIdMap, tableUriToSrcNameMap, schemaRegistryService, this.sinceSCN, binLogEventMappers, scnGenerator, this); mysqlTxnManager.setShutdownRequested(false); OpenReplicationListener orl = new OpenReplicationListener(mysqlTxnManager, eventsMap, schemaChangeEventProcessor, binlogFilePrefix); openReplicator.setBinlogFileName(binlogFile); openReplicator.setBinlogPosition(offset); openReplicator.setBinlogEventListener(orl); openReplicator.start(); } catch (URISyntaxException u) { LOGGER.error("Exception occurred while processing uri : " + u); return; } catch (InvalidConfigException e) { LOGGER.error("Exception occurred while processing uri : " + e); return; } catch (Exception e) { LOGGER.error("Error occurred while starting open replication.." + e); return; } LOGGER.info("Open Replicator has been started successfully for the file " + binlogFile); } /** * Builds event managers for the given sources. * @param sourceConfig logical source configuration * @param pConfig physical source configuration * @return event factory for the given source * @throws DatabusException Generic Databus Exception * @throws EventCreationException Thrown when event creation failed for a databus source * @throws UnsupportedKeyException Thrown when the data type of the "key" field is not a supported type * @throws InvalidConfigException Throws when invalid source config is present in configuration provided */ public MysqlAvroEventManager<T> buildEventManagers(LogicalSourceStaticConfig sourceConfig, PhysicalSourceStaticConfig pConfig) throws DatabusException, EventCreationException, UnsupportedKeyException, InvalidConfigException { MysqlAvroEventManager<T> manager = new MysqlAvroEventManager<T>(sourceConfig.getId(), (short) pConfig.getId()); return manager; } /** * Returns the logid ( upper 32 bits of the SCN ) * For e.g., mysql-bin.000001 is said to have an id 000001 * @param scn system change number * @return logid */ public static int logid(long scn) { if (scn == -1 || scn == 0) { return 1; } return (int) ((scn >> 32) & 0xFFFFFFFF); } /** * Returns the binlogoffset ( lower 32 bits of the SCN ) * @param scn system change number * @return binlogoffset */ public static int offset(long scn) { if (scn == -1 || scn == 0) { return 4; } return (int) (scn & 0xFFFFFFFF); } /** * Interface method implementation. Returns {@link BinLogEventMapper#getUniqueName()} * @see com.linkedin.databus2.producers.EventProducer#getName() */ @Override public String getName() { return this.name; } /** * Interface method implementation. * @see com.linkedin.databus2.producers.EventProducer#getSCN() */ public long getSCN() { return this.sinceSCN.get(); } public void updateSCN(long latestScn) { this.sinceSCN.set(latestScn); } /** * Interface method implementation. Returns inverted status of {@link #isRunning()} * @see com.linkedin.databus2.producers.EventProducer#isPaused() */ @Override public boolean isPaused() { return !this.openReplicator.isRunning(); } /** * Interface method implementation. Returns {@link OpenReplicator#isRunning()} * @see com.linkedin.databus2.producers.EventProducer#isRunning() */ @Override public boolean isRunning() { return this.openReplicator.isRunning(); } /** * Interface method implementation. Stops the Open Replicator * @see com.linkedin.databus2.producers.EventProducer#shutdown() */ @Override public void shutdown() { LOGGER.info("Shutdown has been requested. MYSQLEventProducer shutttng down"); try { mysqlTxnManager.setShutdownRequested(true); LOGGER.info("Open Replicator Shutting down"); this.openReplicator.stop(10, TimeUnit.SECONDS); LOGGER.info("Open Replicator shutdown complete"); super.shutdown(); } catch (Exception e) { LOGGER.error("Error while stopping open replicator", e); } LOGGER.info("MYSQLEventProducer shutdown completed"); } /** * Returns the host from where the bin log is being read * @return the bin log host name */ public String getBinLogHost() { return this.openReplicator.getHost(); } /** Methods that are not supported and therefore throw {@link UnsupportedOperationException} */ public void pause() { throw new UnsupportedOperationException("'pause' is not supported on this event producer"); } public void unpause() { throw new UnsupportedOperationException("'unpause' is not supported on this event producer"); } public void waitForShutdown() throws InterruptedException, IllegalStateException { throw new UnsupportedOperationException("'waitForShutdown' is not supported on this event producer"); } public void waitForShutdown(long time) throws InterruptedException, IllegalStateException { throw new UnsupportedOperationException( "'waitForShutdown(long time)' is not supported on this event producer"); } /** Getters and Setters for this class */ public Map<Integer, BinLogEventMapper<T>> getBinLogEventMappers() { return binLogEventMappers; } public void setBinLogEventMappers(Map<Integer, BinLogEventMapper<T>> binLogEventMapper) { this.binLogEventMappers = binLogEventMapper; } public MysqlTransactionManager getMysqlTxnManager() { return mysqlTxnManager; } public void setMysqlTxnManager(MysqlTransactionManager mysqlTxnManager) { this.mysqlTxnManager = mysqlTxnManager; } public Map<Integer, BinLogEventProcessor> getEventsMap() { return eventsMap; } public void setEventsMap(Map<Integer, BinLogEventProcessor> eventsMap) { this.eventsMap = eventsMap; } public SchemaChangeEventProcessor getSchemaChangeEventProcessor() { return schemaChangeEventProcessor; } public void setSchemaChangeEventProcessor(SchemaChangeEventProcessor schemaChangeEventProcessor) { this.schemaChangeEventProcessor = schemaChangeEventProcessor; } public SCNGenerator getScnGenerator() { return scnGenerator; } public void setScnGenerator(SCNGenerator scnGenerator) { this.scnGenerator = scnGenerator; } /** * Extracts individual attributes such as username, password, hostname, port ,server id etc from the uri of the * format mysql://or_test%2For_test@localhost:3306/3306/mysql-bin * @param uri uri of the format mysql://or_test%2For_test@localhost:3306/3306/mysql-bin * @return returns bin log prefix */ protected String processUri(URI uri) throws InvalidConfigException { String userInfo = uri.getUserInfo(); if (null == userInfo) { String errorMessage = "missing user info in: " + uri; LOGGER.error(errorMessage); throw new InvalidConfigException(errorMessage); } int slashPos = userInfo.indexOf('/'); if (slashPos < 0) { slashPos = userInfo.length(); } else if (0 == slashPos) { String errorMessage = "missing user name in user info: " + userInfo; LOGGER.error(errorMessage); throw new InvalidConfigException(errorMessage); } String userName = userInfo.substring(0, slashPos); String userPass = slashPos < userInfo.length() - 1 ? userInfo.substring(slashPos + 1) : null; String hostName = uri.getHost(); int port = uri.getPort(); if (port < 0) port = DEFAULT_MYSQL_PORT; String path = uri.getPath(); if (null == path) { String errorMessage = "missing path: " + uri; LOGGER.error(errorMessage); throw new InvalidConfigException(errorMessage); } Matcher matcher = PATH_PATTERN.matcher(path); if (!matcher.matches()) { String errorMessage = "invalid path:" + path; LOGGER.error(errorMessage); throw new InvalidConfigException(errorMessage); } String[] group = matcher.group().split("/"); if (group.length != 3) { String errorMessage = "Invalid format " + Arrays.toString(group); LOGGER.error(errorMessage); throw new InvalidConfigException(errorMessage); } String serverIdStr = group[SERVER_ID]; int serverId = -1; try { serverId = Integer.parseInt(serverIdStr); } catch (NumberFormatException e) { String errorMessage = "incorrect mysql serverid:" + serverId; LOGGER.error(errorMessage); throw new InvalidConfigException(errorMessage); } /** Assign them to incoming variables */ if (null != openReplicator) { openReplicator.setUser(userName); if (null != userPass) { openReplicator.setPassword(userPass); } openReplicator.setHost(hostName); openReplicator.setPort(port); openReplicator.setServerId(serverId); } LOGGER.debug("Extracted bin log prefix is " + group[BIN_LOG_PREFIX]); return group[BIN_LOG_PREFIX]; } }