Java tutorial
/* * Copyright 2012-2015, the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.flipkart.aesop.runtime.producer.hbase; import java.net.InetAddress; import java.util.Iterator; import java.util.List; import java.util.Map.Entry; import java.util.concurrent.atomic.AtomicBoolean; import com.flipkart.aesop.runtime.producer.spi.SCNGenerator; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.springframework.beans.factory.InitializingBean; import org.springframework.util.Assert; import org.trpr.platform.core.PlatformException; import org.trpr.platform.core.impl.logging.LogFactory; import org.trpr.platform.core.spi.logging.Logger; import com.flipkart.aesop.runtime.producer.AbstractEventProducer; import com.linkedin.databus.core.DbusEventInfo; import com.linkedin.databus.core.DbusEventKey; import com.linkedin.databus.core.DbusOpcode; import com.linkedin.databus2.core.DatabusException; import com.linkedin.databus2.schemas.utils.SchemaHelper; import com.ngdata.sep.EventListener; import com.ngdata.sep.SepEvent; import com.ngdata.sep.SepModel; import com.ngdata.sep.impl.SepConsumer; import com.ngdata.sep.impl.SepModelImpl; import com.ngdata.sep.util.zookeeper.ZkUtil; import com.ngdata.sep.util.zookeeper.ZooKeeperItf; /** * <code>HBaseEventProducer</code> is a sub-type of {@link AbstractEventProducer} that listens to HBase WAL edits using the hbase-sep module library classes * such as {@link SepConsumer} and {@link EventListener} and in turn creates change events of {@link GenericRecord} sub-type T. * * @author Regunath B * @version 1.0, 28 Jan 2014 */ public class HBaseEventProducer<T extends GenericRecord> extends AbstractEventProducer implements InitializingBean { /** Logger for this class*/ private static final Logger LOGGER = LogFactory.getLogger(HBaseEventProducer.class); /** The HBase replication configuration parameter */ private static final String HBASE_REPLICATION_CONFIG = "hbase.replication"; private static final String ZK_QUORUM_CONFIG = "hbase.zookeeper.quorum"; private static final String ZK_CLIENT_PORT_CONFIG = "hbase.zookeeper.property.clientPort"; /** Timeout in case, SEP is taking more time to consume the events. After timeout, RS will push the same load again**/ private static final String RPC_TIMEOUT_CONFIG = "hbase.rpc.timeout"; /** The localhost */ private static final String LOCAL_HOST_NAME = "localhost"; /** The default ZK settings*/ private static final int ZK_CLIENT_PORT = 2181; private static final int ZK_SESSION_TIMEOUT = 20000; // 20 seconds private static final int RPC_TIMEOUT = 60000; //60 sec default /** The default number of worker threads that process WAL edit events*/ private static final int WORKER_THREADS = 1; /**RPC Timeout for SEP to consume the events sent by RS**/ protected int rpcTimeout = RPC_TIMEOUT; /** The SEP consumer instance initialized by this Producer*/ protected SepConsumer sepConsumer; /** The SepEventMapper for translating WAL edits to change events*/ protected SepEventMapper<T> sepEventMapper; /** The Zookeeper connection properties*/ protected String zkQuorum; protected int zkClientPort = ZK_CLIENT_PORT; protected Integer zkSessionTimeout = ZK_SESSION_TIMEOUT; /** The number of WAL edits processing worker threads*/ protected int workerThreads = WORKER_THREADS; /** The ScnGenerator for generating relayer Scn*/ protected SCNGenerator scnGenerator = new MonotonicSequenceSCNGenerator(); private volatile AtomicBoolean shutdownRequested = new AtomicBoolean(false); /** Host name where this producer is running i.e. local host name*/ private String localHost; /** * Interface method implementation. Checks for mandatory dependencies and creates the SEP consumer * @see org.springframework.beans.factory.InitializingBean#afterPropertiesSet() */ public void afterPropertiesSet() throws Exception { Assert.notNull(this.zkQuorum, "'zkQuorum' cannot be null. Zookeeper quorum list must be specified. This HBase Events producer will not be initialized"); if (this.zkQuorum.contains(":")) { throw new IllegalStateException( "'zkQuorum' is comma separated list of only hosts. Specify port using 'zkClientPort' : " + this.zkQuorum); } Assert.notNull(this.sepEventMapper, "'sepEventMapper' cannot be null. No WAL edits event mapper found. This HBase Events producer will not be initialized"); if (this.zkQuorum.contains(LOCAL_HOST_NAME)) { // we dont want 'localhost' to resolve to other names - say from /etc/hosts if ZK is also running locally during testing this.localHost = LOCAL_HOST_NAME; } else { this.localHost = InetAddress.getLocalHost().getHostName(); } } /** * Interface method implementation. Starts up the SEP consumer * @see com.linkedin.databus2.producers.EventProducer#start(long) */ public void start(long sinceSCN) { shutdownRequested.set(false); this.sinceSCN.set(sinceSCN); LOGGER.info("Starting SEP subscription : " + this.getName()); LOGGER.info("ZK quorum hosts : " + this.zkQuorum); LOGGER.info("ZK client port : " + this.zkClientPort); LOGGER.info("ZK session timeout : " + this.zkSessionTimeout); LOGGER.info("RPC timeout : " + this.rpcTimeout); LOGGER.info("Using hostname to bind to : " + this.localHost); LOGGER.info("Using worker threads : " + this.workerThreads); LOGGER.info("Listening to WAL edits from : " + this.sinceSCN); try { Configuration hbaseConf = HBaseConfiguration.create(); // enable replication to get WAL edits hbaseConf.setBoolean(HBASE_REPLICATION_CONFIG, true); // need to explicitly set the ZK host and port details - hosts separated from port - see SepModelImpl constructor source code hbaseConf.set(ZK_QUORUM_CONFIG, this.zkQuorum); hbaseConf.setInt(ZK_CLIENT_PORT_CONFIG, this.zkClientPort); hbaseConf.setInt(RPC_TIMEOUT_CONFIG, this.rpcTimeout); StringBuilder zkQuorumWithPort = new StringBuilder(); String[] zkHostsList = this.zkQuorum.split(","); for (String zkHost : zkHostsList) { zkQuorumWithPort.append(zkHost); zkQuorumWithPort.append(":"); zkQuorumWithPort.append(this.zkClientPort); zkQuorumWithPort.append(","); } LOGGER.info("ZK util connect string (host:port) : " + zkQuorumWithPort.toString()); ZooKeeperItf zk = ZkUtil.connect(zkQuorumWithPort.toString(), this.zkSessionTimeout); StringBuilder hbaseConfBuilder = new StringBuilder(); Iterator<Entry<String, String>> it = hbaseConf.iterator(); while (it.hasNext()) { Entry<String, String> entry = it.next(); if (entry.getKey().equalsIgnoreCase(HBASE_REPLICATION_CONFIG) || entry.getKey().equalsIgnoreCase(ZK_QUORUM_CONFIG) || entry.getKey().equalsIgnoreCase(ZK_CLIENT_PORT_CONFIG)) { hbaseConfBuilder.append(entry.getKey()); hbaseConfBuilder.append(":"); hbaseConfBuilder.append(entry.getValue()); hbaseConfBuilder.append(","); } } LOGGER.info("SEP Model Hbase configuration = " + hbaseConfBuilder.toString()); SepModel sepModel = new SepModelImpl(zk, hbaseConf); final String subscriptionName = this.getName(); if (!sepModel.hasSubscription(subscriptionName)) { sepModel.addSubscriptionSilent(subscriptionName); } this.sepConsumer = new SepConsumer(subscriptionName, generateSEPSCN(this.sinceSCN.get()), new RelayAppender(), this.workerThreads, this.localHost, zk, hbaseConf); this.sepConsumer.start(); } catch (Exception e) { LOGGER.error( "Error starting WAL edits consumer. Producer not started!. Error message : " + e.getMessage(), e); } } /** * The SEP EventListener that consumes {@link SepEvent} instances and appends them to the Databus event buffer after suitable conversion/interpretation. */ class RelayAppender implements EventListener { public void processEvents(List<SepEvent> sepEvents) { if (shutdownRequested.get()) { return; } long lastSavedSCN = sinceSCN.get(); eventBuffer.startEvents(); for (SepEvent sepEvent : sepEvents) { T changeEvent = sepEventMapper.mapSepEvent(sepEvent); byte[] schemaId = SchemaHelper.getSchemaId(changeEvent.getSchema().toString()); byte[] serializedEvent = serializeEvent(changeEvent); // In case of multiple region servers, events in the sepEvents can be out of order wrt of event timestamp // because of which SCN numbers will not be monotonically increasing If SCN is generated from event timestamp hence some events might get skipped // because, for databus to process the event, SCN should be monotonically increasing in the same window // Hence using eventEntryTimestamp instead of event timestamp for SCN generator long eventEntryTimestamp = System.nanoTime(); DbusEventKey eventKey = new DbusEventKey(sepEvent.getRow()); // we use the SepEvent row key as the identifier long eventScnNumber = scnGenerator.getSCN(eventEntryTimestamp, localHost); DbusEventInfo eventInfo = new DbusEventInfo(DbusOpcode.UPSERT, eventScnNumber, (short) physicalSourceStaticConfig.getId(), (short) physicalSourceStaticConfig.getId(), System.nanoTime(), (short) physicalSourceStaticConfig.getSources()[0].getId(), // here we use the Logical Source Id schemaId, serializedEvent, false, true); eventBuffer.appendEvent(eventKey, eventInfo, dbusEventsStatisticsCollector); sinceSCN.set(Math.max(lastSavedSCN, eventScnNumber)); } eventBuffer.endEvents(sinceSCN.get(), dbusEventsStatisticsCollector); try { maxScnReaderWriter.saveMaxScn(sinceSCN.get()); } catch (DatabusException e) { LOGGER.error( "Unable to persist last processed SCN. SCN value is stale. Error is : " + e.getMessage(), e); throw new PlatformException( "Unable to write last processed SCN to log. Signalling for re-delivery of WAL edits from : " + lastSavedSCN); } LOGGER.debug("Processed SEP event count : " + sepEvents.size()); } } /** * Utility to generate SEP SCN (WALEdits) from Relayer Scn. * @param relayerSCN - lastSavedSCN on the relayer. lastSavedSCN is (SepSCN<<10|running_number) * @return sepSCN */ private long generateSEPSCN(long relayerSCN) { if (relayerSCN == -1 || relayerSCN == 0) { return relayerSCN; } return relayerSCN >> 10; } /** * Interface method implementation. Stops the SEP consumer * @see com.linkedin.databus2.producers.EventProducer#shutdown() */ public void shutdown() { LOGGER.info("Shutdown has been requested. HBaseEventProducer shutting down"); this.shutdownRequested.set(true); this.sepConsumer.stop(); super.shutdown(); LOGGER.info("HBaseEventProducer shut down complete"); } /** * Interface method implementation. Returns inverted status of {@link #isRunning()} * @see com.linkedin.databus2.producers.EventProducer#isPaused() */ public boolean isPaused() { return !this.isRunning(); } /** * Interface method implementation. Returns {@link SepConsumer#isRunning()} * @see com.linkedin.databus2.producers.EventProducer#isRunning() */ public boolean isRunning() { return this.sepConsumer.isRunning(); } /** Methods that are not supported and therefore throw {@link UnsupportedOperationException}*/ public void pause() { throw new UnsupportedOperationException("'pause' is not supported on this event producer"); } public void unpause() { throw new UnsupportedOperationException("'unpause' is not supported on this event producer"); } public void waitForShutdown() throws InterruptedException, IllegalStateException { throw new UnsupportedOperationException("'waitForShutdown' is not supported on this event producer"); } public void waitForShutdown(long time) throws InterruptedException, IllegalStateException { throw new UnsupportedOperationException( "'waitForShutdown(long time)' is not supported on this event producer"); } /** Start Setter/Getter methods*/ public String getZkQuorum() { return zkQuorum; } public void setZkQuorum(String zkQuorum) { this.zkQuorum = zkQuorum; } public Integer getZkClientPort() { return zkClientPort; } public void setZkClientPort(int zkClientPort) { this.zkClientPort = zkClientPort; } public Integer getZkSessionTimeout() { return zkSessionTimeout; } public void setZkSessionTimeout(Integer zkSessionTimeout) { this.zkSessionTimeout = zkSessionTimeout; } public int getWorkerThreads() { return workerThreads; } public void setWorkerThreads(int workerThreads) { this.workerThreads = workerThreads; } public SepEventMapper<T> getSepEventMapper() { return sepEventMapper; } public void setSepEventMapper(SepEventMapper<T> sepEventMapper) { this.sepEventMapper = sepEventMapper; } public SCNGenerator getScnGenerator() { return scnGenerator; } public void setScnGenerator(SCNGenerator scnGenerator) { this.scnGenerator = scnGenerator; } public int getRpcTimeout() { return rpcTimeout; } public void setRpcTimeout(int rpcTimeout) { this.rpcTimeout = rpcTimeout; } /** End Setter/Getter methods*/ }