Java tutorial
/** * Copyright (c) MuleSoft, Inc. All rights reserved. http://www.mulesoft.com * The software in this package is published under the terms of the CPAL v1.0 * license, a copy of which has been included with this distribution in the * LICENSE.txt file. **/ /** * This file was automatically generated by the Mule Development Kit */ package com.sfelf.connectors; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.Map; import org.bson.types.BSONTimestamp; import org.mule.MessageExchangePattern; import org.mule.api.annotations.Category; import org.mule.api.annotations.Connector; import org.mule.api.annotations.Connect; import org.mule.api.annotations.Mime; import org.mule.api.annotations.Source; import org.mule.api.annotations.Transformer; import org.mule.api.annotations.ValidateConnection; import org.mule.api.annotations.ConnectionIdentifier; import org.mule.api.annotations.Disconnect; import org.mule.api.annotations.display.FriendlyName; import org.mule.api.annotations.display.Password; import org.mule.api.annotations.display.Placement; import org.mule.api.annotations.param.ConnectionKey; import org.mule.api.annotations.param.Default; import org.mule.api.annotations.param.Optional; import org.mule.api.ConnectionException; import org.mule.api.ConnectionExceptionCode; import org.mule.api.callback.SourceCallback; import org.mule.transformer.types.MimeTypes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.mongodb.BasicDBObject; import com.mongodb.Bytes; import com.mongodb.DB; import com.mongodb.DBCollection; import com.mongodb.DBCursor; import com.mongodb.DBObject; import com.mongodb.MongoClient; import com.mongodb.MongoException; import com.mongodb.ServerAddress; import com.mongodb.WriteConcern; import com.mongodb.util.JSON; /** * mongoDB Oplog Cursor Connector * * @author sfelf.com */ @Connector(name = "mongooplogcursor", schemaVersion = "1.0-SNAPSHOT", friendlyName = "Mongo Oplog Cursor", minMuleVersion = "3.3") @Category(name = "org.mule.tooling.category.endpoints", description = "Endpoints") public class mongoOplogCursorConnector { private MongoClient mongoClient; private DB oplogDB; private Boolean logLastTimestamp; private DB logDB; private String logCollection; private static final Logger LOGGER = LoggerFactory.getLogger(mongoOplogCursorConnector.class); private static final String LAST_TIMESTAMP_LOG_FIELD = "lastTimeStamp"; private static final int SORT_ORDER_DESCENDING = -1; private static final WriteConcern LOG_WRITE_CONCERN = WriteConcern.ACKNOWLEDGED; private static final String OPLOG_DB_NAME = "local"; private static final String OPLOG_COLLECTION_NAME = "oplog.rs"; private static final int NO_DOC_SLEEP_TIME = 1000; /** * Method invoked to establish the connection with the Mongo server. * * @param servers Contains the hostname and port of the mongoDB server separated by a ":" (ex. localhost:27017). * NOTE: If you are using a replica set you may pass in all the servers with each server separated * by a comma (ex. localhost:27017,localhost:27018,localhost:27019). * @param username The username to use for authentication. This should be left blank if authentication is disabled. * @param password The password to use for authentication. This should be left blank if authentication is disabled. * @param logLastTimestamp {@link Boolean} to control whether to log last timestamp. If true then DB and Collection Name must be specified. * @param logDBName The name of the db where the timestamp of the last update processed will stored/read from. If left * blank the timestamp will not be logged and the cursor will start after the last record in the oplog. * @param logCollectionName The name of the collection where the timestamp of the last update processed will stored/read from. If * left blank the timestamp will not be logged and the cursor will start after the last record in the oplog. * @throws ConnectionException If unable to establish a connection */ @Connect public void connect( @ConnectionKey @Placement(order = 1) @Default("localhost:27017,localhost:27018,localhost:27019") String servers, @ConnectionKey @Optional @Placement(order = 2) @Default("") final String username, @Password @Optional @Placement(order = 3) @Default("") final String password, @FriendlyName("Log Last Timestamp") @Placement(group = "Cache Last Timestamp", order = 4) @Default("false") final Boolean logLastTimestamp, @FriendlyName("Last Timestamp DB Name") @Optional @Placement(group = "Cache Last Timestamp", order = 5) @Default("") final String logDBName, @FriendlyName("Last Timestamp Collection Name") @Optional @Placement(group = "Cache Last Timestamp", order = 6) @Default("") final String logCollectionName) throws ConnectionException { try { this.logCollection = logCollectionName; this.logLastTimestamp = logLastTimestamp; this.mongoClient = getMongoClient(servers); this.oplogDB = getMongoDB(OPLOG_DB_NAME, username, password); if (logLastTimestamp) { this.logDB = getMongoDB(logDBName, username, password); } } catch (final MongoException e) { throw new ConnectionException(ConnectionExceptionCode.UNKNOWN, null, e.getMessage()); } catch (final UnknownHostException e) { throw new ConnectionException(ConnectionExceptionCode.UNKNOWN_HOST, null, e.getMessage()); } } /** * Disconnects the oplogDB and logDB as well as the mongoClient */ @Disconnect public void disconnect() { oplogDB = disconnectDB(oplogDB); logDB = disconnectDB(logDB); disconnectClient(); } /** * Returns true if the {@link MongoClient} connector is still open and the oplogDB is not null and logDB is not null if it was defined. * * @return {@link Boolean} */ @ValidateConnection public boolean isConnected() { LOGGER.debug("Checking if still connected:" + "\n" + "mongoClient = " + this.mongoClient + "\n" + "oplogDB = " + this.oplogDB + "\n" + "Log Last Timestamp = " + this.logLastTimestamp + "\n" + "logDB = " + this.logDB); if (this.logLastTimestamp != null && this.logLastTimestamp) { return this.oplogDB != null && this.logDB != null && this.mongoClient != null && this.mongoClient.getConnector().isOpen(); } else { return this.oplogDB != null && this.mongoClient != null && this.mongoClient.getConnector().isOpen(); } } /** * Returns the connection id * * @return {@link String} */ @ConnectionIdentifier public String connectionId() { return mongoClient == null ? "n/a" : mongoClient.toString(); } /** * <b>dbobjectToJson</b> - Convert DBObject to Json. * <p/> * {@sample.xml ../../../doc/mongoOplogCursor-connector.xml.sample mongooplogcursor:dbobjectToJson} * * @param input the input for this transformer * @return the converted string representation */ @Mime(MimeTypes.JSON) @Transformer(sourceTypes = { DBObject.class }) public static String dbobjectToJson(final DBObject input) { return JSON.serialize(input); } /** * <b>dbObjectToMap</b> - Convert a DBObject into Map. * <p/> * {@sample.xml ../../../doc/mongoOplogCursor-connector.xml.sample mongooplogcursor:dbobjectToMap} * * @param input the input for this transformer * @return the converted Map representation */ @SuppressWarnings("rawtypes") @Transformer(sourceTypes = { DBObject.class }) public static Map dbobjectToMap(final DBObject input) { return input.toMap(); } /** * <b>OplogCursor</b> - Establishes a tailable cursor to the mongoDB oplog for the specified namespace and operations * and returns the results to the Mulesoft inbound endpoint. The returned payload is a {@link DBObject}. * <p/> * Throws a {@link MongoOplogCursorException} if any unexpected issues arise while trying to * establish the tailable cursor or while processing the results from the cursor. * <p/> * {@sample.xml ../../../doc/mongoOplogCursor-connector.xml.sample mongooplogcursor:oplogCursor} * @param namespace The exact namespace (ex. database.collection) for which the endpoint should return messages when * entries are logged in the oplog for that namespace. * @param fullDocument {@link Boolean} to control if messages will include the full oplog document or just the following fields: * <ul> * <li>ts - timestamp * <li>op - operation (i=insert, u=update, d=delete) * <li>ns - namespace * <li>o._id - _id of the record inserted or updated * <li>o2._id - _id of the record updated * </ul> * @param monitorInserts {@link Boolean} to control if messages will be generated for Inserts (Optional: defaults to false) * @param monitorUpdates {@link Boolean} to control if messages will be generated for Updates (Optional: defaults to false) * @param monitorDeletes {@link Boolean} to control if messages will be generated for Deletes (Optional: defaults to false) * @param callback The callback to be called when a message is received * <p/> * @throws MongoOplogCursorException If {@link #isConnected() isConnected} == false */ @Source(primaryNodeOnly = true, exchangePattern = MessageExchangePattern.ONE_WAY) public void oplogCursor( @FriendlyName("Namespace") @Placement(group = "Query Options", order = 1) String namespace, @FriendlyName("Return full document") @Optional @Placement(group = "Query Options", order = 2) @Default("false") Boolean fullDocument, @FriendlyName("Monitor Insert Operations") @Optional @Placement(group = "Operations", order = 3) @Default("false") Boolean monitorInserts, @FriendlyName("Monitor Update Operations") @Optional @Placement(group = "Operations", order = 4) @Default("false") Boolean monitorUpdates, @FriendlyName("Monitor Delete Operations") @Optional @Placement(group = "Operations", order = 5) @Default("false") Boolean monitorDeletes, SourceCallback callback) throws MongoOplogCursorException { if (!isConnected()) { throw new MongoOplogCursorException( "Unable to initiate cursor because MongoClient is not connected: " + mongoClient); } DBCollection log = getCollection(logDB, logCollection, true); DBCollection oplog = getCollection(oplogDB, OPLOG_COLLECTION_NAME, false); BSONTimestamp lastTimestamp = getLastTimestamp(oplog, log, namespace); LOGGER.debug("Last Timestamp: " + lastTimestamp); while (!Thread.interrupted()) { DBObject query = getOplogQuery(lastTimestamp, namespace, monitorInserts, monitorUpdates, monitorDeletes); DBCursor cursor = createCursor(oplog, query, fullDocument); LOGGER.debug("New cursor: " + cursor.toString()); try { while (cursor.hasNext()) { final DBObject doc = cursor.next(); LOGGER.debug("New document: " + doc); if (doc != null) { callback.process(doc); lastTimestamp = (BSONTimestamp) doc.get("ts"); updateLastTimestampLog(log, namespace, lastTimestamp); LOGGER.debug("Updated Last Timestamp: " + lastTimestamp); } else { LOGGER.debug("Sleeping until next document ready"); Thread.sleep(NO_DOC_SLEEP_TIME); } } } catch (Exception e) { LOGGER.debug( "Caught Exception while reading from cursor: " + e.getClass() + " - " + e.getMessage()); } finally { LOGGER.debug("Closing Cursor and attempting to acquire a new cursor"); try { cursor.close(); } catch (Exception e) { LOGGER.debug("Caught Exception while closing cursor: " + e.getClass() + " - " + e.getMessage()); } } } LOGGER.debug("Thread Interrupted:" + "\n" + "log = " + log + "\n" + "oplog = " + oplog); } /** * Method invoked to establish a {@link MongoClient} to the supplied servers. * * @param servers A comma separated list of host port pairs separated by a colon (ex: localhost:27017,localhost:27018,localhost:27019). * @return {@link MongoClient} * @throws MongoException * @throws UnknownHostException */ private MongoClient getMongoClient(String servers) throws MongoException, UnknownHostException { List<ServerAddress> serverList = buildServerList(servers); MongoClient mongoClient; if (serverList.size() == 1) { mongoClient = new MongoClient(serverList.get(0)); } else { mongoClient = new MongoClient(serverList); } LOGGER.debug("Connected to: " + serverList.toString()); LOGGER.debug("Mongo configured: " + mongoClient.toString()); return mongoClient; } /** * Method invoked to establish a {@link DB} to the supplied database which is authenticated with username and password if supplied * * @param database The name of the database * @param username The username to use for authentication * @param password The password to use for authentication * @return {@link DB} */ private DB getMongoDB(String database, String username, String password) { DB db = null; if (database != null) { db = this.mongoClient.getDB(database); LOGGER.debug("Mongo DB configured: " + db.toString()); performAuthentication(db, username, password); } return db; } /** * Method to parse and build a list of {@link ServerAddress} from the supplied servers string * * @param servers A comma separated list of host port pairs separated by a colon (ex: localhost:27017,localhost:27018,localhost:27019). * @return {@link List}<{@link ServerAddress}> * @throws UnknownHostException */ private List<ServerAddress> buildServerList(String servers) throws UnknownHostException { List<ServerAddress> serverList = new ArrayList<ServerAddress>(); String[] serverArray = servers.split(","); for (int i = 0; i < serverArray.length; i++) { LOGGER.debug("Adding mongo server: " + serverArray[i]); if (!serverArray[i].contains(":")) { throw new UnknownHostException("Invalid host:port pair in servers string: " + serverArray[i]); } String[] server = serverArray[i].split(":", 2); int port = 0; try { port = Integer.parseInt(server[1]); } catch (NumberFormatException e) { throw new UnknownHostException( "Invalid port number in the host:port pair in servers string: " + serverArray[i]); } serverList.add(new ServerAddress(server[0], port)); } return serverList; } /** * Method used to authenticate the supplied db with the supplied username and password. If either the supplied username or * password are null then authentication will not be attempted. * * @param db The {@link DB} to perform the authentication on * @param username The username to use for authentication * @param password The password to use for authentication * @throws MongoException * @throws IllegalStateException */ private void performAuthentication(DB db, String username, String password) throws MongoException, IllegalStateException { if (username != null && password != null) { LOGGER.debug("Attempting to authenticate with: " + username); db.authenticate(username, password.toCharArray()); LOGGER.debug("Authenticated with: " + username); } } /** * Disconnects the supplied db by calling the cleanCursors() and requestDone() methods. * * @param db The {@link DB} to disconnect * @return A null {@link DB} */ private DB disconnectDB(DB db) { LOGGER.debug("Disconnecting database:" + "\n" + "db = " + db); if (db != null) { try { db.cleanCursors(true); } catch (Exception e) { LOGGER.warn("Failed to clean cursors for " + db.toString(), e); } try { db.requestDone(); } catch (Exception e) { LOGGER.warn("Failed to end the current request for " + db.toString(), e); } } return null; } /** * Closes the {@link MongoClient} specified in the class parameter mongoClient and sets the parameter to null. */ private void disconnectClient() { LOGGER.debug("Disconnecting connection:" + "\n" + "mongoClient = " + this.mongoClient); if (this.mongoClient != null) { try { this.mongoClient.close(); } catch (Exception e) { LOGGER.warn("Failed to close the current MongoClient connection.", e); } this.mongoClient = null; } } /** * Returns a {@link DBCollection} in the specified {@link DB} with the specified collectionName. Throws an exception * if the collection doesn't exist in the specified {@link DB} and addNewCollection is false. * * @param db The {@link DB} containing the collection * @param collectionName The name of the collection * @param addNewCollection A {@link Boolean} which specifies if the collection should be created if it doesn't exist in the specified db. * @return {@link DBCollection} * @throws MongoOplogCursorException */ private DBCollection getCollection(DB db, String collectionName, Boolean addNewCollection) throws MongoOplogCursorException { DBCollection collection = null; if (db != null && collectionName != null) { LOGGER.debug( "Getting collection: " + db.getName() + "." + collectionName + " Add new: " + addNewCollection); if (db.collectionExists(collectionName) || addNewCollection) { collection = db.getCollection(collectionName); LOGGER.debug("Collection: " + collection); } else { throw new MongoOplogCursorException("Unable to establish cursor as the " + collectionName + " collection doesn't exist in the " + db.getName() + " db."); } } return collection; } /** * Returns a {@link BSONTimestamp} of either the timestamp in the specified log collection for the specified namespace or * the timestamp of the newest entry in the oplog.rs. * * @param oplog The {@link DBCollection} containing the oplog (local.oplog.rs) * @param log The {@link DBCollection} containing a log of the timestamp for the last entry processed for the specified namespace * @param namespace The namespace for which the last timestamp is being requested * @return {@link BSONTimestamp} * @throws MongoOplogCursorException */ private BSONTimestamp getLastTimestamp(DBCollection oplog, DBCollection log, String namespace) throws MongoOplogCursorException { BSONTimestamp lastTimestamp = null; if (log != null) { DBObject queryParams = new BasicDBObject("_id", namespace); DBObject result = log.findOne(queryParams); if (result != null && result.get(LAST_TIMESTAMP_LOG_FIELD) != null) { lastTimestamp = (BSONTimestamp) result.get(LAST_TIMESTAMP_LOG_FIELD); } } if (lastTimestamp == null) { DBObject orderBy = new BasicDBObject("$natural", SORT_ORDER_DESCENDING); DBCursor cursor = oplog.find().sort(orderBy); if (cursor.hasNext()) { lastTimestamp = (BSONTimestamp) cursor.next().get("ts"); } else { throw new MongoOplogCursorException("Unable to establish cursor as the oplog is empty"); } } return lastTimestamp; } /** * Returns a {@link DBObject} like { "ts": lastTimestamp, "ns": namespace, op: ["i", "u", "d"] } with the contents of the op array * determined by the supplied values for inserts, updates and deletes respectively. * * @param lastTimestamp The value for the ts key * @param namespace The vale for the ns key * @param inserts {@link Boolean} specifying if "i" should be in the op array * @param updates {@link Boolean} specifying if "u" should be in the op array * @param deletes {@link Boolean} specifying if "d" should be in the op array * @return {@link DBObject} * @throws MongoOplogCursorException if ( inserts && updates && deletes ) == false */ private DBObject getOplogQuery(BSONTimestamp lastTimestamp, String namespace, Boolean inserts, Boolean updates, Boolean deletes) throws MongoOplogCursorException { List<String> opList = new ArrayList<String>(); if (inserts) { opList.add("i"); } if (updates) { opList.add("u"); } if (deletes) { opList.add("d"); } if (opList.size() == 0) { throw new MongoOplogCursorException( "Unable to establish cursor as no operation was selected to monitor"); } DBObject query = new BasicDBObject(); query.put("ts", new BasicDBObject("$gt", lastTimestamp)); query.put("ns", namespace); query.put("op", new BasicDBObject("$in", opList)); return query; } /** * Returns {@link DBCursor} for the specified oplog collection with the specified query applied. If fullDocument * is false then only the fields specified by {@link #getOplogFields() getOplogFields()} will be returned. * * @param oplog {@link DBCollection} specifying the collection to use when creating the cursor * @param query {@link DBObject} specifying the query to use when creating the cursor * @param fullDocument {@link Boolean} specifying if the full document should be returned or the fields specified by {@link #getOplogFields() getOplogFields()} * @return {@link DBCursor} */ private DBCursor createCursor(DBCollection oplog, DBObject query, Boolean fullDocument) { DBCursor cursor; if (fullDocument) { cursor = oplog.find(query); } else { DBObject fields = getOplogFields(); cursor = oplog.find(query, fields); } return cursor.addOption(Bytes.QUERYOPTION_TAILABLE).addOption(Bytes.QUERYOPTION_AWAITDATA); } /** * Updates the document with "_id" = namespace in the {@link DBCollection} specified by log setting the field * {@value #LAST_TIMESTAMP_LOG_FIELD} = lastTimestamp if {@link #logLastTimestamp} == true. * * @param log {@link DBCollection} to update * @param namespace value of the _id field in the document to update * @param lastTimestamp value to assign to the {@value #LAST_TIMESTAMP_LOG_FIELD} field */ private void updateLastTimestampLog(DBCollection log, String namespace, BSONTimestamp lastTimestamp) { if (this.logLastTimestamp) { DBObject query = new BasicDBObject("_id", namespace); DBObject update = new BasicDBObject("$set", new BasicDBObject(LAST_TIMESTAMP_LOG_FIELD, lastTimestamp)); log.update(query, update, true, false, LOG_WRITE_CONCERN); } } /** * Returns a {@link DBObject} with the following value { "ts":1, "op":1, "ns":1, "o._id":1, "o2._id":1 } * * @return {@link DBObject} */ private DBObject getOplogFields() { DBObject fields = new BasicDBObject(); fields.put("ts", 1); fields.put("op", 1); fields.put("ns", 1); fields.put("o._id", 1); fields.put("o2._id", 1); return fields; } /** * MongoOplogCursorException * * @author sfelf.com */ public class MongoOplogCursorException extends Exception { private static final long serialVersionUID = -375024617796323956L; public MongoOplogCursorException(final String message) { super(message); } public MongoOplogCursorException(final String message, final Throwable throwable) { super(message, throwable); } } }