Java tutorial
/* * Copyright 2010-2013 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package gemlite.core.internal.db; import gemlite.core.internal.domain.DomainRegistry; import gemlite.core.internal.domain.IMapperTool; import gemlite.core.util.LogUtil; import java.io.FileInputStream; import java.sql.BatchUpdateException; import java.sql.Connection; import java.sql.Driver; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.SQLIntegrityConstraintViolationException; import java.sql.SQLNonTransientConnectionException; import java.sql.SQLNonTransientException; import java.sql.SQLTransientException; import java.sql.Statement; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.ConcurrentSkipListMap; import java.util.logging.Level; import java.util.regex.Pattern; import org.apache.commons.beanutils.PropertyUtils; import org.apache.logging.log4j.Logger; import com.gemstone.gemfire.DataSerializable; import com.gemstone.gemfire.cache.Operation; import com.gemstone.gemfire.cache.RegionDestroyedException; import com.gemstone.gemfire.cache.asyncqueue.AsyncEvent; import com.gemstone.gemfire.cache.asyncqueue.AsyncEventListener; /** * region???? ????? ?? */ @SuppressWarnings({ "rawtypes", "unchecked" }) public class DBSynchronizer implements AsyncEventListener { protected String dbUrl; protected String userName; protected String passwd; protected String errorFile; /** * ??,? */ protected int numErrorTries = 0; /** * ? */ protected static final int DEFAULT_ERROR_TRIES = 3; /** * ??? */ protected String driverClass; protected Driver driver; /** * the current JDBC connection being used by this instance */ protected Connection conn; /** true if this instance has been closed or not initialized */ protected volatile boolean shutDown = true; protected final AsyncEventHelper helper = AsyncEventHelper.newInstance(); protected final Logger logger = LogUtil.getCoreLog(); protected final HashMap<String, PreparedStatement> insertStmntMap = new HashMap<String, PreparedStatement>(); protected final HashMap<String, PreparedStatement> updtStmntMap = new HashMap<String, PreparedStatement>(); protected final HashMap<String, PreparedStatement> deleteStmntMap = new HashMap<String, PreparedStatement>(); // keys used in external property file protected static final String DBDRIVER = "driver"; protected static final String DBURL = "url"; protected static final String USER = "user"; protected static final String PASSWORD = "password"; protected static final String SECRET = "secret"; protected static final String TRANSFORMATION = "transformation"; protected static final String KEYSIZE = "keysize"; protected static final String ERRORFILE = "errorfile"; protected static final String ERRORTRIES = "errortries"; // Log strings protected static final String DB_SYNCHRONIZER__1 = "DBSynchronizer::" + "processEvents: Exception while fetching prepared statement " + "for event '%s': %s"; protected static final String DB_SYNCHRONIZER__2 = "DBSynchronizer::" + "processEvents: Unexpected Exception occured while processing " + "Events. The list of unprocessed events is: %s. " + "Attempt will be made to rollback the changes."; protected static final String DB_SYNCHRONIZER__3 = "DBSynchronizer::" + "processEvents: Operation failed for event '%s' " + "due to exception: %s"; protected static final String DB_SYNCHRONIZER__4 = "DBSynchronizer::" + "closeStatements: Exception in closing prepared statement " + "with DML string: %s"; protected static final String DB_SYNCHRONIZER__5 = "DBSynchronizer::" + "close: Exception in closing SQL Connection: %s"; protected static final String DB_SYNCHRONIZER__6 = "DBSynchronizer::" + "init: Exception while initializing connection for driver class '%s' " + "and db url = %s"; protected static final String DB_SYNCHRONIZER__7 = "DBSynchronizer::" + "processEvents: Exception occured while committing '%s' " + "to external DB: %s"; protected static final String DB_SYNCHRONIZER__8 = "DBSynchronizer::init" + ": Illegal format of init string '%s', expected <driver>,<URL>,..."; protected static final String DB_SYNCHRONIZER__9 = "DBSynchronizer::" + "init: Exception in loading properties file '%s' for initialization"; protected static final String DB_SYNCHRONIZER__10 = "DBSynchronizer::" + "init: missing Driver or URL properties in file '%s'"; protected static final String DB_SYNCHRONIZER__11 = "DBSynchronizer::" + "init: unknown property '%s' in file '%s'"; protected static final String DB_SYNCHRONIZER__12 = "DBSynchronizer::" + "init: both password and secret properties specified in file '%s'"; protected static final String DB_SYNCHRONIZER__13 = "DBSynchronizer::" + "init: initialized with URL '%s' using driver class '%s'"; /** * Holds event that failed to be applied to underlying database and the time * of failure */ private static class ErrorEvent implements Comparable { AsyncEvent ev; long errortime; @Override public int compareTo(Object o) { ErrorEvent ee = (ErrorEvent) o; // If events are equal, nevermind the time, else allow sorting by // failure time, earlier first. if (ee.ev.equals(this.ev)) { return 0; } else if (ee.errortime > this.errortime) { return -1; } else { return 1; } } } /** keeps track of the retries that have been done for an event */ protected final ConcurrentSkipListMap<ErrorEvent, Object[]> errorTriesMap = new ConcurrentSkipListMap<ErrorEvent, Object[]>(); /** * Enumeration that defines the action to be performed in case an exception * is received during processing by * {@link DBSynchronizer#processEvents(List)} */ protected static enum SqlExceptionHandler { /** * ignore the exception and continue to process other events in the * current batch */ IGNORE { @Override public void execute(DBSynchronizer synchronizer) { // No -op synchronizer.logger.info("DBSynchronizer::Ignoring error"); } @Override public boolean breakTheLoop() { return false; } }, /** * ignore the exception and break the current batch of events being * processed */ IGNORE_BREAK_LOOP { @Override public boolean breakTheLoop() { return true; } @Override public void execute(DBSynchronizer synchronizer) { // No -op } }, /** * create a new database connection since the current one is no longer * usable */ REFRESH { @Override public void execute(DBSynchronizer synchronizer) { synchronized (synchronizer) { try { if (!synchronizer.conn.isClosed()) { if (synchronizer.logger.isInfoEnabled()) { synchronizer.logger .info("DBSynchronizer::" + "SqlExceptionHandler: before rollback"); } // For safe side just roll back the transaction so // far synchronizer.conn.rollback(); if (synchronizer.logger.isInfoEnabled()) { synchronizer.logger .info("DBSynchronizer::" + "SqlExceptionHandler: after rollback"); } } } catch (SQLException sqle) { synchronizer.helper.log(synchronizer.logger, Level.WARNING, sqle, "DBSynchronizer::SqlExceptionHandler: " + "could not successfully rollback"); } synchronizer.basicClose(); if (!synchronizer.shutDown) { // Do not recreate the connection in case of shutdown synchronizer.logger.info("DBSynchronizer::Attempting to reconnect to database"); synchronizer.instantiateConnection(); } } } }, /** close the current connection */ CLEANUP { @Override public void execute(DBSynchronizer synchronizer) { synchronized (synchronizer) { try { if (synchronizer.conn != null && !synchronizer.conn.isClosed()) { if (synchronizer.logger.isInfoEnabled()) { synchronizer.logger .info("DBSynchronizer::" + "SqlExceptionHandler: before rollback"); } // For safeside just roll back the transactions so // far synchronizer.conn.rollback(); if (synchronizer.logger.isInfoEnabled()) { synchronizer.logger .info("DBSynchronizer::" + "SqlExceptionHandler: after rollback"); } } } catch (SQLException sqle) { synchronizer.helper.log(synchronizer.logger, Level.WARNING, sqle, "DBSynchronizer::SqlExceptionHandler: " + "could not successfully rollback"); } synchronizer.basicClose(); } } }; /** * execute an action specified by different enumeration values after an * unexpected exception is received by * {@link DBSynchronizer#processEvents(List)} */ public abstract void execute(DBSynchronizer synchronizer); /** * Returns true if processing for the current batch of events has to be * terminated for the current exception. Default handling is to return * true, unless specified otherwise by an enumerated action. */ public boolean breakTheLoop() { return true; } } /** * Close this {@link DBSynchronizer} instance. * * To prevent a possible concurrency issue between closing thread & the * processor thread, access to this method is synchronized on 'this' */ public synchronized void close() { // Flush any pending error events to XML log this.flushErrorEventsToLog(); this.shutDown = true; this.basicClose(); this.helper.close(); } /** * Basic actions to be performed to close the {@link DBSynchronizer} * instance though the instance will itself not be marked as having shut * down. * * To prevent a possible concurrency issue between closing thread & the * processor thread, access to this method is synchronized on 'this' */ public final synchronized void basicClose() { closeStatements(this.insertStmntMap); closeStatements(this.updtStmntMap); closeStatements(this.deleteStmntMap); try { if (this.conn != null && !this.conn.isClosed()) { this.conn.close(); } } catch (SQLException sqle) { if (logger.isInfoEnabled()) { helper.logFormat(logger, Level.INFO, sqle, DB_SYNCHRONIZER__5, this.conn); } } } /* * to prevent a possible concurrency issue between closing thread & the * processor thread, access to this method is synchronized on 'this' */ protected void closeStatements(Map<String, PreparedStatement> psMap) { Iterator<Map.Entry<String, PreparedStatement>> itr = psMap.entrySet().iterator(); while (itr.hasNext()) { Map.Entry<String, PreparedStatement> entry = itr.next(); try { entry.getValue().close(); } catch (SQLException sqle) { if (logger.isInfoEnabled()) { helper.logFormat(logger, Level.INFO, sqle, DB_SYNCHRONIZER__4, entry.getKey()); } } finally { itr.remove(); } } } public void init(String driverClass, String dbUrl, String userName, String passwd) { this.driverClass = driverClass; this.dbUrl = dbUrl; this.userName = userName; this.passwd = passwd; this.initConnection(); } /** * Initialize this {@link DBSynchronizer} instance, creating a new JDBC * connection to the backend database as per the provided parameter.<BR> * * The recommended format of the parameter string is: <BR> * * file=<path> <BR> * * The file is a properties file specifying the driver, JDBC URL, user and * password.<BR> * * Driver=<driver-class><BR> * URL=<JDBC URL><BR> * User=<user name><BR> * <BR> * Secret=<encrypted password><BR> * Transformation=<transformation for the encryption cipher><BR> * KeySize=<size of the private key to use for encryption><BR> * -- OR --<BR> * Password=<password><BR> * * The password provided in the "Secret" property should be an encrypted one * generated using the "gfxd encrypt-password external" command, else the * "Password" property can be used to specify the password in plain-text. * The "Transformation" and "KeySize" properties optionally specify the * transformation and key size used for encryption else the defaults are * used ("AES" and 128 respectively). User and password are optional and * when not provided then JDBC URL will be used as is for connection. * * The above properties may also be provided inline like below:<BR> * <BR> * <driver-class>,<JDBC * URL>[,<user>[,<password>|secret * =<secret>][,transformation=<transformation>][,keysize=<key * size>]<BR> * <BR> * The user and password parts are optional and can be possibly embedded in * the JDBC URL itself. The password can be encrypted one generated using * the "gfxd encrypt-password external" command in which case it should be * prefixed with "secret=". It can also specify the transformation and * keysize using the optional "transformation=..." and "keysize=..." * properties. */ public void init(String initParamStr) { this.driver = null; this.driverClass = null; this.dbUrl = null; this.userName = null; this.passwd = null; this.numErrorTries = 0; // check the new "file=<properties file>" option first if (initParamStr.startsWith("file=")) { String propsFile = initParamStr.substring("file=".length()); FileInputStream fis = null; final Properties props = new Properties(); try { fis = new FileInputStream(propsFile); props.load(fis); } catch (Exception e) { throw helper.newRuntimeException(String.format(DB_SYNCHRONIZER__9, propsFile), e); } finally { try { if (fis != null) { fis.close(); } } catch (Exception e) { // ignored } } try { for (Map.Entry<Object, Object> entry : props.entrySet()) { String key = ((String) entry.getKey()).trim(); String value = ((String) entry.getValue()).trim(); if (DBDRIVER.equalsIgnoreCase(key)) { this.driverClass = value; } else if (DBURL.equalsIgnoreCase(key)) { this.dbUrl = value; } else if (USER.equalsIgnoreCase(key)) { this.userName = value; } else if (PASSWORD.equalsIgnoreCase(key)) { this.passwd = value; } else if (ERRORFILE.equalsIgnoreCase(key)) { this.errorFile = value; } else if (ERRORTRIES.equalsIgnoreCase(key)) { this.numErrorTries = Integer.parseInt(value); } else { throw new IllegalArgumentException(String.format(DB_SYNCHRONIZER__11, key, propsFile)); } } } catch (IllegalArgumentException e) { throw e; } catch (Exception e) { throw helper.newRuntimeException(String.format(DB_SYNCHRONIZER__9, propsFile), e); } if (this.driverClass == null || this.driverClass.length() == 0 || this.dbUrl == null || this.dbUrl.length() == 0) { throw new IllegalArgumentException(String.format(DB_SYNCHRONIZER__10, propsFile)); } } else { inlineInit(initParamStr); } // helper.createEventErrorLogger(errorFile); this.initConnection(); } protected void inlineInit(String initParamStr) { logger.info("DBSynchronizer::Inline init parameters:" + initParamStr); String[] params = initParamStr.split(","); if (params.length < 2) { throw new IllegalArgumentException(String.format(DB_SYNCHRONIZER__8, initParamStr)); } int paramNo = 1; for (String param : params) { param = param.trim(); StringBuilder value = new StringBuilder(); if (isArgPresent(param, DBDRIVER, value)) { this.driverClass = value.toString().trim(); } else if (isArgPresent(param, DBURL, value)) { this.dbUrl = value.toString().trim(); } else if (isArgPresent(param, USER + '=', value)) { this.userName = value.toString().trim(); } else if (isArgPresent(param, PASSWORD + '=', value)) { this.passwd = value.toString().trim(); } else if (isArgPresent(param, ERRORFILE + '=', value)) { this.errorFile = value.toString().trim(); } else if (isArgPresent(param, ERRORTRIES + '=', value)) { this.numErrorTries = Integer.parseInt(value.toString()); } else if (paramNo == 1) { // Assume this is the driver name this.driverClass = param.trim(); } else if (paramNo == 2) { // Assume this is the db url this.dbUrl = param.trim(); } else if (paramNo == 3) { // The third param is expected to be username if not explicitly // provided. this.userName = param.trim(); } else if (paramNo == 4) { this.passwd = param.trim(); } ++paramNo; } } protected boolean isArgPresent(String s, String prefix, StringBuilder extracted) { if ((s.length() > prefix.length() && prefix.equalsIgnoreCase(s.substring(0, prefix.length())))) { extracted.append(s.substring(prefix.length())); return true; } return false; } protected String trimIgnoreCase(String s, String prefix) { if (s.length() > prefix.length() && prefix.equalsIgnoreCase(s.substring(0, prefix.length()))) { return s.substring(prefix.length()); } else { return null; } } protected synchronized void initConnection() { String maskedPasswordDbUrl = null; if (this.dbUrl != null) { maskedPasswordDbUrl = maskPassword(this.dbUrl); } try { Class.forName(this.driverClass).newInstance(); this.driver = DriverManager.getDriver(this.dbUrl); } catch (Exception e) { throw helper.newRuntimeException( String.format(DB_SYNCHRONIZER__6, this.driverClass, maskedPasswordDbUrl), e); } this.instantiateConnection(); if (this.logger.isInfoEnabled()) { this.helper.logFormat(this.logger, Level.INFO, null, DB_SYNCHRONIZER__13, maskedPasswordDbUrl, this.driverClass); } this.shutDown = false; } protected synchronized void instantiateConnection() { if (this.driver == null) { initConnection(); return; } String maskedPasswordDbUrl = null; try { // use Driver directly for connect instead of looping through all // drivers as DriverManager.getConnection() would do, to avoid // hitting any broken drivers in the process (vertica driver is // known to // fail in acceptsURL with this set of properties) final Properties props = new Properties(); // the user/password property names are standard ones also used by // DriverManager.getConnection(String, String, String) itself, so // will work for all drivers if (this.userName != null) { props.put("user", this.userName); } if (this.passwd != null) { props.put("password", this.passwd); } this.conn = this.driver.connect(this.dbUrl, props); // null to GC password as soon as possible props.clear(); try { // try to set the default isolation to at least READ_COMMITTED // need it for proper HA handling if (this.conn.getTransactionIsolation() < Connection.TRANSACTION_READ_COMMITTED && this.conn .getMetaData().supportsTransactionIsolationLevel(Connection.TRANSACTION_READ_COMMITTED)) { this.conn.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); if (this.dbUrl != null) { maskedPasswordDbUrl = maskPassword(this.dbUrl); } logger.info("explicitly set the transaction isolation level to " + "READ_COMMITTED for URL: " + maskedPasswordDbUrl); } } catch (SQLException sqle) { // ignore any exception here } this.conn.setAutoCommit(false); this.shutDown = false; } catch (Exception e) { if (this.dbUrl != null) { maskedPasswordDbUrl = maskPassword(this.dbUrl); } // throttle retries for connection failures try { Thread.sleep(200); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } throw helper.newRuntimeException( String.format(DB_SYNCHRONIZER__6, this.driverClass, maskedPasswordDbUrl), e); } } /** mask the known password patterns from URL for exception/log messages */ protected static final String maskPassword(final String dbUrl) { String maskedPasswordDbUrl = Pattern.compile("(password|passwd|pwd|secret)=[^;]*", Pattern.CASE_INSENSITIVE) .matcher(dbUrl).replaceAll("$1=***"); return maskedPasswordDbUrl; } public boolean processEvents(List<AsyncEvent> events) { if (this.shutDown) { return false; } boolean completedSucessfully = false; String listOfEventsString = null; // The retval will be considered true only if the list was iterated // completely. If the List iteration was incomplete we will return // false so that the events are not removed during failure. // As for individual events, they can get exceptions due to constraint // violations etc but will not cause return value to be false. Statement stmt = null; PreparedStatement ps = null; // keep track of the previous prepared statement in case we can optimize // by create a batch when the previous and current statements match PreparedStatement prevPS = null; AsyncEvent prevEvent = null; boolean prevPSHasBatch = false; Iterator<AsyncEvent> itr = events.iterator(); AsyncEvent event = null; String eventString = null; String prevEventStr = null; try { while (!(completedSucessfully = !itr.hasNext())) { event = itr.next(); Operation operation = event.getOperation(); if (logger.isDebugEnabled()) { eventString = event.toString(); if (prevEvent != null) { prevEventStr = prevEvent.toString(); } logger.info("DBSynchronizer::processEvents :processing PK based " + "event=" + eventString + " AsyncEvent Operation=" + operation); } else { eventString = null; prevEventStr = null; } try { if (operation.isPutAll() || operation.isCreate()) ps = getExecutableInsertPrepStmntPKBased(event, prevPS); else if (operation.isUpdate()) ps = getExecutableUpdatePrepStmntPKBased(event, prevPS); else if (operation.isDestroy()) ps = getExecutableDeletePrepStmntPKBased(event, prevPS); else { logger.error("DBSynchronizer::processEvents: unexpected " + "eventType " + operation + " for " + event); continue; } } catch (SQLException sqle) { SqlExceptionHandler handler = handleSQLException(sqle, DB_SYNCHRONIZER__1, null, event, eventString, logger, true); if (handler.breakTheLoop()) { break; } } catch (RegionDestroyedException rde) { if (logger.isInfoEnabled()) { logger.info("DBSynchronizer::processEvents: WBCLEvent " + event + " will be discarded as the underlying region " + "for the table has been destroyed"); } continue; } if (logger.isDebugEnabled()) { if (eventString == null) { eventString = event.toString(); } logger.debug("DBSynchronizer::processEvents: Statement=" + (ps != null ? ps : stmt) + " for event=" + eventString); } try { int num; if (prevPS != null && prevPS != ps) { try { if (prevPSHasBatch) { prevPS.addBatch(); if (logger.isDebugEnabled()) { logger.info("DBSynchronizer::processEvents executing " + "batch statement for prepared statement=" + prevPS + " for event=" + prevEventStr); } final int[] res = prevPS.executeBatch(); num = res.length; prevPSHasBatch = false; } else { num = prevPS.executeUpdate(); } if (logger.isDebugEnabled()) { logger.info("DBSynchronizer::processEvents total num rows " + "modified=" + num + " for prepared statement=" + prevPS + " for event=" + prevEventStr); } // clear event from failure map if present helper.removeEventFromFailureMap(prevEvent); } catch (SQLException sqle) { if (prevPSHasBatch) { try { prevPS.clearBatch(); } catch (SQLException e) { // ignored } prevPSHasBatch = false; } SqlExceptionHandler handler = handleSQLException(sqle, DB_SYNCHRONIZER__3, prevPS, prevEvent, prevEventStr, logger, false); if (handler.breakTheLoop()) { break; } prevPS = null; prevEvent = null; prevPSHasBatch = false; } } // in case previous prepared statement matches the current // one, // it will already be added as a batch when setting the // arguments // by AsyncEventHelper#setColumnInPrepStatement() else if (prevPS != null && ps != null) { prevPSHasBatch = true; if (logger.isDebugEnabled()) { logger.info("DBSynchronizer::processEvents added new row " + "as a batch for prepared statement=" + ps + " for event=" + eventString); } } prevPS = ps; prevEvent = event; } catch (SQLException sqle) { if (prevPS != null && prevPSHasBatch) { try { prevPS.clearBatch(); } catch (SQLException e) { // ignored } } SqlExceptionHandler handler = handleSQLException(sqle, DB_SYNCHRONIZER__3, ps != null ? ps : stmt, event, eventString, logger, false); if (handler.breakTheLoop()) { break; } } } // end of while (event list processing loop) // now handle the last statement in the above loop since it is still // pending due to anticipated batching if (completedSucessfully) { try { if (logger.isInfoEnabled()) { if (listOfEventsString == null) { listOfEventsString = events.toString(); } logger.info("DBSynchronizer::processEvents: " + "before commit of events=" + listOfEventsString); } int num; // first the case when the previous statement was a batched // one // so add current one as batch and execute if (prevPSHasBatch) { ps.addBatch(); if (logger.isDebugEnabled()) { logger.info("DBSynchronizer::processEvents executing batch " + "statement for prepared statement=" + ps + " for event=" + eventString); } final int[] res = ps.executeBatch(); num = res.length; } // next the case of a non BULK_INSERT operation; // BULK_INSERT operations are always executed as a single // batch // by itself, so will never reach here else if (ps != null) { num = ps.executeUpdate(); if (event != null) { // clear event from failure map if present helper.removeEventFromFailureMap(event); } } else { num = 0; } // clear event from failure map if present helper.removeEventFromFailureMap(event); if (logger.isDebugEnabled()) { if (ps != null) { logger.info("DBSynchronizer::processEvents num rows modified=" + num + " for prepared statement=" + ps + " for event=" + eventString); } } this.conn.commit(); if (logger.isInfoEnabled()) { if (listOfEventsString == null) { listOfEventsString = events.toString(); } logger.info("DBSynchronizer::processEvents: " + "committed successfully for events=" + listOfEventsString); } } catch (SQLException sqle) { if (ps != null && prevPSHasBatch) { try { ps.clearBatch(); } catch (SQLException e) { // ignored } } SqlExceptionHandler handler = handleSQLException(sqle, DB_SYNCHRONIZER__7, ps != null ? ps : stmt, event, eventString, logger, true); if (handler != SqlExceptionHandler.IGNORE) { completedSucessfully = false; } } } } catch (Exception e) { if (logger != null && logger.isErrorEnabled() && !(event != null && helper.skipFailureLogging(event))) { StringBuilder sb = new StringBuilder(); if (event != null) { if (eventString == null) { eventString = event.toString(); } sb.append("[FAILED: ").append(eventString).append(" ]"); } while (itr.hasNext()) { sb.append("[ ").append(itr.next().toString()).append(" ]"); } helper.logFormat(logger, Level.SEVERE, e, DB_SYNCHRONIZER__2, sb.toString()); } SqlExceptionHandler.CLEANUP.execute(this); completedSucessfully = false; } if (completedSucessfully) { // on successful completion, log any pending errors to XML file; // when // unsuccessful then we know that batch will be retried so don't log // in // that case else it can get logged multiple times // clear event from failure map if present flushErrorEventsToLog(); } if (logger.isDebugEnabled()) { logger.info("DBSynchronizer::processEvents: processed " + events.size() + " events, success=" + completedSucessfully); } return completedSucessfully; } private void flushErrorEventsToLog() { Iterator<ErrorEvent> it = errorTriesMap.keySet().iterator(); while (it.hasNext()) { ErrorEvent ee = it.next(); Object[] tries = errorTriesMap.get(ee); if (tries != null && tries[1] != null) { try { helper.log(logger, Level.SEVERE, (SQLException) tries[1], ((SQLException) tries[1]).getMessage()); } catch (Exception e) { // failed to even log the exception if (logger.isWarnEnabled()) { helper.log(logger, Level.WARNING, e, e.getMessage()); } } } } errorTriesMap.clear(); } /** * Get or create a {@link PreparedStatement} for an insert operation. */ protected PreparedStatement getExecutableInsertPrepStmntPKBased(AsyncEvent pkEvent, PreparedStatement prevPS) throws SQLException { final String regionName = pkEvent.getRegion().getName(); PreparedStatement ps = this.insertStmntMap.get(regionName); IMapperTool tool = DomainRegistry.getMapperTool(regionName); String tableName = DomainRegistry.regionToTable(regionName); List<String> valueFields = tool.getValueFieldNames(); if (ps == null) { final String dmlString = AsyncEventHelper.getInsertString(tableName, valueFields); if (logger.isDebugEnabled()) { logger.info("DBSynchronizer::getExecutableInsertPrepStmntPKBased: " + "preparing '" + dmlString + "' for event: " + pkEvent); } ps = conn.prepareStatement(dmlString); this.insertStmntMap.put(tableName, ps); } else if (prevPS == ps) { // add a new batch of values ps.addBatch(); } int paramIndex = 1; Class valueClass = tool.getValueClass(); for (int colIdx = 0; colIdx < valueFields.size(); colIdx++) { String field = valueFields.get(colIdx); try { Map map = PropertyUtils.describe(pkEvent.getDeserializedValue()); Object val = map.get(field); String type = valueClass.getDeclaredField(field).getType().getName(); helper.setColumnInPrepStatement(type, val, ps, this, paramIndex); } catch (Exception e) { throw new SQLException(e); } paramIndex++; } return ps; } /** * Get or create a {@link PreparedStatement} for a primary key based delete * operation. */ protected PreparedStatement getExecutableDeletePrepStmntPKBased(AsyncEvent pkEvent, PreparedStatement prevPS) throws SQLException { final String regionName = pkEvent.getRegion().getName(); PreparedStatement ps = this.deleteStmntMap.get(regionName); IMapperTool tool = DomainRegistry.getMapperTool(regionName); String tableName = DomainRegistry.regionToTable(regionName); List<String> keyFields = tool.getKeyFieldNames(); if (ps == null) { final String dmlString = AsyncEventHelper.getDeleteString(tableName, keyFields); if (logger.isDebugEnabled()) { logger.info("DBSynchronizer::getExecutableInsertPrepStmntPKBased: preparing '" + dmlString + "' for event: " + pkEvent); } ps = conn.prepareStatement(dmlString); this.deleteStmntMap.put(regionName, ps); } else if (prevPS == ps) { // add a new batch of values ps.addBatch(); } //key? if (pkEvent.getKey() instanceof DataSerializable) setKeysInPrepStatement(pkEvent.getKey(), keyFields, tool.getValueClass(), ps, 1); else setSingleKeysInPrepStatement(pkEvent.getKey(), tool.getKeyClass(), ps, 1); return ps; } /** * Get or create a {@link PreparedStatement} for a primary key based update * operation. */ protected PreparedStatement getExecutableUpdatePrepStmntPKBased(AsyncEvent pkEvent, PreparedStatement prevPS) throws SQLException { final String regionName = pkEvent.getRegion().getName(); IMapperTool tool = DomainRegistry.getMapperTool(regionName); String tableName = DomainRegistry.regionToTable(regionName); List<String> valueFields = tool.getValueFieldNames(); final int numUpdatedCols = valueFields.size(); StringBuilder searchKeyBuff = new StringBuilder(tableName); int paramIndex; for (paramIndex = 0; paramIndex < numUpdatedCols; paramIndex++) { searchKeyBuff.append('_'); searchKeyBuff.append(valueFields.get(paramIndex)); } String searchKey = searchKeyBuff.toString(); final Object pkValues = pkEvent.getDeserializedValue(); final List<String> keyFields = tool.getKeyFieldNames(); PreparedStatement ps = this.updtStmntMap.get(searchKey); if (ps == null) { final String dmlString = AsyncEventHelper.getUpdateString(tableName, keyFields, valueFields); if (logger.isDebugEnabled()) { logger.info("DBSynchronizer::getExecutableInsertPrepStmntPKBased: " + "preparing '" + dmlString + "' for event: " + pkEvent); } ps = conn.prepareStatement(dmlString); this.updtStmntMap.put(searchKey, ps); } else if (prevPS == ps) { // add a new batch of values ps.addBatch(); } // Set updated col values Class valueClass = tool.getValueClass(); Class keyClass = tool.getKeyClass(); for (paramIndex = 1; paramIndex <= numUpdatedCols; paramIndex++) { String field = valueFields.get(paramIndex - 1); try { Map map = PropertyUtils.describe(pkEvent.getDeserializedValue()); Object val = map.get(field); String type = valueClass.getDeclaredField(field).getType().getName(); helper.setColumnInPrepStatement(type, val, ps, this, paramIndex); } catch (Exception e) { throw new SQLException(e); } } // Now set the Pk values setKeysInPrepStatement(pkValues, keyFields, valueClass, ps, paramIndex); return ps; } /** * Set the key column values in {@link PreparedStatement} for a primary key * based update or delete operation. */ protected void setKeysInPrepStatement(final Object keyValues, final List<String> keyFields, Class valueClass, final PreparedStatement ps, int startIndex) throws SQLException { final int numKeyCols = keyFields.size(); if (logger.isDebugEnabled()) { StringBuilder sb = new StringBuilder().append("DBSynchronizer::setKeysInPrepStatement: setting key {"); for (int col = 0; col < numKeyCols; col++) { if (col > 1) { sb.append(','); } // ???? // ?? String field = keyFields.get(col); try { Map map = PropertyUtils.describe(keyValues); Object val = map.get(field); sb.append(val); } catch (Exception e) { throw new SQLException(e); } } sb.append('}'); logger.info(sb.toString()); } for (int colIndex = 0; colIndex < numKeyCols; colIndex++, startIndex++) { String field = keyFields.get(colIndex); try { Map map = PropertyUtils.describe(keyValues); Object val = map.get(field); String type = valueClass.getDeclaredField(field).getType().getName(); helper.setColumnInPrepStatement(type, val, ps, this, startIndex); } catch (Exception e) { throw new SQLException(e); } } } /** * ?key * @param keyValue * @param keyClass * @param ps * @param startIndex * @throws SQLException */ protected void setSingleKeysInPrepStatement(final Object keyValue, Class keyClass, final PreparedStatement ps, int startIndex) throws SQLException { if (logger.isDebugEnabled()) { StringBuilder sb = new StringBuilder().append("DBSynchronizer::setKeysInPrepStatement: setting key {"); sb.append(keyValue); sb.append('}'); logger.info(sb.toString()); } try { Object val = (Object) keyValue; String type = keyClass.getName(); helper.setColumnInPrepStatement(type, val, ps, this, startIndex); } catch (Exception e) { throw new SQLException(e); } } /** * Returns an {@link SqlExceptionHandler} for the given {@link SQLException} * . */ protected SqlExceptionHandler handleSQLException(SQLException sqle) { String sqlState = sqle.getSQLState(); // What to do if SQLState is null? Checking through the exception // message for common strings for now but DB specific errorCode and // other // such checks will be better. // Below was due to a bug in wrapper OracleDriver being used and // normally // this can never be null. if (sqlState == null) { // no SQLState so fallback to string matching in the message // for BatchUpdateException it will look at the nextException if (sqle instanceof BatchUpdateException && sqle.getNextException() != null) { // "42Y96" represents an unknown exception but batch exception // will // look at the nextException in any case sqlState = "42Y96"; } else { // if connection has been closed then refresh it try { synchronized (this) { if (this.conn == null || this.conn.isClosed()) { return SqlExceptionHandler.REFRESH; } } } catch (Exception e) { return SqlExceptionHandler.REFRESH; } // treat like a connection failure by default return checkExceptionString(sqle.toString().toLowerCase(), SqlExceptionHandler.REFRESH); } } // check for exception type first SqlExceptionHandler handler = checkExceptionType(sqle); if (handler != null) { return handler; } // next check SQLStates //about SQLStates see http://blog.csdn.net/cangyingaoyou/article/details/7402243 if (sqlState.startsWith("25") || sqlState.startsWith("42")) { // constraint violations can happen in retries, so default action is // to // IGNORE them; when errorFile is provided then it will be logged to // that in XML format in any case return SqlExceptionHandler.IGNORE; } else if (sqlState.startsWith("22") || sqlState.startsWith("23")) { // if numErrorTries is defined, then retry some number of times else // ignore after having logged warning since retry is not likely to // help return this.numErrorTries > 0 ? SqlExceptionHandler.IGNORE_BREAK_LOOP : SqlExceptionHandler.IGNORE; } else if (sqlState.startsWith("08")) { return SqlExceptionHandler.REFRESH; } else if (sqlState.startsWith("40")) { // these are transient transaction/lock exceptions so retry whole // batch return SqlExceptionHandler.IGNORE_BREAK_LOOP; } else { if (sqle instanceof BatchUpdateException && sqle.getNextException() != null) { return handleSQLException(sqle.getNextException()); } // if connection has been closed then refresh it try { synchronized (this) { if (this.conn == null || this.conn.isClosed()) { //return SqlExceptionHandler.REFRESH; //,?,??TODO ?? return SqlExceptionHandler.IGNORE; } } } catch (Exception e) { return SqlExceptionHandler.REFRESH; } return checkExceptionString(sqle.toString().toLowerCase(), SqlExceptionHandler.REFRESH); } } protected SqlExceptionHandler checkExceptionType(SQLException sqle) { if (sqle != null) { if (sqle instanceof SQLNonTransientConnectionException) { // will need to connect again return SqlExceptionHandler.REFRESH; } if (sqle instanceof SQLIntegrityConstraintViolationException) { // constraint violations can happen in retries, so default // action is to // IGNORE them; when errorFile is provided then it will be // logged to // that in XML format in any case return SqlExceptionHandler.IGNORE; } if (sqle instanceof SQLNonTransientException) { // if numErrorTries is defined, then retry some number of times // else // ignore after having logged warning since retry is not likely // to help return this.numErrorTries > 0 ? SqlExceptionHandler.IGNORE_BREAK_LOOP : SqlExceptionHandler.IGNORE; } if (sqle instanceof SQLTransientException) { // skip the remaining batch and retry whole batch again return SqlExceptionHandler.IGNORE_BREAK_LOOP; } if (sqle instanceof BatchUpdateException) { return checkExceptionType(sqle.getNextException()); } } return null; } protected SqlExceptionHandler checkExceptionString(String message, SqlExceptionHandler defaultHandler) { if (message.contains("constraint")) { // likely a constraint violation // constraint violations can happen in retries, so default action is // to // IGNORE them; when errorFile is provided then it will be logged to // that in XML format in any case return SqlExceptionHandler.IGNORE; } else if (message.contains("syntax")) { // if numErrorTries is defined, then retry some number of times else // ignore after having logged warning since retry is not likely to // help return this.numErrorTries > 0 ? SqlExceptionHandler.IGNORE_BREAK_LOOP : SqlExceptionHandler.IGNORE; } else if (message.contains("connect")) { // likely a connection error return SqlExceptionHandler.REFRESH; } else { return defaultHandler; } } /** * Log exception including stack traces for info logging with * {@link #traceDBSynchronizer}, and returns an {@link SqlExceptionHandler} * for the given {@link SQLException}. */ protected SqlExceptionHandler handleSQLException(SQLException sqle, String format, Statement stmt, AsyncEvent event, String eventString, Logger logger, boolean logWarning) throws SQLException { SqlExceptionHandler handler = handleSQLException(sqle); if (event != null && this.numErrorTries > 0) { ErrorEvent ee = new ErrorEvent(); ee.ev = event; ee.errortime = System.currentTimeMillis(); Object[] tries = this.errorTriesMap.get(ee); if (tries != null) { Integer numTries = (Integer) tries[0]; if (numTries >= this.numErrorTries) { // at this point ignore this exception and move to others in // the batch handler = SqlExceptionHandler.IGNORE; logWarning = false; } tries[0] = Integer.valueOf(numTries.intValue() + 1); tries[1] = sqle; } else { this.errorTriesMap.put(ee, new Object[] { 1, sqle }); } } boolean skipLogging = false; if (event != null && (logWarning || logger.isDebugEnabled())) { if (eventString == null) { skipLogging = helper.skipFailureLogging(event); eventString = event.toString(); } } if (!skipLogging) { if (logWarning) { if (logger.isWarnEnabled()) { helper.logFormat(logger, Level.WARNING, sqle, format, eventString, sqle); SQLException next = sqle.getNextException(); if (next != null) { helper.logFormat(logger, Level.WARNING, next, format, eventString, sqle.getNextException()); } } } if (logger.isDebugEnabled()) { if (logger.isWarnEnabled()) { String stmtStr = (stmt != null ? ("executing statement=" + stmt) : "preparing statement"); helper.log(logger, Level.WARNING, sqle, "DBSynchronizer::" + "processEvents: Exception while " + stmtStr + " for event=" + eventString); if (sqle.getNextException() != null) { helper.log(logger, Level.WARNING, sqle.getNextException(), "DBSynchronizer::processEvents: next exception"); } } } } handler.execute(this); return handler; } // @Override // public synchronized void start() { // if (this.shutDown) { // this.instantiateConnection(); // } // } }