org.openiam.idm.srvc.synch.srcadapter.RDBMSAdapter.java Source code

Java tutorial

Introduction

Here is the source code for org.openiam.idm.srvc.synch.srcadapter.RDBMSAdapter.java

Source

/*
 * Copyright 2009, OpenIAM LLC 
 * This file is part of the OpenIAM Identity and Access Management Suite
 *
 *   OpenIAM Identity and Access Management Suite is free software: 
 *   you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License 
 *   version 3 as published by the Free Software Foundation.
 *
 *   OpenIAM is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   Lesser GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with OpenIAM.  If not, see <http://www.gnu.org/licenses/>. *
 */

/**
 *
 */
package org.openiam.idm.srvc.synch.srcadapter;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.openiam.base.ws.Response;
import org.openiam.base.ws.ResponseCode;
import org.openiam.base.ws.ResponseStatus;
import org.openiam.idm.srvc.synch.domain.SynchReviewEntity;
import org.openiam.idm.srvc.synch.dto.*;
import org.openiam.idm.srvc.synch.service.MatchObjectRule;
import org.openiam.idm.srvc.synch.service.TransformScript;
import org.openiam.idm.srvc.synch.service.ValidationScript;
import org.openiam.idm.srvc.synch.util.DatabaseUtil;
import org.openiam.provision.service.ProvisionService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.sql.*;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

/**
 * Reads a CSV file for use during the synchronization process
 *
 * @author suneet
 */
@Component
public class RDBMSAdapter extends AbstractSrcAdapter {

    private LineObject rowHeader = new LineObject();

    private static final Log log = LogFactory.getLog(RDBMSAdapter.class);

    private Connection con = null;

    @Value("${rdbmsvadapter.thread.count}")
    private int THREAD_COUNT;

    @Value("${rdbmsvadapter.thread.delay.beforestart}")
    private int THREAD_DELAY_BEFORE_START;

    @Override
    public SyncResponse startSynch(final SynchConfig config) {
        return startSynch(config, null, null);
    }

    @Override
    public SyncResponse startSynch(final SynchConfig config, SynchReviewEntity sourceReview,
            final SynchReviewEntity resultReview) {

        log.debug("RDBMS SYNCH STARTED ^^^^^^^^");

        SyncResponse res = new SyncResponse(ResponseStatus.SUCCESS);
        SynchReview review = null;
        if (sourceReview != null) {
            review = synchReviewDozerConverter.convertToDTO(sourceReview, false);
        }
        LineObject rowHeaderForReport = null;
        InputStream input = null;

        try {
            final ValidationScript validationScript = org.mule.util.StringUtils.isNotEmpty(
                    config.getValidationRule()) ? SynchScriptFactory.createValidationScript(config, review) : null;
            final List<TransformScript> transformScripts = SynchScriptFactory.createTransformationScript(config,
                    review);
            final MatchObjectRule matchRule = matchRuleFactory.create(config.getCustomMatchRule()); // check if matchRule exists

            if (validationScript == null || transformScripts == null || matchRule == null) {
                res = new SyncResponse(ResponseStatus.FAILURE);
                res.setErrorText("The problem in initialization of RDBMSAdapter, please check validationScript= "
                        + validationScript + ", transformScripts=" + transformScripts + ", matchRule=" + matchRule
                        + " all must be set!");
                res.setErrorCode(ResponseCode.INVALID_ARGUMENTS);
                return res;
            }

            if (sourceReview != null && !sourceReview.isSourceRejected()) {
                return startSynchReview(config, sourceReview, resultReview, validationScript, transformScripts,
                        matchRule);
            }

            if (!connect(config)) {
                SyncResponse resp = new SyncResponse(ResponseStatus.FAILURE);
                resp.setErrorCode(ResponseCode.FAIL_SQL_ERROR);
                return resp;
            }

            java.util.Date lastExec = null;

            if (config.getLastExecTime() != null) {
                lastExec = config.getLastExecTime();
            }
            final String changeLog = config.getQueryTimeField();
            StringBuilder sql = new StringBuilder(config.getQuery());
            // if its incremental synch, then add the change log parameter
            if (config.getSynchType().equalsIgnoreCase("INCREMENTAL")) {
                // execute the query
                if (StringUtils.isNotEmpty(sql.toString()) && (lastExec != null)) {

                    String temp = sql.toString().toUpperCase();
                    // strip off any trailing semi-colons. Not needed for jbdc
                    temp = StringUtils.removeEnd(temp, ";");

                    if (temp.contains("WHERE")) {
                        sql.append(" AND ");
                    } else {
                        sql.append(" WHERE ");
                    }
                    sql.append(changeLog).append(" >= ?");
                }
            }

            log.debug("-SYNCH SQL=" + sql.toString());
            log.debug("-last processed record =" + lastExec);

            PreparedStatement ps = con.prepareStatement(sql.toString());
            if (config.getSynchType().equalsIgnoreCase("INCREMENTAL") && (lastExec != null)) {
                ps.setTimestamp(1, new Timestamp(lastExec.getTime()));
            }
            ResultSet rs = ps.executeQuery();

            // get the list of columns
            ResultSetMetaData rsMetadata = rs.getMetaData();
            DatabaseUtil.populateTemplate(rsMetadata, rowHeader);

            //Read Resultset to List
            List<LineObject> results = new LinkedList<LineObject>();
            while (rs.next()) {
                LineObject rowObj = rowHeader.copy();
                DatabaseUtil.populateRowObject(rowObj, rs, changeLog);
                results.add(rowObj);
            }

            // test
            log.debug("Result set contains following number of columns : " + rowHeader.getColumnMap().size());

            // Multithreading
            int allRowsCount = results.size();
            if (allRowsCount > 0) {
                int threadCoount = THREAD_COUNT;
                int rowsInOneExecutors = allRowsCount / threadCoount;
                int remains = rowsInOneExecutors > 0 ? allRowsCount % (rowsInOneExecutors * threadCoount) : 0;
                if (remains != 0) {
                    threadCoount++;
                }
                log.debug("Thread count = " + threadCoount + "; Rows in one thread = " + rowsInOneExecutors
                        + "; Remains rows = " + remains);
                System.out.println("Thread count = " + threadCoount + "; Rows in one thread = " + rowsInOneExecutors
                        + "; Remains rows = " + remains);
                List<Future> threadResults = new LinkedList<Future>();
                // store the latest processed record by thread indx
                final Map<String, Timestamp> recentRecordByThreadInx = new HashMap<String, Timestamp>();
                final ExecutorService service = Executors.newCachedThreadPool();
                for (int i = 0; i < threadCoount; i++) {
                    final int threadIndx = i;
                    final int startIndex = i * rowsInOneExecutors;
                    // Start index for current thread
                    int shiftIndex = threadCoount > THREAD_COUNT && i == threadCoount - 1 ? remains
                            : rowsInOneExecutors;
                    // Part of the rowas that should be processing with this thread
                    final List<LineObject> part = results.subList(startIndex, startIndex + shiftIndex);
                    threadResults.add(service.submit(new Runnable() {
                        @Override
                        public void run() {
                            try {
                                Timestamp mostRecentRecord = proccess(config, resultReview, provService, part,
                                        validationScript, transformScripts, matchRule, resultReview, startIndex);
                                recentRecordByThreadInx.put("Thread_" + threadIndx, mostRecentRecord);
                            } catch (ClassNotFoundException e) {
                                log.error(e);
                                /*
                                synchStartLog.updateSynchAttributes("FAIL", ResponseCode.CLASS_NOT_FOUND.toString(), e.toString());
                                auditHelper.logEvent(synchStartLog);
                                */
                            }
                        }
                    }));
                    //Give THREAD_DELAY_BEFORE_START seconds time for thread to be UP (load all cache and begin the work)
                    Thread.sleep(THREAD_DELAY_BEFORE_START);
                }
                Runtime.getRuntime().addShutdownHook(new Thread() {
                    public void run() {
                        service.shutdown();
                        try {
                            if (!service.awaitTermination(SHUTDOWN_TIME, TimeUnit.MILLISECONDS)) { //optional *
                                log.warn("Executor did not terminate in the specified time."); //optional *
                                List<Runnable> droppedTasks = service.shutdownNow(); //optional **
                                log.warn("Executor was abruptly shut down. " + droppedTasks.size()
                                        + " tasks will not be executed."); //optional **
                            }
                        } catch (InterruptedException e) {
                            log.error(e);
                            /*
                            synchStartLog.updateSynchAttributes("FAIL", ResponseCode.INTERRUPTED_EXCEPTION.toString(), e.toString());
                            auditHelper.logEvent(synchStartLog);
                            */
                            SyncResponse resp = new SyncResponse(ResponseStatus.FAILURE);
                            resp.setErrorCode(ResponseCode.INTERRUPTED_EXCEPTION);
                        }
                    }
                });
                waitUntilWorkDone(threadResults);

            }
        } catch (ClassNotFoundException cnfe) {
            log.error(cnfe);
            res = new SyncResponse(ResponseStatus.FAILURE);
            res.setErrorCode(ResponseCode.CLASS_NOT_FOUND);
            return res;
        } catch (FileNotFoundException fe) {
            fe.printStackTrace();
            log.error(fe);
            //            auditBuilder.addAttribute(AuditAttributeName.DESCRIPTION, "FileNotFoundException: "+fe.getMessage());
            //            auditLogProvider.persist(auditBuilder);
            SyncResponse resp = new SyncResponse(ResponseStatus.FAILURE);
            resp.setErrorCode(ResponseCode.FILE_EXCEPTION);
            log.debug("RDBMS SYNCHRONIZATION COMPLETE WITH ERRORS ^^^^^^^^");
            return resp;
        } catch (IOException io) {
            io.printStackTrace();
            /*
            synchStartLog.updateSynchAttributes("FAIL", ResponseCode.IO_EXCEPTION.toString(), io.toString());
            auditHelper.logEvent(synchStartLog);
            */
            SyncResponse resp = new SyncResponse(ResponseStatus.FAILURE);
            resp.setErrorCode(ResponseCode.IO_EXCEPTION);
            log.debug("RDBMS SYNCHRONIZATION COMPLETE WITH ERRORS ^^^^^^^^");
            return resp;

        } catch (SQLException se) {

            log.error(se);
            closeConnection();
            /*
            synchStartLog.updateSynchAttributes("FAIL", ResponseCode.SQL_EXCEPTION.toString(), se.toString());
            auditHelper.logEvent(synchStartLog);
            */
            SyncResponse resp = new SyncResponse(ResponseStatus.FAILURE);
            resp.setErrorCode(ResponseCode.SQL_EXCEPTION);
            resp.setErrorText(se.toString());
            return resp;

        } catch (InterruptedException e) {
            log.error(e);
            SyncResponse resp = new SyncResponse(ResponseStatus.FAILURE);
            resp.setErrorCode(ResponseCode.INTERRUPTED_EXCEPTION);

        } finally {
            if (resultReview != null) {
                if (CollectionUtils.isNotEmpty(resultReview.getReviewRecords())) { // add header row
                    resultReview.addRecord(generateSynchReviewRecord(rowHeader, true));
                }
            }

            closeConnection();
        }

        log.debug("RDBMS SYNCH COMPLETE.^^^^^^^^");
        return new SyncResponse(ResponseStatus.SUCCESS);

    }

    private Timestamp proccess(SynchConfig config, SynchReviewEntity review, ProvisionService provService,
            List<LineObject> part, final ValidationScript validationScript,
            final List<TransformScript> transformScripts, MatchObjectRule matchRule, SynchReviewEntity resultReview,
            int ctr) throws ClassNotFoundException {
        Timestamp mostRecentRecord = null;
        for (LineObject rowObj : part) {
            log.debug("-RDBMS ADAPTER: SYNCHRONIZING  RECORD # ---" + ctr++);
            log.debug(" - Record update time=" + rowObj.getLastUpdate());

            if (mostRecentRecord == null) {
                mostRecentRecord = rowObj.getLastUpdate();

            } else {
                // if current record is newer than what we saved, then update the most recent record value
                if (mostRecentRecord.before(rowObj.getLastUpdate())) {
                    log.debug("- MostRecentRecord value updated to=" + rowObj.getLastUpdate());
                    mostRecentRecord.setTime(rowObj.getLastUpdate().getTime());
                }
            }

            processLineObject(rowObj, config, resultReview, validationScript, transformScripts, matchRule);

        }
        return mostRecentRecord;
    }

    public Response testConnection(SynchConfig config) {
        try {
            Class.forName(config.getDriver());

            con = DriverManager.getConnection(config.getConnectionUrl(), config.getSrcLoginId(),
                    config.getSrcPassword());
            closeConnection();
            Response resp = new Response(ResponseStatus.SUCCESS);
            return resp;
        } catch (SQLException e) {
            Response resp = new Response(ResponseStatus.FAILURE);
            resp.setErrorCode(ResponseCode.SQL_EXCEPTION);
            resp.setErrorText(e.getMessage());
            return resp;
        } catch (ClassNotFoundException e) {
            Response resp = new Response(ResponseStatus.FAILURE);
            resp.setErrorCode(ResponseCode.CLASS_NOT_FOUND);
            resp.setErrorText(e.getMessage());
            return resp;
        }
    }

    private boolean connect(SynchConfig config) {

        try {
            Class.forName(config.getDriver());

            con = DriverManager.getConnection(config.getConnectionUrl(), config.getSrcLoginId(),
                    config.getSrcPassword());
            return true;
        } catch (SQLException se) {
            se.printStackTrace();
        } catch (ClassNotFoundException cf) {
            cf.printStackTrace();
        }
        return false;
    }

    private void closeConnection() {
        try {
            con.close();
        } catch (SQLException se) {
            se.printStackTrace();
        }
    }
}