org.kuali.ole.docstore.process.RebuildIndexesHandler.java Source code

Java tutorial

Introduction

Here is the source code for org.kuali.ole.docstore.process.RebuildIndexesHandler.java

Source

package org.kuali.ole.docstore.process;

import org.apache.commons.lang.time.StopWatch;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.kuali.ole.DocumentUniqueIDPrefix;
import org.kuali.ole.RepositoryBrowser;
import org.kuali.ole.RepositoryManager;
import org.kuali.ole.docstore.common.document.*;
import org.kuali.ole.docstore.common.document.content.instance.FormerIdentifier;
import org.kuali.ole.docstore.common.document.content.instance.Instance;
import org.kuali.ole.docstore.common.document.content.instance.xstream.InstanceOlemlRecordProcessor;
import org.kuali.ole.docstore.common.util.BatchBibTreeDBUtil;
import org.kuali.ole.docstore.common.util.BibInfoStatistics;
import org.kuali.ole.docstore.common.util.ReindexBatchStatistics;
import org.kuali.ole.docstore.discovery.service.SolrServerManager;
import org.kuali.ole.docstore.discovery.solr.work.bib.marc.WorkBibMarcDocBuilder;
import org.kuali.ole.docstore.document.rdbms.RdbmsWorkBibMarcDocumentManager;
import org.kuali.ole.docstore.document.rdbms.RdbmsWorkEInstanceDocumentManager;
import org.kuali.ole.docstore.document.rdbms.RdbmsWorkInstanceDocumentManager;
import org.kuali.ole.docstore.engine.service.index.solr.BibMarcIndexer;
import org.kuali.ole.docstore.engine.service.index.solr.DocumentIndexer;
import org.kuali.ole.docstore.indexer.solr.IndexerService;
import org.kuali.ole.docstore.metrics.reindex.ReIndexingBatchStatus;
import org.kuali.ole.docstore.metrics.reindex.ReIndexingStatus;
import org.kuali.ole.docstore.model.enums.DocCategory;
import org.kuali.ole.docstore.model.enums.DocFormat;
import org.kuali.ole.docstore.model.enums.DocType;
import org.kuali.ole.docstore.engine.service.storage.rdbms.pojo.BibRecord;
import org.kuali.ole.docstore.engine.service.storage.rdbms.pojo.EInstanceRecord;
import org.kuali.ole.docstore.engine.service.storage.rdbms.pojo.InstanceRecord;
import org.kuali.ole.docstore.model.xmlpojo.ingest.AdditionalAttributes;
import org.kuali.ole.docstore.model.xmlpojo.ingest.Content;
import org.kuali.ole.docstore.model.xmlpojo.ingest.RequestDocument;
import org.kuali.ole.docstore.model.xmlpojo.ingest.ResponseDocument;
import org.kuali.ole.docstore.common.document.content.instance.InstanceCollection;
import org.kuali.ole.docstore.model.xstream.work.oleml.WorkEInstanceOlemlRecordProcessor;
import org.kuali.ole.docstore.service.BeanLocator;
import org.kuali.ole.docstore.service.DocumentIngester;
import org.kuali.ole.docstore.service.ServiceLocator;
import org.kuali.ole.pojo.OleException;
import org.kuali.ole.repository.CheckoutManager;
import org.kuali.ole.repository.NodeHandler;
import org.kuali.rice.krad.service.BusinessObjectService;
import org.kuali.rice.krad.service.KRADServiceLocator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.kuali.rice.core.api.config.property.ConfigContext;

import javax.jcr.Binary;
import javax.jcr.Node;
import javax.jcr.NodeIterator;
import javax.jcr.Session;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.sql.*;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.Date;

/**
 * Class to Rebuild Indexes.
 *
 * @author Rajesh Chowdary K
 * @created May 2, 2012
 */
public class RebuildIndexesHandler implements Runnable {

    private static final Logger LOG = LoggerFactory.getLogger(RebuildIndexesHandler.class);
    private static RebuildIndexesHandler reBuilder = null;
    private boolean isRunning = false;
    private boolean isStop = false;
    private static final Logger logger = LoggerFactory.getLogger(RebuildIndexesHandler.class);
    private String docCategory;
    private String docType;
    private String docFormat;
    private BibInfoStatistics bibInfoStatistics = null;
    private CheckoutManager checkoutManager;
    //    private ReIndexingStatus reIndexingStatus;
    private int batchSize;
    private int startIndex;
    private int endIndex;
    private String updateDate;

    public static String EXCEPION_FILE_NAME = "";
    public static String STATUS_FILE_NAME = "";
    public static String STORAGE_EXCEPTION_FILE_NAME = "";
    public static String STORAGE_STATUS_FILE_NAME = "";
    public static BatchBibTreeDBUtil bibTreeDBUtil = new BatchBibTreeDBUtil();

    private String filePath = System.getProperty("solr.solr.home");

    public synchronized void setRunning(boolean running) {
        isRunning = running;
    }

    public synchronized void setStop(boolean stop) {
        isStop = stop;
    }

    private RebuildIndexesHandler() {
        checkoutManager = new CheckoutManager();
    }

    public static RebuildIndexesHandler getInstance() {
        if (reBuilder == null) {
            reBuilder = new RebuildIndexesHandler();
        }
        return reBuilder;
    }

    /**
     * Method to get running status.
     *
     * @return
     */
    public synchronized boolean isRunning() {
        return isRunning;
    }

    public synchronized boolean isStop() {
        return isStop;
    }

    /**
     * Method to startProcess
     */
    public String startProcess(String docCategory, String docType, String docFormat) throws InterruptedException {
        String status = null;
        if (isRunning()) {
            status = "ReIndexing process is already running. Click 'Show Status' button to know the status. ";
        } else {
            setRunning(true);
            setStop(false);
            status = "ReIndexing process has started. Click 'Show Status' button to know the status. ";
            ReIndexingStatus reIndexingStatus = ReIndexingStatus.getInstance();
            reIndexingStatus.reset();
            if (docCategory == null || docCategory.equals("")) {
                docCategory = "all";
            }
            if (docType == null || docType.equals("")) {
                docType = "all";
            }
            if (docFormat == null || docType.equals("")) {
                docFormat = "all";
            }
            this.docCategory = docCategory;
            this.docType = docType;
            this.docFormat = docFormat;
            Thread reBuilderThread = new Thread(this);
            reBuilderThread.start();
            //            reBuilderThread.join();
            setRunning(false);
        }
        return status;
    }

    public String startProcess(String docCategory, String docType, String docFormat, int batchSize, int startIndex,
            int endIndex, String updateDate) throws InterruptedException {
        String status = null;
        if (isRunning()) {
            status = "ReIndexing process is already running. Click 'Show Status' button to know the status. ";
        } else {
            setRunning(true);
            setStop(false);
            status = "ReIndexing process has started. Click 'Show Status' button to know the status. ";
            ReIndexingStatus reIndexingStatus = ReIndexingStatus.getInstance();
            reIndexingStatus.reset();
            if (docCategory == null || docCategory.equals("")) {
                docCategory = "all";
            }
            if (docType == null || docType.equals("")) {
                docType = "all";
            }
            if (docFormat == null || docType.equals("")) {
                docFormat = "all";
            }
            this.docCategory = docCategory;
            this.docType = docType;
            this.docFormat = docFormat;
            this.batchSize = batchSize;
            this.startIndex = startIndex;
            this.endIndex = endIndex;
            this.updateDate = updateDate;
            Thread reBuilderThread = new Thread(this);
            reBuilderThread.start();
            //            reBuilderThread.join();
            setRunning(false);
        }
        return status;
    }

    public String stopProcess() throws Exception {
        String status = null;
        if (isRunning()) {
            status = "ReIndexing process is running. ReIndexing will stop after current batch. ";
            setStop(true);
            setRunning(false);
        } else {
            status = "ReIndexing process is not running.";
        }
        return status;

    }

    public void run() {
        DocCategoryTypeFormat docCategoryTypeFormat = new DocCategoryTypeFormat();
        List<String> categoryList = docCategoryTypeFormat.getCategories();
        List<String> typeList = null;
        List<String> formatList = null;
        for (String docCategoryCurr : categoryList) {
            if (docCategory.equals("all") || docCategory.equals(docCategoryCurr)) {
                typeList = docCategoryTypeFormat.getDocTypes(docCategoryCurr);
                for (String docTypeCurr : typeList) {
                    if (docType.equals("all") || docType.equals(docTypeCurr)) {
                        formatList = docCategoryTypeFormat.getDocFormats(docCategoryCurr, docTypeCurr);
                        for (String docFormatCurr : formatList) {
                            if (docFormat.equals("all") || docFormat.equals(docFormatCurr)) {
                                if (!isStop()) {
                                    ReIndexingStatus.getInstance().startDocType(docCategoryCurr, docTypeCurr,
                                            docFormatCurr);
                                    reIndex(docCategoryCurr, docTypeCurr, docFormatCurr);
                                } else {
                                    return;
                                }
                            }
                        }
                    }
                }
            }
        }
        setRunning(false);
    }

    private void reIndex(String docCategory, String docType, String docFormat) {
        Session session = null;
        setRunning(true);
        logger.info("Rebuild Indexes Run(" + docCategory + " : " + docType + " : " + docFormat + "): ");
        try {
            if (docCategory.equals(DocCategory.WORK.getCode())) {
                if (docType.equals(DocType.BIB.getDescription())) {
                    if (docFormat.equals(DocFormat.MARC.getCode())
                            || docFormat.equals(DocFormat.DUBLIN_CORE.getCode())
                            || docFormat.equals(DocFormat.DUBLIN_UNQUALIFIED.getCode())) {
                        org.springframework.util.StopWatch stopWatch = new org.springframework.util.StopWatch();
                        stopWatch.start("total time taken");
                        Date date = new Date();
                        EXCEPION_FILE_NAME = "ReindexErrors-" + date.toString() + ".txt";
                        STATUS_FILE_NAME = "ReindexBatchStatus-" + date.toString() + ".txt";
                        BatchBibTreeDBUtil.writeStatusToFile(filePath, RebuildIndexesHandler.EXCEPION_FILE_NAME,
                                "Reindex started at:" + date);
                        BibHoldingItemReindexer bibHoldingItemReindexer = BibHoldingItemReindexer.getInstance();
                        bibHoldingItemReindexer.setTotalBatchStatistics(new ReindexBatchStatistics());
                        bibHoldingItemReindexer.index(batchSize, startIndex, endIndex, updateDate);
                        date = new Date();
                        BatchBibTreeDBUtil.writeStatusToFile(filePath, RebuildIndexesHandler.EXCEPION_FILE_NAME,
                                "Reindex ended at:" + date);
                        stopWatch.stop();
                        logger.info(stopWatch.prettyPrint());
                        //                        workBibMarcAndDublinAll(docCategory, docType, docFormat);
                    } else {
                        logger.info("Rebuild Indexes Run(" + docCategory + " : " + docType + " : " + docFormat
                                + "): FAIL");
                    }
                } else if (docType.equals(DocType.INSTANCE.getDescription())) {
                    if (docFormat.equals(DocFormat.OLEML.getCode())) {
                        workInstanceOLEML(docCategory, docType, docFormat);
                    } else {
                        logger.info("Rebuild Indexes Run(" + docCategory + " : " + docType + " : " + docFormat
                                + "): FAIL");
                    }
                } else if (docType.equals(DocType.LICENSE.getDescription())) {
                    if (docFormat.equals(DocFormat.ONIXPL.getCode()) || docFormat.equals(DocFormat.PDF.getCode())
                            || docFormat.equals(DocFormat.DOC.getCode())) {
                        workLicense(docCategory, docType, docFormat);
                    } else {
                        logger.info("Rebuild Indexes Run(" + docCategory + " : " + docType + " : " + docFormat
                                + "): FAIL");
                    }
                } else if (docType.equals(DocType.EINSTANCE.getCode())) {
                    if (docFormat.equals(DocFormat.OLEML.getCode())) {
                        workEInstanceOLEML(docCategory, docType, docFormat);
                    } else {
                        logger.info("Rebuild Indexes Run(" + docCategory + " : " + docType + " : " + docFormat
                                + "): FAIL");
                    }
                }
            }
        } catch (Exception e) {
            logger.info(e.getMessage(), e);
        } finally {
            try {
                if (isStop) {
                    ReIndexingStatus.getInstance().getDocTypeList().setStatus("Stopped");
                } else {
                    ReIndexingStatus.getInstance().getDocTypeList().setStatus("Done");
                }
                RepositoryManager.getRepositoryManager().logout(session);
            } catch (OleException e) {
                logger.error(e.getMessage(), e);
            }
        }

    }

    private void workEInstanceOLEML(String docCategory, String docType, String docFormat) {
        long totalCount = 0;
        long nodeCount = 0;
        List<RequestDocument> docs = new ArrayList<RequestDocument>();
        WorkEInstanceOlemlRecordProcessor workEInstanceOlemlRecordProcessor = new WorkEInstanceOlemlRecordProcessor();
        try {
            RequestDocument rd = new RequestDocument();
            rd.setCategory(docCategory);
            rd.setType(docType);
            rd.setFormat(docFormat);
            List<ReIndexingBatchStatus> batchStatusList = new ArrayList<ReIndexingBatchStatus>();
            BusinessObjectService businessObjectService = KRADServiceLocator.getBusinessObjectService();
            List<EInstanceRecord> instanceRecords = (List<EInstanceRecord>) businessObjectService
                    .findAll(EInstanceRecord.class);
            StopWatch loadTimer = new StopWatch();
            StopWatch batchTimer = new StopWatch();
            loadTimer.start();
            batchTimer.start();
            for (int i = 0; i < instanceRecords.size(); i++) {
                if (docs.size() == ProcessParameters.BULK_PROCESSOR_SPLIT_SIZE) {
                    if (!isStop()) {
                        ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
                        indexDocs(docs, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
                        indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
                        resetTimers(batchTimer, loadTimer);
                        totalCount = 0;
                        logger.info("Rebuild");
                    } else {
                        return;
                    }
                } else {
                    EInstanceRecord instanceRecord = instanceRecords.get(i);
                    String uuid = DocumentUniqueIDPrefix.getPrefixedId(instanceRecord.getUniqueIdPrefix(),
                            instanceRecord.geteInstanceIdentifier());
                    RequestDocument requestDocument = buildRequestDocumentForCheckout(docCategory, docType,
                            docFormat, uuid);
                    ResponseDocument responseDocument = RdbmsWorkEInstanceDocumentManager.getInstance()
                            .checkoutContent(requestDocument, businessObjectService);
                    String content = responseDocument.getContent().getContent();
                    RequestDocument requestDocumentForIndex = (RequestDocument) rd.clone();
                    requestDocumentForIndex.setAdditionalAttributes(responseDocument.getAdditionalAttributes());
                    requestDocumentForIndex.setId(uuid);
                    requestDocumentForIndex.setUuid(uuid);
                    org.kuali.ole.docstore.model.xmlpojo.work.einstance.oleml.InstanceCollection instanceCollection = workEInstanceOlemlRecordProcessor
                            .fromXML(content);
                    content = workEInstanceOlemlRecordProcessor.toXML(instanceCollection);
                    Content contentObj = new Content();
                    contentObj.setContent(content);
                    contentObj.setContentObject(instanceCollection);
                    requestDocumentForIndex.setContent(contentObj);
                    docs.add(requestDocumentForIndex);
                    totalCount++;
                }
            }
            if (docs.size() > 0 && !isStop()) {
                ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
                indexDocs(docs, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
                indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
            }
        } catch (Exception e) {
            logger.error(
                    "Rebuild Indexes Process(" + docCategory + " : " + docType + " : " + docFormat + ") Processed("
                            + (totalCount - docs.size()) + "), Failed @ batch(" + docs.size() + "): Cause: " + e,
                    e);
        } finally {
            if (isStop) {
                ReIndexingStatus.getInstance().getDocTypeList().setStatus("Stopped");
            } else {
                ReIndexingStatus.getInstance().getDocTypeList().setStatus("Done");
            }
        }

    }

    private void indexBibDocs(List<BibTree> bibTreeList, long records, long recCount,
            List<ReIndexingBatchStatus> batchStatusList, ReIndexingBatchStatus reIndexingBatchStatus) {
        StopWatch indexTimer = new StopWatch();
        DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss:SSS");
        try {
            Date startDate = new Date();
            reIndexingBatchStatus.setBatchStartTime(dateFormat.format(startDate));
            indexTimer.start();
            reIndexingBatchStatus.setStatus("Indexing");
            reIndexingBatchStatus.setBatchIndexingTime(indexTimer.toString());
            reIndexingBatchStatus.setRecordsProcessed(records);
            reIndexingBatchStatus.setBatchEndTime(" ");
            batchStatusList.add(reIndexingBatchStatus);
            ReIndexingStatus.getInstance().getDocTypeList().setReIndBatStatusList(batchStatusList);
            DocumentIndexer documentIndexer = BibMarcIndexer.getInstance();
            BibTrees bibTrees = new BibTrees();
            bibTrees.getBibTrees().addAll(bibTreeList);
            documentIndexer.createTrees(bibTrees);
            //logger.debug(result);
            indexTimer.stop();
            Date endDate = new Date();
            reIndexingBatchStatus.setBatchEndTime(dateFormat.format(endDate));
            reIndexingBatchStatus.setBatchIndexingTime(indexTimer.toString());
            reIndexingBatchStatus.setRecordsProcessed(records);
            reIndexingBatchStatus.setStatus("Done");
            reIndexingBatchStatus.setRecordsRemaining(recCount - records);
            ReIndexingStatus.getInstance().getDocTypeList().setReIndBatStatusList(batchStatusList);
            bibTreeList.clear();
        } catch (Exception e) {
            String firstBibId = bibTreeList.get(0).getBib().getId();
            String lastBibId = bibTreeList.get(bibTreeList.size() - 1).getBib().getId();
            logger.error("Rebuild Indexes Process(" + docCategory + " : " + docType + " : " + docFormat
                    + ") Processed(" + (records - bibTreeList.size()) + "), Failed @ bibId( First BibId: "
                    + firstBibId + "   :  Last BibId : " + lastBibId + "): Cause: " + e, e);
            indexTimer.stop();
            Date endDate = new Date();
            reIndexingBatchStatus.setBatchEndTime(dateFormat.format(endDate));
            reIndexingBatchStatus.setBatchIndexingTime(indexTimer.toString());
            reIndexingBatchStatus.setRecordsProcessed(0L);
            reIndexingBatchStatus.setStatus("Done");
            reIndexingBatchStatus.setRecordsRemaining(recCount - records);
            ReIndexingStatus.getInstance().getDocTypeList().setReIndBatStatusList(batchStatusList);
            bibTreeList.clear();
        }
    }

    private void indexDocs(List<RequestDocument> docs, long records, long recCount,
            List<ReIndexingBatchStatus> batchStatusList, ReIndexingBatchStatus reIndexingBatchStatus) {
        try {
            StopWatch indexTimer = new StopWatch();
            DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss:SSS");
            Date startDate = new Date();
            reIndexingBatchStatus.setBatchStartTime(dateFormat.format(startDate));
            indexTimer.start();
            reIndexingBatchStatus.setStatus("Indexing");
            reIndexingBatchStatus.setBatchIndexingTime(indexTimer.toString());
            reIndexingBatchStatus.setRecordsProcessed(records);
            reIndexingBatchStatus.setBatchEndTime(" ");
            batchStatusList.add(reIndexingBatchStatus);
            ReIndexingStatus.getInstance().getDocTypeList().setReIndBatStatusList(batchStatusList);
            IndexerService indexerService = BeanLocator.getDocstoreFactory().getDocumentIndexManager(
                    docs.get(0).getCategory(), docs.get(0).getType(), docs.get(0).getFormat());
            String result = indexerService.indexDocuments(docs, false);
            logger.debug(result);
            indexTimer.stop();
            Date endDate = new Date();
            reIndexingBatchStatus.setBatchEndTime(dateFormat.format(endDate));
            reIndexingBatchStatus.setBatchIndexingTime(indexTimer.toString());
            reIndexingBatchStatus.setRecordsProcessed(records);
            reIndexingBatchStatus.setStatus("Done");
            reIndexingBatchStatus.setRecordsRemaining(recCount - records);
            ReIndexingStatus.getInstance().getDocTypeList().setReIndBatStatusList(batchStatusList);
            docs.clear();
        } catch (Exception e) {
            logger.error("Rebuild Indexes Processed(" + (records - docs.size()) + "), Failed @ batch(" + docs.size()
                    + "): Cause: " + e + "\n\tContinuous", e);
        }
    }

    private void workBibMarcAndDublinAll(String docCategory, String docType, String docFormat)
            throws SolrServerException, IOException {
        long totalCount = 0;
        long nodeCount = 0;
        int start = 0;
        String sqlQuery = null;
        long startTime = 0;
        long commitEndTime = 0;
        long commitStartTime = 0;
        int batchSize = 50000; //ProcessParameters.BULK_PROCESSOR_SPLIT_SIZE;
        int commitSize = 50000;
        long endIndexBatch = 0;
        String dbVendor = ConfigContext.getCurrentContextConfig().getProperty("db.vendor");
        List<BibTree> bibTrees = new ArrayList<BibTree>();
        //Map<String, BibTree> bibMap = new HashMap<String, BibTree>();
        try {
            String prefix = DocumentUniqueIDPrefix.getPrefix(docCategory, docType, docFormat);
            Map prefixMap = new HashMap(0);
            prefixMap.put("uniqueIdPrefix", prefix);
            startTime = System.currentTimeMillis(); //t1

            BusinessObjectService businessObjectService = KRADServiceLocator.getBusinessObjectService();
            int bibCount = businessObjectService.countMatching(BibRecord.class, prefixMap);
            Connection connection = null;
            PreparedStatement preparedStatement = null;
            if (bibCount > 0) {
                connection = getConnection();
                if (dbVendor.equalsIgnoreCase("mysql")) {
                    sqlQuery = "select * from ole_ds_bib_t b ORDER BY b.bib_id LIMIT ?,?";
                } else {
                    sqlQuery = "select * from (select b.*,ROWNUM r from OLE_DS_BIB_T b) where  r between ? and ?";
                }
                preparedStatement = connection.prepareStatement(sqlQuery);
            }
            List<ReIndexingBatchStatus> batchStatusList = new ArrayList<ReIndexingBatchStatus>();
            StopWatch loadTimer = new StopWatch();
            StopWatch batchTimer = new StopWatch();
            loadTimer.start();
            batchTimer.start();

            for (int i = 0; i < bibCount; i++) {
                if (bibTrees.size() == batchSize) {
                    if (!isStop()) {

                        ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
                        indexBibDocs(bibTrees, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
                        indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
                        resetTimers(batchTimer, loadTimer);
                        totalCount = 0;
                        logger.info("Rebuild");
                        i = start;
                        if (start % commitSize == 0) {
                            commitStartTime = System.currentTimeMillis(); //t2
                            logger.info("Time elapsed since start ====>>>>>>  " + (commitStartTime - startTime)); // t2-t1
                            logger.info("Time elapsed since last commit ====>>>>>>  "
                                    + (commitStartTime - commitEndTime)); //t2-t3
                            logger.info("commit started ====>>>>>>  " + commitStartTime);
                            SolrServer solr = SolrServerManager.getInstance().getSolrServer();
                            solr.commit();
                            logger.info("No..of records committed ====>>>>>>  " + start);
                            commitEndTime = System.currentTimeMillis(); //t3
                            logger.info("Time Taken for commit ====>>>>>>  " + (commitEndTime - commitStartTime)); //t3-t2

                        }
                    } else {
                        return;
                    }
                } else {
                    if (start < bibCount) {
                        long b2time = System.currentTimeMillis();
                        if (dbVendor.equalsIgnoreCase("mysql")) {
                            preparedStatement.setInt(1, start);
                            preparedStatement.setInt(2, batchSize);
                        } else {
                            preparedStatement.setInt(1, start + 1);
                            preparedStatement.setInt(2, start + batchSize);
                        }
                        ResultSet resultSet = preparedStatement.executeQuery();
                        logger.info("time taking for getting records from DB end======>>>>>"
                                + (System.currentTimeMillis() - b2time));
                        while (resultSet.next()) {

                            BibTree bibTree = new BibTree();
                            Bib bib = new BibMarc();
                            bib.setCategory(docCategory);
                            bib.setType(docType);
                            bib.setFormat(docFormat);
                            bib.setCreatedBy(resultSet.getString("CREATED_BY"));
                            bib.setCreatedOn(resultSet.getString("DATE_CREATED"));
                            bib.setStaffOnly((resultSet.getString("STAFF_ONLY").equalsIgnoreCase("Y") ? Boolean.TRUE
                                    : Boolean.FALSE));
                            bib.setContent(resultSet.getString("CONTENT"));
                            bib.setUpdatedBy(resultSet.getString("UPDATED_BY"));
                            bib.setUpdatedOn(resultSet.getString("DATE_UPDATED"));
                            bib.setLastUpdated(resultSet.getString("DATE_UPDATED"));
                            bib.setStatus(resultSet.getString("STATUS"));
                            bib.setStatusUpdatedBy(resultSet.getString("STATUS_UPDATED_BY"));
                            bib.setStatusUpdatedOn(resultSet.getString("STATUS_UPDATED_DATE"));
                            String uuid = DocumentUniqueIDPrefix
                                    .getPrefixedId(resultSet.getString("UNIQUE_ID_PREFIX"), resultSet.getString(1));
                            bib.setId(uuid);
                            bib.setLocalId(uuid);
                            bibTree.setBib(bib);

                            start++;
                            totalCount++;
                            bibTrees.add(bibTree);
                        }
                        resultSet.close();
                    }
                }

            }
            if (bibTrees.size() > 0 && !isStop()) {
                ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
                indexBibDocs(bibTrees, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
                indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
                commitStartTime = System.currentTimeMillis();
                logger.info("commit started : ---->  " + commitStartTime);
                SolrServer solr = SolrServerManager.getInstance().getSolrServer();
                solr.commit();
                logger.info("No..of records committed : ---->  " + start);
                commitEndTime = System.currentTimeMillis();
                logger.info("Time Taken for commit ======>>>  " + (commitEndTime - commitStartTime));

            }
            endIndexBatch = System.currentTimeMillis(); //t1
            logger.info("Time elapsed since end ====>>>>>>  " + endIndexBatch);
        } catch (Exception e) {
            logger.error("Rebuild Indexes Process(" + docCategory + " : " + docType + " : " + docFormat
                    + ") Processed(" + (totalCount - bibTrees.size()) + "), Failed @ batch(" + bibTrees.size()
                    + "): Cause: " + e, e);
        } finally {
            if (isStop) {
                ReIndexingStatus.getInstance().getDocTypeList().setStatus("Stopped");
            } else {
                ReIndexingStatus.getInstance().getDocTypeList().setStatus("Done");
            }
        }
    }

    private void workInstanceOLEML(String docCategory, String docType, String docFormat) {
        long totalCount = 0;
        long nodeCount = 0;
        List<RequestDocument> docs = new ArrayList<RequestDocument>();
        InstanceOlemlRecordProcessor instanceOlemlRecordProcessor = new InstanceOlemlRecordProcessor();
        try {
            RequestDocument rd = new RequestDocument();
            rd.setCategory(docCategory);
            rd.setType(docType);
            rd.setFormat(docFormat);
            List<ReIndexingBatchStatus> batchStatusList = new ArrayList<ReIndexingBatchStatus>();
            BusinessObjectService businessObjectService = KRADServiceLocator.getBusinessObjectService();
            List<InstanceRecord> instanceRecords = (List<InstanceRecord>) businessObjectService
                    .findAll(InstanceRecord.class);
            StopWatch loadTimer = new StopWatch();
            StopWatch batchTimer = new StopWatch();
            loadTimer.start();
            batchTimer.start();
            for (int i = 0; i < instanceRecords.size(); i++) {
                if (docs.size() == ProcessParameters.BULK_PROCESSOR_SPLIT_SIZE) {
                    if (!isStop()) {
                        ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
                        indexDocs(docs, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
                        indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
                        resetTimers(batchTimer, loadTimer);
                        totalCount = 0;
                        logger.info("Rebuild");
                    } else {
                        return;
                    }
                } else {
                    InstanceRecord instanceRecord = instanceRecords.get(i);
                    String uuid = DocumentUniqueIDPrefix.getPrefixedId(instanceRecord.getUniqueIdPrefix(),
                            instanceRecord.getInstanceId());
                    RequestDocument requestDocument = buildRequestDocumentForCheckout(docCategory, docType,
                            docFormat, uuid);
                    ResponseDocument responseDocument = RdbmsWorkInstanceDocumentManager.getInstance()
                            .checkoutContent(requestDocument, businessObjectService);
                    String content = responseDocument.getContent().getContent();
                    RequestDocument requestDocumentForIndex = (RequestDocument) rd.clone();
                    requestDocumentForIndex.setAdditionalAttributes(responseDocument.getAdditionalAttributes());
                    requestDocumentForIndex.setId(uuid);
                    requestDocumentForIndex.setUuid(uuid);
                    InstanceCollection instanceCollection = instanceOlemlRecordProcessor.fromXML(content);
                    //                    logger.debug("REBUILD_INDEXING_LINKING " + ProcessParameters.REBUILD_INDEXING_LINKING);
                    //                    if (!ProcessParameters.REBUILD_INDEXING_LINKING) {
                    //                        instanceCollection.getInstance().get(0).getResourceIdentifier().clear();
                    //                    }
                    content = instanceOlemlRecordProcessor.toXML(instanceCollection);
                    Content contentObj = new Content();
                    contentObj.setContent(content);
                    contentObj.setContentObject(instanceCollection);
                    requestDocumentForIndex.setContent(contentObj);
                    docs.add(requestDocumentForIndex);
                    totalCount++;
                }
            }
            if (docs.size() > 0 && !isStop()) {
                ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
                indexDocs(docs, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
                indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
            }
        } catch (Exception e) {
            logger.error(
                    "Rebuild Indexes Process(" + docCategory + " : " + docType + " : " + docFormat + ") Processed("
                            + (totalCount - docs.size()) + "), Failed @ batch(" + docs.size() + "): Cause: " + e,
                    e);
        } finally {
            if (isStop) {
                ReIndexingStatus.getInstance().getDocTypeList().setStatus("Stopped");
            } else {
                ReIndexingStatus.getInstance().getDocTypeList().setStatus("Done");
            }
        }
    }

    private RequestDocument buildRequestDocumentForCheckout(String docCategory, String docType, String docFormat,
            String uuid) {
        RequestDocument requestDocument = new RequestDocument();
        requestDocument.setCategory(docCategory);
        requestDocument.setType(docType);
        requestDocument.setFormat(docFormat);
        requestDocument.setUuid(uuid);
        return requestDocument;
    }

    private void linkingInstanceWithBib(InstanceCollection instanceCollection, Session session, Node fileNode) {
        for (Instance instance : instanceCollection.getInstance()) {
            instance.getResourceIdentifier().clear();
            for (FormerIdentifier frids : instance.getFormerResourceIdentifier()) {
                try {
                    if (frids != null && frids.getIdentifier() != null
                            && frids.getIdentifier().getIdentifierValue() != null
                            && frids.getIdentifier().getIdentifierValue().trim().length() >= 0) {
                        List<SolrDocument> solrBibDocs = ServiceLocator.getIndexerService()
                                .getSolrDocument("SystemControlNumber", frids.getIdentifier().getIdentifierValue());
                        SolrInputDocument solrInputDocument = new SolrInputDocument();
                        WorkBibMarcDocBuilder marcDocBuilder = new WorkBibMarcDocBuilder();
                        List<SolrInputDocument> solrInputDocs = new ArrayList<SolrInputDocument>();
                        if (solrBibDocs != null && solrBibDocs.size() > 0) {
                            for (SolrDocument solrbibDoc : solrBibDocs) {
                                if (checkApplicability(frids.getIdentifier().getIdentifierValue(),
                                        solrbibDoc.getFieldValue("SystemControlNumber"))) {

                                    compareObjNAddValue(instance.getInstanceIdentifier(),
                                            solrbibDoc.getFieldValue("instanceIdentifier"), solrbibDoc,
                                            "instanceIdentifier");
                                    solrInputDocument = new SolrInputDocument();
                                    marcDocBuilder.buildSolrInputDocFromSolrDoc(solrbibDoc, solrInputDocument);
                                    solrInputDocs.add(solrInputDocument);
                                    String bibId = compareListRString(solrbibDoc.getFieldValue("id"));
                                    instance.getResourceIdentifier().add(bibId);
                                    modifyContentAddLinkedIdsInDocStore(instance, bibId, session, fileNode);
                                    indexSolrDocs(solrInputDocs);
                                }
                            }
                        }
                    }
                } catch (Exception e) {
                    logger.error("error message" + e.getMessage(), e);
                }
            }
        }
    }

    private void modifyContentAddLinkedIdsInDocStore(Instance instance, String id, Session session, Node fileNode) {

        try {
            Node bibNode = getNodeByUUID(session, id);
            bibNode.setProperty("instanceIdentifier", instance.getInstanceIdentifier());
            fileNode.setProperty("bibIdentifier", id);

            InstanceOlemlRecordProcessor recordProcessor = new InstanceOlemlRecordProcessor();
            NodeIterator nodeIterator = fileNode.getNodes();
            while (nodeIterator.hasNext()) {
                Node instNode = nodeIterator.nextNode();
                if (instNode.getName().equalsIgnoreCase("instanceFile")) {
                    InstanceCollection instCol = new InstanceCollection();
                    Instance inst = new Instance();
                    inst.setResourceIdentifier(instance.getResourceIdentifier());
                    inst.setFormerResourceIdentifier(instance.getFormerResourceIdentifier());
                    inst.setExtension(instance.getExtension());
                    inst.setInstanceIdentifier(instance.getInstanceIdentifier());
                    List<Instance> instanceList = new ArrayList<Instance>();
                    instanceList.add(inst);
                    instCol.setInstance(instanceList);

                    byte[] documentBytes = recordProcessor.toXML(instCol).getBytes();
                    Binary binary = null;
                    if (documentBytes != null && instNode != null && documentBytes.length > 0) {
                        binary = session.getValueFactory().createBinary(new ByteArrayInputStream(documentBytes));
                        instNode.getNode("jcr:content").setProperty("jcr:data", binary);
                    }
                }
            }
        } catch (Exception e) {
            logger.error("error while updating Docstore in reindexing Process" + e.getMessage(), e);
        }
    }

    private void indexSolrDocs(List<SolrInputDocument> solrInputDocs) {

        try {
            ServiceLocator.getIndexerService().indexSolrDocuments(solrInputDocs);
            logger.info("Linking Bib and Instance Records (" + solrInputDocs.size() + "): ");
            solrInputDocs.clear();
        } catch (Exception e) {
            logger.error("Linking Bib and Instance Records (" + (solrInputDocs.size()) + "), Failed @ batch("
                    + solrInputDocs.size() + "): Cause: " + e + "\n\tContinuous", e);
        }
    }

    private boolean checkApplicability(Object value, Object fieldValue) {
        if (fieldValue instanceof Collection) {
            for (Object object : (Collection) fieldValue) {
                if (object.equals(value)) {
                    return true;
                }
            }
            return false;
        } else {
            return value.equals(fieldValue);
        }
    }

    private String compareListRString(Object id) {
        if (id != null) {
            if (id instanceof List) {
                List<String> idList = (List<String>) id;
                return idList.get(0);
            } else if (id instanceof String) {
                String strId = (String) id;
                return strId;
            }
        }
        return null;
    }

    private void compareObjNAddValue(String id, Object idObj, SolrDocument solrDoc, String identifier) {
        if (idObj != null) {
            if (idObj instanceof List) {
                List<String> instBibIdList = (List<String>) idObj;
                if (!instBibIdList.contains(id)) {
                    solrDoc.addField(identifier, id);
                }
            } else if (idObj instanceof String) {
                String instBibId = (String) idObj;
                if (!instBibId.equalsIgnoreCase(id)) {
                    solrDoc.addField(identifier, id);
                }
            }
        } else {
            solrDoc.addField(identifier, id);
        }
    }

    private void workLicense(String docCategory, String docType, String docFormat) {
        Session session = null;
        long totalCount = 0;
        long nodeCount = 0;
        List<RequestDocument> docs = new ArrayList<RequestDocument>();
        try {
            session = RepositoryManager.getRepositoryManager().getSession(ProcessParameters.BULK_DEFAULT_USER,
                    ProcessParameters.BULK_DEFUALT_ACTION);
            RequestDocument rd = new RequestDocument();
            rd.setCategory(docCategory);
            rd.setType(docType);
            rd.setFormat(docFormat);
            DocumentIngester docIngester = new DocumentIngester();
            Node nodeFormat = docIngester.getStaticFormatNode(rd, session);
            NodeIterator nodesL1 = nodeFormat.getNodes();
            List<ReIndexingBatchStatus> batchStatusList = new ArrayList<ReIndexingBatchStatus>();
            StopWatch loadTimer = new StopWatch();
            StopWatch batchTimer = new StopWatch();
            loadTimer.start();
            RepositoryBrowser repositoryBrowser = new RepositoryBrowser();
            while (nodesL1.hasNext()) {
                Node nodeL1 = nodesL1.nextNode();
                NodeIterator nodesFile = nodeL1.getNodes();
                nodeCount = nodesFile.getSize();
                batchTimer.start();
                while (nodesFile.hasNext()) {
                    if (docs.size() == ProcessParameters.BULK_PROCESSOR_SPLIT_SIZE && !isStop()) {
                        if (!isStop()) {
                            ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
                            indexDocs(docs, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
                            indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
                            resetTimers(batchTimer, loadTimer);
                            totalCount = 0;
                            logger.info("Rebuild");
                        } else {
                            return;
                        }
                    } else {

                        Node fileNode = nodesFile.nextNode();
                        String content = null;
                        if (docFormat.equals(DocFormat.ONIXPL.getCode())) {
                            content = checkoutManager.getData(fileNode);
                        } else if (docFormat.equals(DocFormat.PDF.getCode())
                                || docFormat.equals(DocFormat.DOC.getCode())) {
                            content = checkoutManager.checkOutBinary(fileNode.getIdentifier(),
                                    ProcessParameters.BULK_DEFAULT_USER, ProcessParameters.BULK_DEFUALT_ACTION,
                                    docFormat);
                        }
                        RequestDocument reqDoc = (RequestDocument) rd.clone();
                        reqDoc.setId(fileNode.getIdentifier());
                        reqDoc.setUuid(fileNode.getIdentifier());
                        Content contentObj = new Content();
                        contentObj.setContent(content);
                        reqDoc.setContent(contentObj);
                        docs.add(reqDoc);
                        totalCount++;
                    }
                }
            }
            if (docs.size() > 0 && !isStop()) {
                ReIndexingBatchStatus reIndexingBatchStatus = indexBeforeParams(loadTimer);
                indexDocs(docs, totalCount, nodeCount, batchStatusList, reIndexingBatchStatus);
                indexAfterParams(batchTimer, reIndexingBatchStatus, batchStatusList);
            }
        } catch (Exception e) {
            logger.error(
                    "Rebuild Indexes Process(" + docCategory + " : " + docType + " : " + docFormat + ") Processed("
                            + (totalCount - docs.size()) + "), Failed @ batch(" + docs.size() + "): Cause: " + e,
                    e);
        } finally {
            try {
                if (isStop) {
                    ReIndexingStatus.getInstance().getDocTypeList().setStatus("Stopped");
                } else {
                    ReIndexingStatus.getInstance().getDocTypeList().setStatus("Done");
                }
                RepositoryManager.getRepositoryManager().logout(session);
            } catch (OleException e) {
                logger.error(e.getMessage(), e);
            }
        }
    }

    private void resetTimers(StopWatch batchTimer, StopWatch loadTimer) {
        batchTimer.reset();
        batchTimer.start();
        loadTimer.reset();
        loadTimer.start();
    }

    private void indexAfterParams(StopWatch batchTimer, ReIndexingBatchStatus reIndexingBatchStatus,
            List<ReIndexingBatchStatus> batchStatusList) {
        batchTimer.stop();
        reIndexingBatchStatus.setBatchTotalTime(batchTimer.toString());
        ReIndexingStatus.getInstance().getDocTypeList().setReIndBatStatusList(batchStatusList);
    }

    private ReIndexingBatchStatus indexBeforeParams(StopWatch loadTimer) {
        loadTimer.stop();
        ReIndexingBatchStatus reIndexingBatchStatus = new ReIndexingBatchStatus();
        reIndexingBatchStatus.setBatchTotalTime(" ");
        reIndexingBatchStatus.setBatchLoadTime(loadTimer.toString());
        return reIndexingBatchStatus;
    }

    private Node getNodeByUUID(Session newSession, String uuid) throws OleException {
        return new NodeHandler().getNodeByUUID(newSession, uuid);
    }

    public Connection getConnection() {
        Connection connection = null;
        try {
            /* InputStream in =getClass().getResourceAsStream("/mysql.properties");
            Properties properties = new Properties();
            properties.load(in);*/
            String connectionUrl = ConfigContext.getCurrentContextConfig().getProperty("datasource.url");
            String userName = ConfigContext.getCurrentContextConfig().getProperty("datasource.username");
            String passWord = ConfigContext.getCurrentContextConfig().getProperty("datasource.password");
            String driverName = ConfigContext.getCurrentContextConfig().getProperty("jdbc.driver");
            Class.forName(driverName);
            connection = DriverManager.getConnection(connectionUrl, userName, passWord);
        } catch (Exception e) {
            LOG.error("Exception : ", e);
        }
        return connection;
    }

    public String showStatus() {
        BibHoldingItemReindexer bibHoldingItemReindexer = BibHoldingItemReindexer.getInstance();
        return bibHoldingItemReindexer.showStats();
    }

    public String showBibStatus() {
        if (bibInfoStatistics == null) {
            bibInfoStatistics = new BibInfoStatistics();
        }
        return bibInfoStatistics.toString();
    }

    public String storeBibInfo(int batchSize) throws Exception {

        Date date = new Date();
        String STORAGE_EXCEPTION_FILE_NAME = "BibInfoLoadingErrors-" + date.toString() + ".txt";
        String STORAGE_STATUS_FILE_NAME = "BibInfoLoadingStatus" + date.toString() + ".txt";

        long startTime = System.currentTimeMillis();
        bibInfoStatistics = new BibInfoStatistics();
        bibInfoStatistics.setStartDateTime(date);

        bibTreeDBUtil.init(0, 0, null);

        int batchNo = 0;
        int count = bibTreeDBUtil.storeBibInfo(batchSize, filePath, STORAGE_EXCEPTION_FILE_NAME, bibInfoStatistics,
                batchNo);
        long batchStartTime = startTime;
        long batchEndTime = System.currentTimeMillis();
        long totalTimeForBatch = batchEndTime - batchStartTime;
        BatchBibTreeDBUtil.writeStatusToFile(filePath, STORAGE_STATUS_FILE_NAME,
                "Time taken for batch " + totalTimeForBatch);
        while (count > 0) {
            Date batchStartDate = new Date();
            batchStartTime = System.currentTimeMillis();
            bibInfoStatistics.setBatchStartDateTime(batchStartDate);
            count = bibTreeDBUtil.storeBibInfo(batchSize, filePath, STORAGE_EXCEPTION_FILE_NAME, bibInfoStatistics,
                    batchNo++);
            batchEndTime = System.currentTimeMillis();
            Date batchEndDate = new Date();
            bibInfoStatistics.setBatchEndDateTime(batchEndDate);
            bibInfoStatistics.setBatchTotalTime((batchEndTime - batchStartTime));
            totalTimeForBatch = batchEndTime - batchStartTime;
            BatchBibTreeDBUtil.writeStatusToFile(filePath, STORAGE_STATUS_FILE_NAME,
                    "Time taken for batch " + totalTimeForBatch);
        }

        long endTime = System.currentTimeMillis();
        Date endDate = new Date();
        bibInfoStatistics.setEndDateTime(endDate);
        long totalTime = endTime - startTime;
        bibInfoStatistics.setTotalTime(totalTime);
        BatchBibTreeDBUtil.writeStatusToFile(filePath, STORAGE_STATUS_FILE_NAME, "Total Time taken " + totalTime);
        return bibInfoStatistics.toString();
    }

}