in.cdac.medinfo.csnotk.csnolib.agents.LuceneSearchAgent.java Source code

Java tutorial

Introduction

Here is the source code for in.cdac.medinfo.csnotk.csnolib.agents.LuceneSearchAgent.java

Source

/*******************************************************************************
 * Copyright 2014 Centre for Development of Advanced Computing(C-DAC), Pune
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ******************************************************************************/
package in.cdac.medinfo.csnotk.csnolib.agents;

import in.cdac.medinfo.csnotk.csnolib.api.SearchAgent;
import in.cdac.medinfo.csnotk.csnolib.commons.Constants;
import in.cdac.medinfo.csnotk.csnolib.commons.EnumSuffix;
import in.cdac.medinfo.csnotk.csnolib.db.DBUtil;
import in.cdac.medinfo.csnotk.csnolib.model.Description;
import in.cdac.medinfo.csnotk.csnolib.util.CSNOLogger;
import in.cdac.medinfo.csnotk.csnolib.util.KeyBasedComparator;
import in.cdac.medinfo.csnotk.csnolib.util.PropertyReader;
import in.cdac.medinfo.csnotk.csnolib.util.SnoDomainUtil;

import java.io.File;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;

/**
 * Implementation of document search functionality with Lucene version 47.0.
 * Search functionality and implementation of index creation is provided by this class.
 */
public class LuceneSearchAgent implements SearchAgent {

    private static final long serialVersionUID = 1L;
    /**
     * static Fields
     */
    private IndexWriter indexWriter = null;
    private IndexSearcher searcher = null;
    private Analyzer analyzer = null;
    private IndexWriterConfig indexConfig = null;
    private static Directory indexDirectory = null;
    private static IndexReader indexReader = null;
    private static Object mutex = new Object();

    /**
     * 
     * @return indexWriter
     * @throws IOException
     */
    private IndexWriter getIndexWriter() throws IOException {
        if (this.indexWriter == null) {
            this.indexWriter = new IndexWriter(getIndexDirectorytoWrite(), getIndexConfig());
        }
        return indexWriter;
    }

    /**
     * 
     * @return indexReader
     * @throws IOException
     */
    private IndexReader getIndexReader() throws IOException {
        if (indexReader == null) {
            synchronized (mutex) {
                indexReader = DirectoryReader.open(getIndexDirectorytoRead());
            }
        }
        return indexReader;
    }

    /**
     * 
     * @param reader
     * @return indexSearcher
     */
    private IndexSearcher getIndexSearcher(IndexReader reader) {
        if (this.searcher == null) {
            this.searcher = new IndexSearcher(reader);
        }
        return this.searcher;
    }

    /**
     * 
     * @return analyzer
     */
    private Analyzer getAnalyzer() {
        if (this.analyzer == null) {
            this.analyzer = new StandardAnalyzer(Version.LUCENE_47);
            ;
        }
        return this.analyzer;
    }

    /**
     * 
     * @return indexWriter
     */
    private IndexWriterConfig getIndexConfig() {
        if (this.indexConfig == null) {
            this.indexConfig = new IndexWriterConfig(Version.LUCENE_47, this.getAnalyzer());
        }
        return this.indexConfig;
    }

    /**
     * 
     * @return indexDirectory
     * @throws IOException
     */
    private static Directory getIndexDirectorytoRead() throws IOException {
        if (indexDirectory == null) {
            synchronized (mutex) {
                Properties properties = null;
                try {
                    properties = PropertyReader.loadSystemProperties();
                } catch (IOException e) {
                    //Get the Stack trace and form the exception message.
                    StackTraceElement arrStackTraceElement[];
                    arrStackTraceElement = e.getStackTrace();
                    String strMessage = e.getClass() + Constants.NEW_LINE + Constants.CLASS_NAME
                            + arrStackTraceElement[0].getClassName() + Constants.NEW_LINE + Constants.METHOD_NAME
                            + arrStackTraceElement[0].getMethodName() + Constants.NEW_LINE + Constants.LINE_NUMBER
                            + arrStackTraceElement[0].getLineNumber() + Constants.NEW_LINE
                            + Constants.MESSAGE_DESCRIPTION + e.getMessage();
                    //Log the Exception
                    CSNOLogger.logException(strMessage);
                }
                //public RAMDirectory(Directory dir,IOContext context) throws IOException
                //Creates a new RAMDirectory instance from a different Directory implementation. This can be used to 
                //load a disk-based index into memory.

                //Warning: This class is not intended to work with huge indexes. Everything beyond several hundred 
                //megabytes will waste resources (GC cycles), because it uses an internal buffer size of 1024 bytes,
                //producing millions of byte[1024] arrays. This class is optimized for small memory-resident indexes. 
                //It also has bad concurrency on multithreaded environments.
                //Note that the resulting RAMDirectory instance is fully independent from the original Directory (it is a complete copy).
                //Any subsequent changes to the original Directory will not be visible in the RAMDirectory instance.
                indexDirectory = new RAMDirectory(
                        FSDirectory.open(new File(properties.getProperty(Constants.INDEX_DIR))),
                        IOContext.READONCE);
            }
        }
        return indexDirectory;
    }

    private static Directory getIndexDirectorytoWrite() throws IOException {
        if (indexDirectory == null) {
            Properties properties = null;
            try {
                properties = PropertyReader.loadSystemProperties();
            } catch (IOException e) {
                //Get the Stack trace and form the exception message.
                StackTraceElement arrStackTraceElement[];
                arrStackTraceElement = e.getStackTrace();
                String strMessage = e.getClass() + Constants.NEW_LINE + Constants.CLASS_NAME
                        + arrStackTraceElement[0].getClassName() + Constants.NEW_LINE + Constants.METHOD_NAME
                        + arrStackTraceElement[0].getMethodName() + Constants.NEW_LINE + Constants.LINE_NUMBER
                        + arrStackTraceElement[0].getLineNumber() + Constants.NEW_LINE
                        + Constants.MESSAGE_DESCRIPTION + e.getMessage();
                //Log the Exception
                CSNOLogger.logException(strMessage);
            }
            indexDirectory = FSDirectory.open(new File(properties.getProperty(Constants.INDEX_DIR)));
        }
        return indexDirectory;
    }

    /**
     * 
     * @param descriptionList
     * @return indexWriter
     * @throws IOException
     */
    //@SuppressWarnings("deprecation")
    private IndexWriter addDescriptionsDocument(List<Description> descriptionList) throws IOException {

        for (Description description : descriptionList) {
            /* create document to index
             */
            Document document = new Document();

            /* create fields to be indexed for document
             */
            Field idField = new TextField(Constants.ID, description.getId() + "", Field.Store.YES);
            Field activeField = new TextField(Constants.ACTIVE, description.getStatus() + "", Field.Store.YES);
            Field conceptIdField = new TextField(Constants.CONCEPTID, description.getConceptId() + "",
                    Field.Store.YES);
            Field typeIdField = new TextField(Constants.TYEPID, description.getType() + "", Field.Store.YES);

            //As term need be indexed and to be shown in results , so use it as a text field.
            Field termField = new TextField(Constants.TERM, description.getTerm(), Field.Store.YES);

            /* Add fields to document
             */
            document.add(idField);
            document.add(activeField);
            document.add(conceptIdField);
            document.add(typeIdField);
            document.add(termField);

            /* add document to index writer
             */
            this.indexWriter.addDocument(document);
            //   if(document.toString().contains("Mnire"))
            //      System.out.println("Found");
            //   else
            //      System.out.println(document);

            //   Document doc = new Document();
            //   doc.add(new TextField("title", title, Field.Store.YES));

            // use a string field for isbn because we don't want it tokenized
            //   doc.add(new StringField("isbn", isbn, Field.Store.YES));
            //   w.addDocument(doc);
        }
        return this.indexWriter;
    }

    /**
     * @param query
     * @param hitsPerPage
     * @param inOrder
     * @return scoreDocs
     * @throws IOException
     */
    private ScoreDoc[] search(Query query, int hitsPerPage, boolean inOrder) throws IOException {
        Properties properties = null;
        int maxRecords = 10;
        try {
            properties = PropertyReader.loadSystemProperties();
            maxRecords = Integer.parseInt(properties.getProperty("max.records"));
        } catch (IOException e) {
            //Get the Stack trace and form the exception message.
            StackTraceElement arrStackTraceElement[];
            arrStackTraceElement = e.getStackTrace();
            String strMessage = e.getClass() + Constants.NEW_LINE + Constants.CLASS_NAME
                    + arrStackTraceElement[0].getClassName() + Constants.NEW_LINE + Constants.METHOD_NAME
                    + arrStackTraceElement[0].getMethodName() + Constants.NEW_LINE + Constants.LINE_NUMBER
                    + arrStackTraceElement[0].getLineNumber() + Constants.NEW_LINE + Constants.MESSAGE_DESCRIPTION
                    + e.getMessage();
            //Log the Exception
            CSNOLogger.logException(strMessage);
        }

        TopScoreDocCollector topScoreDocCollector = null;

        if (hitsPerPage == 0) {
            hitsPerPage = hitsPerPage + maxRecords;
            topScoreDocCollector = TopScoreDocCollector.create(hitsPerPage, inOrder);
            topScoreDocCollector = TopScoreDocCollector.create(Constants.DESCRIPTION_All_RECORDS, inOrder);

        } else {
            topScoreDocCollector = TopScoreDocCollector.create(hitsPerPage, inOrder);
        }
        this.getIndexSearcher(this.getIndexReader()).search(query, topScoreDocCollector);

        TopDocs topDocs = topScoreDocCollector.topDocs();

        //System.out.println("TOP DOC Size: " + topDocs.scoreDocs.length);

        return topDocs.scoreDocs;
    }

    /**
     * 
     * @param searchTerm
     * @param suffix
     * @return descriptions
     * @throws ParseException
     */
    private Set<Description> termSuffixSuggest(String searchTerm, EnumSuffix suffix, int maxRecords)
            throws ParseException {

        String originalTerm = searchTerm;
        QueryParser queryParser = new QueryParser(Version.LUCENE_47, Constants.TERM, getAnalyzer());
        String suffixValue = "(" + suffix.getValue().toLowerCase() + ")";
        searchTerm = searchTerm.toLowerCase() + "*";
        Query query = null;
        ScoreDoc[] results = null;
        try {
            query = queryParser.parse(suffixValue + "AND " + searchTerm);
            this.getIndexSearcher(this.getIndexReader());
            results = this.search(query, maxRecords, true);
            return getTopDescriptions(results, originalTerm);
        } catch (ParseException e) {
            //Get the Stack trace and form the exception message.
            StackTraceElement arrStackTraceElement[];
            arrStackTraceElement = e.getStackTrace();
            String strMessage = e.getClass() + Constants.NEW_LINE + Constants.CLASS_NAME
                    + arrStackTraceElement[0].getClassName() + Constants.NEW_LINE + Constants.METHOD_NAME
                    + arrStackTraceElement[0].getMethodName() + Constants.NEW_LINE + Constants.LINE_NUMBER
                    + arrStackTraceElement[0].getLineNumber() + Constants.NEW_LINE + Constants.MESSAGE_DESCRIPTION
                    + e.getMessage();
            //Log the Exception
            CSNOLogger.logException(strMessage);
        } catch (IOException e) {
            //Get the Stack trace and form the exception message.
            StackTraceElement arrStackTraceElement[];
            arrStackTraceElement = e.getStackTrace();
            String strMessage = e.getClass() + Constants.NEW_LINE + Constants.CLASS_NAME
                    + arrStackTraceElement[0].getClassName() + Constants.NEW_LINE + Constants.METHOD_NAME
                    + arrStackTraceElement[0].getMethodName() + Constants.NEW_LINE + Constants.LINE_NUMBER
                    + arrStackTraceElement[0].getLineNumber() + Constants.NEW_LINE + Constants.MESSAGE_DESCRIPTION
                    + e.getMessage();
            //Log the Exception
            CSNOLogger.logException(strMessage);
        }
        return null;
    }

    /**
     * @param searchTerm
     * @param suffix
     * @return descriptions
     */
    @Override
    public Set<Description> search(String searchTerm, EnumSuffix suffix, int maxRecords) {
        try {
            return termSuffixSuggest(searchTerm, suffix, maxRecords);
        } catch (ParseException e) {
            //Get the Stack trace and form the exception message.
            StackTraceElement arrStackTraceElement[];
            arrStackTraceElement = e.getStackTrace();
            String strMessage = e.getClass() + Constants.NEW_LINE + Constants.CLASS_NAME
                    + arrStackTraceElement[0].getClassName() + Constants.NEW_LINE + Constants.METHOD_NAME
                    + arrStackTraceElement[0].getMethodName() + Constants.NEW_LINE + Constants.LINE_NUMBER
                    + arrStackTraceElement[0].getLineNumber() + Constants.NEW_LINE + Constants.MESSAGE_DESCRIPTION
                    + e.getMessage();
            //Log the Exception
            CSNOLogger.logException(strMessage);
        }
        return null;
    }

    /**
     * @param suffix
     * @param maxRecords
     * @return descriptions
     */
    @Override
    public Set<Description> search(EnumSuffix suffix, int maxRecords) {
        String suffixValue = "(" + suffix.getValue().toLowerCase() + ")";
        QueryParser queryParser = new QueryParser(Version.LUCENE_47, Constants.TERM, getAnalyzer());
        ScoreDoc[] results = null;
        try {
            Query query = queryParser.parse("(" + suffixValue + ")");
            this.getIndexSearcher(this.getIndexReader());
            results = this.search(query, maxRecords, true);
            return getTopDescriptions(results, null);
        } catch (ParseException e) {
            //Get the Stack trace and form the exception message.
            StackTraceElement arrStackTraceElement[];
            arrStackTraceElement = e.getStackTrace();
            String strMessage = e.getClass() + Constants.NEW_LINE + Constants.CLASS_NAME
                    + arrStackTraceElement[0].getClassName() + Constants.NEW_LINE + Constants.METHOD_NAME
                    + arrStackTraceElement[0].getMethodName() + Constants.NEW_LINE + Constants.LINE_NUMBER
                    + arrStackTraceElement[0].getLineNumber() + Constants.NEW_LINE + Constants.MESSAGE_DESCRIPTION
                    + e.getMessage();
            //Log the Exception
            CSNOLogger.logException(strMessage);
        } catch (IOException e) {
            //Get the Stack trace and form the exception message.
            StackTraceElement arrStackTraceElement[];
            arrStackTraceElement = e.getStackTrace();
            String strMessage = e.getClass() + Constants.NEW_LINE + Constants.CLASS_NAME
                    + arrStackTraceElement[0].getClassName() + Constants.NEW_LINE + Constants.METHOD_NAME
                    + arrStackTraceElement[0].getMethodName() + Constants.NEW_LINE + Constants.LINE_NUMBER
                    + arrStackTraceElement[0].getLineNumber() + Constants.NEW_LINE + Constants.MESSAGE_DESCRIPTION
                    + e.getMessage();
            //Log the Exception
            CSNOLogger.logException(strMessage);
        }
        return null;
    }

    // Following method can be used for searching based on Phrase Query for exact match of term
    /**
     * @param inputQuery
     * @param maxRecords
     * @return top Descriptions
     */

    //   @Override
    //   public Set<Description> search(String inputQuery, int maxRecords)
    //   {
    //      boolean searchOnConceptID= false;
    //      inputQuery = inputQuery.trim();
    //      PhraseQuery query = new PhraseQuery();
    //      
    //      //following logic is top check whether search is on Concept ID or Term
    //      try
    //      {
    //         if(null != inputQuery && !inputQuery.isEmpty())
    //         {
    //            Integer.parseInt(inputQuery);
    //            searchOnConceptID = true;
    //         }
    //      }
    //      catch (NumberFormatException nfe) 
    //      {
    //         searchOnConceptID = false;
    //      }
    //      inputQuery = inputQuery.toLowerCase();
    //      //following regex is used for removing all special characters from search term
    //      inputQuery = inputQuery.replaceAll("[^\\w\\s\\-_]", "");
    //      //following regex is used for removing underscore * hyphen
    //      inputQuery = inputQuery.replaceAll("_", "").replaceAll("-", "");
    //
    //      ScoreDoc[] results = null;
    //      
    //      if(!searchOnConceptID)
    //      {
    //         //following regex is used for removing numbers
    //         inputQuery = inputQuery.replaceAll("[\\d]", "");
    //         String[] words = inputQuery.split(" ");
    //         for (String word : words) 
    //         {
    //            query.add(new Term(Constants.TERM, word));
    //         }
    //      }
    //      else
    //      {
    //         query.add(new Term(Constants.ID, inputQuery));
    //      }
    //      
    //      try
    //      {
    //         results = this.search(query, maxRecords, true);
    //         return getTopDescriptions(results,inputQuery);
    //      }
    //      catch (IOException e) 
    //      {
    //         //Get the Stack trace and form the exception message.
    //         StackTraceElement arrStackTraceElement[];
    //         arrStackTraceElement = e.getStackTrace();
    //         String strMessage = e.getClass() + Constants.NEW_LINE + Constants.CLASS_NAME + arrStackTraceElement[0].getClassName() + Constants.NEW_LINE + Constants.METHOD_NAME + arrStackTraceElement[0].getMethodName() + Constants.NEW_LINE +  Constants.LINE_NUMBER + arrStackTraceElement[0].getLineNumber()+ Constants.NEW_LINE +  Constants.MESSAGE_DESCRIPTION + e.getMessage();
    //         //Log the Exception
    //         CSNOLogger.logException(strMessage);
    //      }
    //      return null;
    //   } 
    /**
     * @param inputQuery
     * @param maxRecords
     * @return top Descriptions
     */
    @Override
    public Set<Description> search(String inputQuery, int maxRecords) {
        boolean searchOnID = false;
        inputQuery = inputQuery.trim();

        //following logic is top check whether search is on Concept ID or Term
        /**
         * JAVA SCRIPT LONG LIMIT:  +/-9007199254740992
         * JAVA LONG LIMIT: +/- 9223372036854775807
         * Sample SNOMED ID: 900000000000441000
         */
        try {
            if (null != inputQuery && !inputQuery.isEmpty()) {
                Long.parseLong(inputQuery);
                if (inputQuery.length() < 6) {
                    searchOnID = false;
                } else {
                    searchOnID = true;
                }

            }
        } catch (NumberFormatException nfe) {
            searchOnID = false;
        }

        String originalTerm = inputQuery;
        inputQuery = "*" + inputQuery.toLowerCase() + "*";
        ScoreDoc[] results = null;

        QueryParser queryParser = null;
        Set<Description> emptySet = Collections.emptySet();
        if (searchOnID) {
            String manipulate = originalTerm;
            String checkPoint = manipulate.substring(manipulate.length() - 3, manipulate.length() - 1);
            if (checkPoint.equalsIgnoreCase(Constants.CONCEPT_CHECKPONT)) {
                queryParser = new QueryParser(Version.LUCENE_47, Constants.CONCEPTID, getAnalyzer());
            } else if (checkPoint.equalsIgnoreCase(Constants.DESC_CHECKPONT)) {
                queryParser = new QueryParser(Version.LUCENE_47, Constants.ID, getAnalyzer());
            } else {
                //return null;
                return emptySet;
            }
        } else {
            if (inputQuery.contains(":")) {
                inputQuery = inputQuery.replace(":", ": ");
            }
            //TODO: Re-factoring needed. Contains should be replaced with Regular Expression to check for the special characters.
            if (inputQuery.contains("}") || inputQuery.contains("{") || inputQuery.contains("]")
                    || inputQuery.contains("]") || inputQuery.contains(",") || inputQuery.contains("+")
                    || inputQuery.contains("-") || inputQuery.contains("&&") || inputQuery.contains("||")
                    || inputQuery.contains("!") || inputQuery.contains("^") || inputQuery.contains("\"")
                    || inputQuery.contains("~") || inputQuery.contains(".") || inputQuery.contains("//")
                    || inputQuery.contains("(") || inputQuery.contains(")") || inputQuery.contains("'")
                    || inputQuery.contains("<") || inputQuery.contains(">") || inputQuery.contains("=")
                    || inputQuery.contains(":") || inputQuery.contains("`") || inputQuery.contains("@")
                    || inputQuery.contains("#") || inputQuery.contains("_") || inputQuery.contains(";")
                    || inputQuery.contains("0") || inputQuery.contains("1") || inputQuery.contains("2")
                    || inputQuery.contains("3") || inputQuery.contains("4") || inputQuery.contains("5")
                    || inputQuery.contains("6") || inputQuery.contains("7") || inputQuery.contains("8")
                    || inputQuery.contains("9")) {
                inputQuery = QueryParser.escape(inputQuery);
                //System.out.println("INPUTQUERY : " + inputQuery);
            }
            queryParser = new QueryParser(Version.LUCENE_47, Constants.TERM, getAnalyzer());
        }
        queryParser.setDefaultOperator(QueryParser.Operator.AND);
        queryParser.setAllowLeadingWildcard(true);

        try {
            Query query = null;
            query = queryParser.parse(inputQuery);
            //System.out.println("PARSED QUERY : " + query);
            results = this.search(query, maxRecords, true);
            return getTopDescriptions(results, originalTerm);
        } catch (ParseException e) {
            //Get the Stack trace and form the exception message.
            StackTraceElement arrStackTraceElement[];
            arrStackTraceElement = e.getStackTrace();
            String strMessage = e.getClass() + Constants.NEW_LINE + Constants.CLASS_NAME
                    + arrStackTraceElement[0].getClassName() + Constants.NEW_LINE + Constants.METHOD_NAME
                    + arrStackTraceElement[0].getMethodName() + Constants.NEW_LINE + Constants.LINE_NUMBER
                    + arrStackTraceElement[0].getLineNumber() + Constants.NEW_LINE + Constants.MESSAGE_DESCRIPTION
                    + e.getMessage();
            //Log the Exception
            CSNOLogger.logException(strMessage);
        } catch (IOException e) {
            //Get the Stack trace and form the exception message.
            StackTraceElement arrStackTraceElement[];
            arrStackTraceElement = e.getStackTrace();
            String strMessage = e.getClass() + Constants.NEW_LINE + Constants.CLASS_NAME
                    + arrStackTraceElement[0].getClassName() + Constants.NEW_LINE + Constants.METHOD_NAME
                    + arrStackTraceElement[0].getMethodName() + Constants.NEW_LINE + Constants.LINE_NUMBER
                    + arrStackTraceElement[0].getLineNumber() + Constants.NEW_LINE + Constants.MESSAGE_DESCRIPTION
                    + e.getMessage();
            //Log the Exception
            CSNOLogger.logException(strMessage);
        }
        return emptySet;
    }

    /**
     * 
     * @param results
     * @return descriptions
     * @throws IOException
     */
    private Set<Description> getTopDescriptions(ScoreDoc[] results, String term) throws IOException {
        Set<Description> topDescriptions = null;
        List<Description> list = new ArrayList<Description>();
        for (int i = 0; i < results.length; i++) {
            int docId = results[i].doc;
            Document document = this.getIndexSearcher(getIndexReader()).doc(docId);
            Description description = new Description();
            description.setTerm(document.get(Constants.TERM));
            description.setId(document.get(Constants.ID));
            description.setStatus(Short.parseShort(document.get(Constants.ACTIVE)));
            description.setConceptId(document.get(Constants.CONCEPTID));
            description.setType(document.get(Constants.TYEPID));
            list.add(description);
        }
        if (term != null && !term.isEmpty()) {
            Collections.sort(list, new KeyBasedComparator(term));
        }
        topDescriptions = new LinkedHashSet<Description>(list);

        return topDescriptions;
    }

    /**
     * Lucene index creation utility.
     */
    private void createIndex() {
        try {
            Properties properties = null;
            long maxRecords = 10;
            try {
                properties = PropertyReader.loadSystemProperties();
                maxRecords = Integer.parseInt(properties.getProperty("max.records"));
            } catch (IOException e) {
                //Get the Stack trace and form the exception message.
                StackTraceElement arrStackTraceElement[];
                arrStackTraceElement = e.getStackTrace();
                String strMessage = e.getClass() + Constants.NEW_LINE + Constants.CLASS_NAME
                        + arrStackTraceElement[0].getClassName() + Constants.NEW_LINE + Constants.METHOD_NAME
                        + arrStackTraceElement[0].getMethodName() + Constants.NEW_LINE + Constants.LINE_NUMBER
                        + arrStackTraceElement[0].getLineNumber() + Constants.NEW_LINE
                        + Constants.MESSAGE_DESCRIPTION + e.getMessage();
                //Log the Exception
                CSNOLogger.logException(strMessage);
            }

            this.getIndexWriter();
            SnoDomainUtil domainUtil = new SnoDomainUtil();

            long offset = 0;

            String url = properties.getProperty("jdbc_url");
            String userName = properties.getProperty("username");
            String password = properties.getProperty("password");

            Connection connection = DBUtil.getConnection(url, userName, password);
            Statement statement = DBUtil.getStatement(connection);

            long totalRecords = DBUtil.getTotalRows("sct2_description", statement);
            DBUtil.closeStatement(statement);
            DBUtil.closeConnection(connection);

            domainUtil.setOffset(offset);
            domainUtil.setNumberOfRecords(maxRecords);

            if (totalRecords > 0) {
                do {
                    this.addDescriptionsDocument(
                            domainUtil.getDomain(Constants.descriptionClass, offset, maxRecords));
                    offset = offset + maxRecords;
                    domainUtil.setOffset(offset);
                    domainUtil.setNumberOfRecords(maxRecords);
                } while (totalRecords > offset);
            }

            this.getIndexWriter().close();

        } catch (IOException e) {
            //Get the Stack trace and form the exception message.
            StackTraceElement arrStackTraceElement[];
            arrStackTraceElement = e.getStackTrace();
            String strMessage = e.getClass() + Constants.NEW_LINE + Constants.CLASS_NAME
                    + arrStackTraceElement[0].getClassName() + Constants.NEW_LINE + Constants.METHOD_NAME
                    + arrStackTraceElement[0].getMethodName() + Constants.NEW_LINE + Constants.LINE_NUMBER
                    + arrStackTraceElement[0].getLineNumber() + Constants.NEW_LINE + Constants.MESSAGE_DESCRIPTION
                    + e.getMessage();
            //Log the Exception
            CSNOLogger.logException(strMessage);
        } catch (SQLException e) {
            //Get the Stack trace and form the exception message.
            StackTraceElement arrStackTraceElement[];
            arrStackTraceElement = e.getStackTrace();
            String strMessage = e.getClass() + Constants.NEW_LINE + Constants.CLASS_NAME
                    + arrStackTraceElement[0].getClassName() + Constants.NEW_LINE + Constants.METHOD_NAME
                    + arrStackTraceElement[0].getMethodName() + Constants.NEW_LINE + Constants.LINE_NUMBER
                    + arrStackTraceElement[0].getLineNumber() + Constants.NEW_LINE + Constants.MESSAGE_DESCRIPTION
                    + e.getMessage();
            //Log the Exception
            CSNOLogger.logException(strMessage);
        }
    }

    /**
    For Deleteting the old created index
    */
    private void deleteIndex() {

        Properties properties = null;

        try {
            properties = PropertyReader.loadSystemProperties();
            indexDirectory = FSDirectory.open(new File(properties.getProperty(Constants.INDEX_DIR)));

            if (indexDirectory != null) {
                String[] list = indexDirectory.listAll();

                if (list != null) {
                    for (int i = 0; i < list.length; i++) {
                        indexDirectory.deleteFile(list[i].toString());

                    }
                    //indexDirectory = FSDirectory.open(new File(properties.getProperty(Constants.INDEX_DIR)));   
                }
            }

        } catch (IOException e) {
            StackTraceElement arrStackTraceElement[];
            arrStackTraceElement = e.getStackTrace();
            String strMessage = e.getClass() + Constants.NEW_LINE + Constants.CLASS_NAME
                    + arrStackTraceElement[0].getClassName() + Constants.NEW_LINE + Constants.METHOD_NAME
                    + arrStackTraceElement[0].getMethodName() + Constants.NEW_LINE + Constants.LINE_NUMBER
                    + arrStackTraceElement[0].getLineNumber() + Constants.NEW_LINE + Constants.MESSAGE_DESCRIPTION
                    + e.getMessage();
            //Log the Exception
            CSNOLogger.logException(strMessage);

        }

    }

    /**
     * For testing purpose
     * @param args
     * @throws ParseException
     * @throws IOException
     * @throws SQLException
     */
    public static void main(String[] args) throws ParseException, IOException, SQLException {
        LuceneSearchAgent searchAgent = new LuceneSearchAgent();
        // Delete obsulate lucene index if exist
        searchAgent.deleteIndex();
        // Create new lucene index for SNOMED CT term description
        searchAgent.createIndex();

        //System.out.println(new LuceneSearchAgent().search("Myocardial infarction", 5000));
    }
}