Java tutorial
/* * ConcourseConnect * Copyright 2009 Concursive Corporation * http://www.concursive.com * * This file is part of ConcourseConnect, an open source social business * software and community platform. * * Concursive ConcourseConnect is free software: you can redistribute it and/or * modify it under the terms of the GNU Affero General Public License as published * by the Free Software Foundation, version 3 of the License. * * Under the terms of the GNU Affero General Public License you must release the * complete source code for any application that uses any part of ConcourseConnect * (system header files and libraries used by the operating system are excluded). * These terms must be included in any work that has ConcourseConnect components. * If you are developing and distributing open source applications under the * GNU Affero General Public License, then you are free to use ConcourseConnect * under the GNU Affero General Public License. * * If you are deploying a web site in which users interact with any portion of * ConcourseConnect over a network, the complete source code changes must be made * available. For example, include a link to the source archive directly from * your web site. * * For OEMs, ISVs, SIs and VARs who distribute ConcourseConnect with their * products, and do not license and distribute their source code under the GNU * Affero General Public License, Concursive provides a flexible commercial * license. * * To anyone in doubt, we recommend the commercial license. Our commercial license * is competitively priced and will eliminate any confusion about how * ConcourseConnect can be used and distributed. * * ConcourseConnect is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more * details. * * You should have received a copy of the GNU Affero General Public License * along with ConcourseConnect. If not, see <http://www.gnu.org/licenses/>. * * Attribution Notice: ConcourseConnect is an Original Work of software created * by Concursive Corporation */ package com.concursive.connect.indexer; import com.concursive.commons.api.DataField; import com.concursive.commons.api.DataRecord; import com.concursive.commons.api.DataRecordFactory; import com.concursive.connect.Constants; import com.concursive.connect.config.ApplicationPrefs; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.snowball.SnowballAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.RAMDirectory; import java.io.File; import java.io.IOException; import java.lang.reflect.Method; import java.sql.Connection; import java.util.HashMap; import java.util.Map; import java.util.concurrent.locks.ReentrantLock; /** * User: jfielek * Date: Dec 16, 2008 * Time: 3:49:29 PM */ public class LuceneIndexer implements IIndexerService { private static Log LOG = LogFactory.getLog(LuceneIndexer.class); // Only one index reference must exist for the application protected Directory fullIndex = null; protected Directory directoryIndex = null; protected boolean directoryIndexInitialized = false; // Only one writer for each index can exist at any given time private final ReentrantLock writeLock = new ReentrantLock(); protected IndexWriter fullWriter = null; protected IndexWriter directoryWriter = null; // Reuse searchers for performance, and for point-in-time index queries protected LuceneIndexerSearch fullSearcher = null; protected LuceneIndexerSearch directorySearcher = null; private boolean directorySearcherReady = false; // Cache the indexer classes for reuse private Map<String, Object> classes = new HashMap<String, Object>(); /** * Sets up any Lucene Indexer classes * * @param context * @return * @throws Exception */ public boolean setup(IndexerContext context) throws Exception { // Make sure the complex queries can use more than the default clause count BooleanQuery.setMaxClauseCount(Integer.MAX_VALUE); // Setup the Indexes so they are in a state in which they can immediately // be accessed ApplicationPrefs prefs = context.getApplicationPrefs(); { // Establish the full directory LOG.info("Starting Lucene disk index"); File path = new File(prefs.get("FILELIBRARY") + Constants.FULL_INDEX); boolean create = !path.exists(); fullIndex = FSDirectory.getDirectory(prefs.get("FILELIBRARY") + Constants.FULL_INDEX); if (create) { LOG.warn("Lucene index not found, creating new index: " + path.getPath()); Analyzer fullAnalyzer = new StandardAnalyzer(); fullWriter = new IndexWriter(fullIndex, fullAnalyzer, true); fullWriter.close(); } } { // Load up the ram directory LOG.info("Creating Lucene RAM index..."); // Create the Ram Directory directoryIndex = new RAMDirectory(); Analyzer directoryAnalyzer = new SnowballAnalyzer("English"); directoryWriter = new IndexWriter(directoryIndex, directoryAnalyzer, true); directoryWriter.close(); LOG.info("Initialization of RAM index complete..."); } // Search using the updated index resetSearchers(); return true; } /** * Initializes any data * * @param context * @param db * @return * @throws Exception */ public boolean initializeData(IndexerContext context, Connection db) throws Exception { LOG.info("Loading Lucene RAM index..."); // Load default data into the RAM index since it isn't persisted obtainWriterLock(); try { ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.profile.dao.ProjectIndexer")).add(this, db, context); directoryIndexInitialized = true; } catch (Exception e) { LOG.error("Could not initialize RAM index", e); } finally { releaseWriterLock(); } return true; } public boolean reindexAllData(IndexerContext context, Connection db) throws Exception { writeLock.lock(); try { if (context.getIndexType() == Constants.INDEXER_FULL) { // Erase the full index LOG.info("Reloading the full index..."); fullWriter = new IndexWriter(fullIndex, new StandardAnalyzer(), true); // Reindex all data ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.profile.dao.ProjectIndexer")) .add(this, db, context); ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.blog.dao.BlogPostIndexer")) .add(this, db, context); ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.wiki.dao.WikiIndexer")).add(this, db, context); ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.discussion.dao.ForumIndexer")) .add(this, db, context); ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.discussion.dao.TopicIndexer")) .add(this, db, context); ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.discussion.dao.ReplyIndexer")) .add(this, db, context); ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.documents.dao.FileItemIndexer")) .add(this, db, context); // ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.lists.dao.TaskCategoryIndexer")).add(this, db, context); // ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.lists.dao.TaskIndexer")).add(this, db, context); ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.issues.dao.TicketIndexer")) .add(this, db, context); ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.plans.dao.RequirementIndexer")) .add(this, db, context); ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.plans.dao.AssignmentFolderIndexer")) .add(this, db, context); ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.plans.dao.AssignmentIndexer")) .add(this, db, context); ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.plans.dao.AssignmentNoteIndexer")) .add(this, db, context); ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.promotions.dao.AdIndexer")) .add(this, db, context); ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.classifieds.dao.ClassifiedIndexer")) .add(this, db, context); ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.reviews.dao.ProjectRatingIndexer")) .add(this, db, context); // Optimize the index fullWriter.flush(); fullWriter.optimize(); fullWriter.close(); // Update the searchers resetSearchers(); return true; } else if (context.getIndexType() == Constants.INDEXER_DIRECTORY) { // Erase the directory index LOG.info("Reloading the directory index..."); directoryWriter = new IndexWriter(directoryIndex, new SnowballAnalyzer("English"), true); // Just the project data belongs in the directory index ((Indexer) getObjectIndexer("com.concursive.connect.web.modules.profile.dao.ProjectIndexer")) .add(this, db, context); // Optimize the index directoryWriter.flush(); directoryWriter.optimize(); directoryWriter.close(); // Update the searchers resetSearchers(); return true; } } catch (Exception e) { LOG.error("Erase index error", e); } finally { writeLock.unlock(); } return false; } public boolean shutdown() throws IOException { writeLock.lock(); try { if (fullIndex != null) { fullIndex.close(); fullIndex = null; } if (directoryIndex != null) { directoryIndex.close(); directoryIndex = null; } } catch (Exception e) { LOG.error("Shutdown error", e); } finally { writeLock.unlock(); } return false; } public void obtainWriterLock() throws Exception { writeLock.lock(); // Open the full writer try { fullWriter = new IndexWriter(fullIndex, new StandardAnalyzer()); } catch (Exception e) { LOG.error("Begin indexing error on fullWriter", e); } // Open the directory writer try { directoryWriter = new IndexWriter(directoryIndex, new SnowballAnalyzer("English")); } catch (Exception e) { LOG.error("Begin indexing error on directoryWriter", e); } } public void releaseWriterLock() throws Exception { try { // Directory Writer directoryWriter.flush(); directoryWriter.optimize(); directoryWriter.close(); // Full Writer fullWriter.flush(); fullWriter.optimize(); fullWriter.close(); } catch (Exception e) { LOG.error("End indexing error", e); } finally { writeLock.unlock(); } // Let the readers know about the updates resetSearchers(); } public boolean indexDeleteItem(Object item) throws IOException { // Delete the item and related data indexDeleteItem(fullWriter, item, "getDeleteTerm"); // Project index if (item.getClass().getName().equals("com.concursive.connect.web.modules.profile.dao.Project")) { // Delete the item and related data indexDeleteItem(directoryWriter, item, "getDeleteTerm"); } return true; } private boolean indexDeleteItem(IndexWriter writer, Object item, String deleteTerm) throws IOException { // Determine the object's indexer key term for deletion Indexer objectIndexer = (Indexer) getObjectIndexer(item.getClass().getName() + "Indexer"); if (objectIndexer == null) { return false; } // Delete the previous item from the index Object o = null; try { // now we are ready for the next to last step..to call upon the method in the // class instance we have. Method method = objectIndexer.getClass().getMethod(deleteTerm, Object.class); o = method.invoke(objectIndexer, item); } catch (NoSuchMethodException nm) { LOG.error("No Such Method Exception for method " + deleteTerm + ". MESAGE = " + nm.getMessage(), nm); } catch (IllegalAccessException ia) { LOG.error("Illegal Access Exception. MESSAGE = " + ia.getMessage(), ia); } catch (Exception e) { LOG.error("Exception. MESSAGE = " + e.getMessage(), e); } if (o != null) { LOG.debug("Deleting with deleteTerm: " + deleteTerm); writer.deleteDocuments((Term) o); } return true; } public boolean indexAddItem(Object item) { // Adding an item w/o specifically stating the modified state always results in it being // treated as modified return indexAddItem(item, true); } public boolean indexAddItem(Object item, boolean modified) { DataRecordFactory factory = DataRecordFactory.INSTANCE; if (factory == null) { LOG.error("DataRecordFactory IS NULL"); } DataRecord record = factory.parse(item); if (record == null) { LOG.error("DataRecord IS NULL"); } if (LOG.isDebugEnabled() && record != null) { StringBuffer sb = new StringBuffer(); for (DataField thisField : record) { sb.append(thisField.getName()).append("="); if (thisField.hasValue()) { sb.append(thisField.getValue()); } sb.append(System.getProperty("line.separator")); } LOG.debug( "Values (modified? " + modified + "):" + System.getProperty("line.separator") + sb.toString()); } try { // Add to the index LOG.debug("FULL"); indexAddItem(fullWriter, item, modified); if (item.getClass().getName().equals("com.concursive.connect.web.modules.profile.dao.Project")) { LOG.debug("DIRECTORY"); indexAddItem(directoryWriter, item, modified); } } catch (Exception e) { LOG.error("indexAddItem error", e); } return true; } public boolean indexAddItem(Object item, boolean modified, int addType) throws Exception { // Determine the index type if (addType == Constants.INDEXER_FULL) { indexAddItem(fullWriter, item, modified); return true; } else if (addType == Constants.INDEXER_DIRECTORY) { indexAddItem(directoryWriter, item, modified); return true; } return false; } protected boolean indexAddItem(IndexWriter writer, Object item, boolean modified) throws Exception { if (writer == null) { throw new NullPointerException("Writer is null"); } if (item == null) { throw new NullPointerException("Item is null"); } if (modified) { // Delete the previous item from the index indexDeleteItem(writer, item, "getSearchTerm"); } try { Indexer objectIndexer = (Indexer) getObjectIndexer(item.getClass().getName() + "Indexer"); if (objectIndexer == null) { return false; } // Add the item to the index Object o = null; try { // now we are ready for the next to last step..to call upon the method in the // class instance we have. Method method = objectIndexer.getClass().getMethod("add", new Class[] { writer.getClass(), Object.class, boolean.class }); o = method.invoke(objectIndexer, new Object[] { writer, item, new Boolean(modified) }); } catch (NoSuchMethodException nm) { LOG.error("No Such Method Exception for method add. MESAGE = " + nm.getMessage(), nm); } catch (IllegalAccessException ia) { LOG.error("Illegal Access Exception. MESSAGE = " + ia.getMessage(), ia); } catch (Exception e) { LOG.error("Exception. MESSAGE = " + e.getMessage(), e); } } catch (Exception io) { io.printStackTrace(System.out); throw new IOException("Writer: " + io.getMessage()); } return true; } public IIndexerSearch getIndexerSearch(int searchType) { if (searchType == Constants.INDEXER_FULL || !directorySearcherReady) { return fullSearcher; } else if (searchType == Constants.INDEXER_DIRECTORY) { return directorySearcher; } return null; } public float getIndexSizeInBytes(int searchType) { if (searchType == Constants.INDEXER_DIRECTORY && directoryIndex != null) { return ((RAMDirectory) directoryIndex).sizeInBytes(); } return -1; } private void resetSearchers() throws IOException { // New analyzers and readers since there is new data Analyzer fullAnalyzer = new StandardAnalyzer(); LuceneIndexerSearch newFullSearcher = new LuceneIndexerSearch(); newFullSearcher.setup(fullIndex, fullAnalyzer, Constants.INDEXER_FULL); fullSearcher = newFullSearcher; // New analyzers and readers since there is new data Analyzer directoryAnalyzer = new SnowballAnalyzer("English"); LuceneIndexerSearch newDirectorySearcher = new LuceneIndexerSearch(); newDirectorySearcher.setup(directoryIndex, directoryAnalyzer, Constants.INDEXER_DIRECTORY); directorySearcher = newDirectorySearcher; if (directoryIndexInitialized) { directorySearcherReady = true; } } private Object getObjectIndexer(String className) { Object classRef = null; if (classes.containsKey(className)) { classRef = classes.get(className); } else { try { classRef = Class.forName(className).newInstance(); classes.put(className, classRef); } catch (ClassNotFoundException cnfe) { LOG.warn("Class Not Found Exception. MESSAGE = " + cnfe.getMessage(), cnfe); } catch (InstantiationException ie) { LOG.error("Instantiation Exception. MESSAGE = " + ie.getMessage(), ie); } catch (IllegalAccessException iae) { LOG.error("Illegal Argument Exception. MESSAGE = " + iae.getMessage(), iae); } } return classRef; } }