Java tutorial
/********************************************************************************** * $URL: https://source.sakaiproject.org/svn/search/trunk/search-impl/impl/src/java/org/sakaiproject/search/journal/impl/JournaledFSIndexStorage.java $ * $Id: JournaledFSIndexStorage.java 105078 2012-02-24 23:00:38Z ottenhoff@longsight.com $ *********************************************************************************** * * Copyright (c) 2003, 2004, 2005, 2006, 2007, 2008, 2009 The Sakai Foundation * * Licensed under the Educational Community License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.opensource.org/licenses/ECL-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * **********************************************************************************/ package org.sakaiproject.search.journal.impl; import java.io.File; import java.io.IOException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantReadWriteLock; import javax.sql.DataSource; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.IndexWriter.MaxFieldLength; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.spell.Dictionary; import org.apache.lucene.search.spell.LuceneDictionary; import org.apache.lucene.search.spell.SpellChecker; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.NIOFSDirectory; import org.sakaiproject.cluster.api.ClusterService; import org.sakaiproject.component.api.ServerConfigurationService; import org.sakaiproject.search.api.SearchService; import org.sakaiproject.search.index.AnalyzerFactory; import org.sakaiproject.search.journal.api.IndexListener; import org.sakaiproject.search.journal.api.IndexMonitorListener; import org.sakaiproject.search.journal.api.IndexStorageProvider; import org.sakaiproject.search.journal.api.JournalErrorException; import org.sakaiproject.search.journal.api.JournalManager; import org.sakaiproject.search.journal.api.JournaledIndex; import org.sakaiproject.search.journal.api.ThreadBinder; import org.sakaiproject.search.transaction.api.IndexTransactionException; import org.sakaiproject.search.util.FileUtils; import org.sakaiproject.thread_local.api.ThreadLocalManager; /** * <pre> * This is a Journaled savePoint of the local FSIndexStorage. It will merge in new * savePoints from the journal. This is going to be performed in a non * transactional way for the moment. * * The index reader must maintain a single * index reader for the JVM. When performing a read update, the single index * reader must be used, but each time the index reader is provided we should * check that the index reader has not been updated. * * If the reader is being updated, then it is not safe to reload it. * </pre> * * @author ieb */ public class JournaledFSIndexStorage implements JournaledIndex, IndexStorageProvider { private static final Log log = LogFactory.getLog(JournaledFSIndexStorage.class); private static final String SEGMENT_LIST_NAME = "local-segments"; /** * Sakai config service */ private ServerConfigurationService serverConfigurationService; /** * The server Id */ private String serverId; /** * The datasource IoC injected */ private DataSource datasource; /** * A lock to protect the local index */ private ReentrantReadWriteLock rwlock = new ReentrantReadWriteLock(true); /** * A holder on the thread used to store the journal entry that is being * processed by the thread */ private ThreadLocal<Long> lastJournalEntryHolder = new ThreadLocal<Long>(); /** * The journal manager that manages the journal that feeds this index */ private JournalManager journalManager; /** * A list of temporary segments */ private List<File> segments = new ArrayList<File>(); /** * the timestamp when the index was updated */ private long lastUpdate; /** * The number of the journal that was last applied to this index */ private long journalSavePoint = -1; /** * The current reader used for deletes and searching, this is a singleton, * only reloaded when changes in the index are detected. A read lock is * required to allow reloading of the index */ private MultiReader multiReader; /** * Listeners of this index */ private List<IndexListener> indexListeners = new ArrayList<IndexListener>(); /** * Have the segments been modified */ private boolean modified = false; /** * A permanent indexWriter. To get the permanent Index Writer a write lock * must be taken on the index. This will not prevent access to the * multReader, but it will prevent it from being reloaded */ private IndexWriter permanentIndexWriter; private AnalyzerFactory analyzerFactory; private IndexSearcher indexSearcher = null; private ClusterService clusterService; private long lastLoad; private long lastLoadTime; private JournalSettings journalSettings; private ThreadLocalManager threadLocalManager; public void destroy() { } /** * @see org.sakaiproject.search.index.impl.FSIndexStorage#init() */ public void init() { serverId = serverConfigurationService.getServerId(); File f = new File(journalSettings.getSearchIndexDirectory(), SEGMENT_LIST_NAME); if (f.exists()) { log.info("Segment List File Exists, using it"); try { loadSegmentList(); } catch (IOException e) { log.error("Unable to load segment list", e); System.exit(-10); } } else { log.info("No Segment List File Exists"); // If no segments list exists, we have to assume that this a clean // node. // we must, clean the index if it exists and delete the db record // that records // where this node is. File searchDirectory = new File(journalSettings.getSearchIndexDirectory()); if (searchDirectory.exists()) { log.warn("Found Existing Search Directory with no local segment list, I will delete " + searchDirectory.getAbsolutePath()); try { FileUtils.deleteAll(searchDirectory); } catch (IOException e) { log.warn("Unable to remove Existing Search Directory ", e); } } deleteJournalSavePoint(); } // ensure that the index is closed to avoid stale locks Runtime.getRuntime().addShutdownHook(new Thread() { /* * * * * @see java.lang.Thread#run() */ @Override public void run() { try { if (multiReader != null) multiReader.close(); } catch (Exception ex) { log.debug(ex); } } }); } /** * */ private void deleteJournalSavePoint() { Connection connection = null; PreparedStatement deleteJournalSavePointPst = null; if (datasource != null) { try { connection = datasource.getConnection(); boolean tableExists = false; PreparedStatement checkJournalSavePoint = null; ResultSet rs = null; try { checkJournalSavePoint = connection.prepareStatement("select count(*) from search_node_status "); rs = checkJournalSavePoint.executeQuery(); if (rs.next()) { tableExists = true; } } catch (Exception ex) { if (log.isDebugEnabled()) { log.debug("Failed to check for existance of table, this is Ok on first start ", ex); } } finally { try { if (rs != null) { rs.close(); } } catch (Exception ex) { log.debug(ex); } try { checkJournalSavePoint.close(); } catch (Exception ex) { log.debug(ex); } } if (tableExists) { //SRCH-3 deleting this entirely can lead to search inconsistency deleteJournalSavePointPst = connection .prepareStatement("update search_node_status set jid = -1 where serverid = ? "); deleteJournalSavePointPst.clearParameters(); deleteJournalSavePointPst.setString(1, serverId); deleteJournalSavePointPst.executeUpdate(); connection.commit(); } } catch (Exception ex) { log.warn("Unable to delete Search Jorunal SavePoint ", ex); } finally { try { deleteJournalSavePointPst.close(); } catch (Exception ex) { log.debug(ex); } try { connection.close(); } catch (Exception ex) { log.debug(ex); } } } } /** * Since this is a singleton, we can cache the savePoint only updating on * change. The Save Point is the number of the journal entry that this node * has merged upto and including * * @see org.sakaiproject.search.maintanence.impl.JournaledObject#getJounalSavePoint() */ public long getJournalSavePoint() { if (journalSavePoint == -1) { Connection connection = null; PreparedStatement getJournalSavePointPst = null; ResultSet rs = null; try { connection = datasource.getConnection(); getJournalSavePointPst = connection .prepareStatement("select jid from search_node_status where serverid = ? "); getJournalSavePointPst.clearParameters(); getJournalSavePointPst.setString(1, serverId); rs = getJournalSavePointPst.executeQuery(); if (rs.next()) { journalSavePoint = rs.getLong(1); lastUpdate = System.currentTimeMillis(); } } catch (Exception ex) { log.warn("Unable to get Search Jorunal SavePoint ", ex); } finally { try { rs.close(); } catch (Exception ex) { log.debug(ex); } try { getJournalSavePointPst.close(); } catch (Exception ex) { log.debug(ex); } try { connection.close(); } catch (Exception ex) { log.debug(ex); } } } return journalSavePoint; } /** * @see org.sakaiproject.search.journal.api.JournaledObject#aquireUpdateLock() */ public boolean aquireUpdateLock() { try { boolean locked = rwlock.writeLock().tryLock(10, TimeUnit.SECONDS); if (locked) { setLastJournalEntry(-1); } return locked; } catch (InterruptedException iex) { log.debug(iex); } return false; } /** * @see org.sakaiproject.search.journal.api.JournaledObject#releaseUpdateLock() */ public void releaseUpdateLock() { if (rwlock.isWriteLockedByCurrentThread()) { rwlock.writeLock().unlock(); setLastJournalEntry(-1); } } /** * @see org.sakaiproject.search.journal.api.JournaledObject#auquireReadLock() */ public boolean aquireReadLock() { try { return rwlock.readLock().tryLock(10, TimeUnit.SECONDS); } catch (InterruptedException iex) { log.debug(iex); } return false; } /** * @see org.sakaiproject.search.journal.api.JournaledObject#releaseReadLock() */ public void releaseReadLock() { rwlock.readLock().unlock(); } /** * @param nextJournalEntry */ public void setLastJournalEntry(long lastJournalEntry) { lastJournalEntryHolder.set(lastJournalEntry); } /** * @return */ public long getLastJournalEntry() { Long l = lastJournalEntryHolder.get(); if (l == null) { return -1; } return l.longValue(); } /** * @see org.sakaiproject.search.journal.api.JournaledIndex#addSegment(java.io.File) */ public void addSegment(File f) { segments.add(f); modified = true; } /** * @see org.sakaiproject.search.journal.api.JournaledIndex#getWorkingSpace() */ public String getWorkingSpace() { return journalSettings.getLocalIndexWorkingSpace(); } /** * @see org.sakaiproject.search.index.impl.BaseIndexStorage#getIndexSearcher() */ public IndexSearcher getIndexSearcher() throws IOException { log.debug("getIndexSearcher(): " + indexSearcher); if (indexSearcher == null) { loadIndexSearcherInternal(); } if (indexSearcher != null) { log.debug("getIndexSearcher(): " + indexSearcher + " threadbinder"); ((ThreadBinder) indexSearcher).bind(threadLocalManager); } return indexSearcher; } /** * @throws IOException */ private void loadIndexSearcherInternal() throws IOException { log.debug("loadIndexSearcherInternal()"); IndexReader tmpIndexReader = multiReader; final IndexReader ir = getIndexReader(); log.debug("index reader is: " + ir); if (spellIndexDirectory == null) { this.createSpellIndex(ir); } if (tmpIndexReader != ir || indexSearcher == null) { long start = System.currentTimeMillis(); RefCountIndexSearcher newIndexSearcher = new RefCountIndexSearcher(ir, this); if (indexSearcher != null) { indexSearcher.close(); // this will be postponed } indexSearcher = newIndexSearcher; long end = System.currentTimeMillis(); lastLoad = end; lastLoadTime = end - start; log.debug("Opened index Searcher in " + (end - start) + " ms"); fireIndexSearcherOpen(indexSearcher); } } /* * * * * @see org.sakaiproject.search.journal.api.JournaledIndex#getDeletionIndexReader() */ public IndexReader getDeletionIndexReader() throws IOException { return getIndexReader(); } /* * * * * @see org.sakaiproject.search.index.IndexStorage#centralIndexExists() */ public boolean centralIndexExists() { return true; } /* * * * * @see org.sakaiproject.search.index.IndexStorage#closeIndexReader(org.apache.lucene.index.IndexReader) */ public void closeIndexReader(IndexReader indexReader) throws IOException { indexReader.close(); } /* * * * * @see org.sakaiproject.search.index.IndexStorage#closeIndexSearcher(org.apache.lucene.search.IndexSearcher) */ public void closeIndexSearcher(IndexSearcher indexSearcher) { IndexReader indexReader = indexSearcher.getIndexReader(); boolean closedAlready = false; try { if (indexReader != null) { indexReader.close(); closedAlready = true; } } catch (Exception ex) { log.error("Failed to close Index Reader " + ex.getMessage()); } try { indexSearcher.close(); } catch (Exception ex) { if (closedAlready) { log.debug("Failed to close Index Searcher " + ex.getMessage()); } else { log.error("Failed to close Index Searcher " + ex.getMessage()); } } } /* * * * * @see org.sakaiproject.search.index.IndexStorage#closeIndexWriter(org.apache.lucene.index.IndexWriter) */ public void closeIndexWriter(IndexWriter indexWrite) throws IOException { throw new UnsupportedOperationException("Only Readers are available from a JournaledFSIndexStorage class"); } /* * * * * @see org.sakaiproject.search.index.IndexStorage#doPostIndexUpdate() */ public void doPostIndexUpdate() throws IOException { throw new UnsupportedOperationException("Only Readers are available from a JournaledFSIndexStorage class"); } /* * * * * @see org.sakaiproject.search.index.IndexStorage#doPreIndexUpdate() */ public void doPreIndexUpdate() throws IOException { throw new UnsupportedOperationException("Only Readers are available from a JournaledFSIndexStorage class"); } public IndexReader getIndexReader() throws IOException { log.debug("getIndexReader()"); boolean current = false; long start = System.currentTimeMillis(); try { // could be null if (multiReader != null) { current = multiReader.isCurrent(); } } catch (Exception ex) { log.warn("Failed to get current status assuming index reload is required ", ex); } long start2 = System.currentTimeMillis(); long x1 = start2; long f1 = start2; long r1 = start2; long r2 = start2; long f2 = start2; long f3 = start2; log.debug("Check 1: modified=" + modified + " multiReader=" + multiReader + " current=" + current); if (modified || multiReader == null || !current) { /* * We must get a read lock to prevent a writer from opening when we * are trying to open for read. Writers will have already taken * write locks and so get the read lock by default. */ x1 = System.currentTimeMillis(); f1 = x1; boolean locked = false; if (rwlock.isWriteLockedByCurrentThread()) { locked = true; } else { locked = aquireReadLock(); } if (locked) { // should check again current = false; try { // could be null if (multiReader != null) { current = multiReader.isCurrent(); } } catch (Exception ex) { log.warn("Failed to get current status assuming index reload is required ", ex); } log.debug("Check 2: modified=" + modified + " multiReader=" + multiReader + " current=" + current); if (modified || multiReader == null || !current) { f1 = System.currentTimeMillis(); try { r1 = System.currentTimeMillis(); getIndexReaderInternal(); modified = false; r2 = System.currentTimeMillis(); } finally { f2 = System.currentTimeMillis(); if (!rwlock.isWriteLockedByCurrentThread()) { releaseReadLock(); } f3 = System.currentTimeMillis(); } } } else { f1 = System.currentTimeMillis(); log.warn("Failed to get read lock on index "); } } long f = System.currentTimeMillis(); if ((f - start) > 1000) { log.info("Long time opening " + (start2 - start) + ":" + (f - start2) + " ms"); log.info("Read Lock aquire " + (f1 - x1) + " ms"); log.info("Index Load aquire " + (r2 - r1) + " ms"); log.info("Read Lock Release " + (f2 - f3) + " ms"); } if (multiReader != null) { ((ThreadBinder) multiReader).bind(threadLocalManager); } log.debug("getIndexReader() returning " + multiReader); return multiReader; } /* * * * * @see org.sakaiproject.search.index.IndexStorage#getIndexReader() */ private IndexReader getIndexReaderInternal() throws IOException { long start = System.currentTimeMillis(); File f = new File(journalSettings.getSearchIndexDirectory()); Directory d = null; if (!f.exists()) { if (!f.mkdirs()) { throw new IOException("can't create index directory " + f.getPath()); } log.debug("Indexing in " + f.getAbsolutePath()); d = FSDirectory.open(new File(journalSettings.getSearchIndexDirectory())); } else { d = FSDirectory.open(new File(journalSettings.getSearchIndexDirectory())); } if (IndexWriter.isLocked(d)) { // this could be dangerous, I am assuming that // the locking mechanism implemented here is // robust and // already prevents multiple modifiers. // A more IndexWriter.unlock(d); log.warn("Unlocked Lucene Directory for update, hope this is Ok"); } if (!d.fileExists("segments.gen")) { IndexWriter iw = new IndexWriter(FSDirectory.open(f), getAnalyzer(), MaxFieldLength.UNLIMITED); iw.setUseCompoundFile(true); iw.setMaxMergeDocs(journalSettings.getLocalMaxMergeDocs()); iw.setMaxBufferedDocs(journalSettings.getLocalMaxBufferedDocs()); iw.setMergeFactor(journalSettings.getLocalMaxMergeFactor()); Document doc = new Document(); doc.add(new Field("indexcreated", (new Date()).toString(), Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.NO)); iw.addDocument(doc); iw.close(); } final IndexReader[] indexReaders = new IndexReader[segments.size() + 1]; indexReaders[0] = IndexReader.open(d, false); int i = 1; for (File s : segments) { FSDirectory fsd = FSDirectory.open(s); if (IndexWriter.isLocked(fsd)) { log.warn("++++++++++++++++++Unlocking Index " + fsd.toString()); IndexWriter.unlock(fsd); } indexReaders[i] = IndexReader.open(FSDirectory.open(s), false); i++; } RefCountMultiReader newMultiReader = new RefCountMultiReader(indexReaders, this); if (multiReader != null) { multiReader.close(); // this will postpone due to the override // above } multiReader = newMultiReader; lastUpdate = System.currentTimeMillis(); log.debug("Reopened Index Reader in " + (lastUpdate - start) + " ms"); // notify anything that wants to listen to the index open and close // events fireIndexReaderOpen(newMultiReader); return multiReader; } /* * * * * @see org.sakaiproject.search.index.IndexStorage#getIndexWriter(boolean) */ public IndexWriter getIndexWriter(boolean create) throws IOException { throw new UnsupportedOperationException("A JournaledFSIndexStorage only supports readers"); } /* * * * * @see org.sakaiproject.search.index.IndexStorage#getLastUpdate() */ public long getLastUpdate() { return lastUpdate; } /* * * * * @see org.sakaiproject.search.index.IndexStorage#getSegmentInfoList() */ public List<Object[]> getSegmentInfoList() { List<Object[]> seginfo = new ArrayList<Object[]>(); try { SizeAction sa = new SizeAction(); File searchDir = new File(journalSettings.getSearchIndexDirectory()); FileUtils.recurse(searchDir, sa); seginfo.add(new Object[] { "mainsegment", sa.sizeToString(sa.getSize()), sa.dateToString(sa.getLastUpdated()) }); sa.reset(); for (File s : segments) { FileUtils.recurse(s, sa); seginfo.add(new Object[] { s.getName(), sa.sizeToString(sa.getSize()), sa.dateToString(sa.getLastUpdated()) }); sa.reset(); } seginfo.add(new Object[] { "Total", sa.sizeToString(sa.getTotalSize()), "" }); } catch (IOException ex) { if (log.isDebugEnabled()) { ex.printStackTrace(); } seginfo.add(new Object[] { "Failed to get Segment Info list " + ex.getClass().getName() + " " + ex.getMessage() }); } return seginfo; } /* * * * * @see org.sakaiproject.search.index.IndexStorage#indexExists() */ public boolean indexExists() { File f = new File(journalSettings.getSearchIndexDirectory()); return f.exists(); } /* * * * * @see org.sakaiproject.search.index.IndexStorage#isMultipleIndexers() */ public boolean isMultipleIndexers() { return true; } /** * @see org.sakaiproject.search.index.IndexStorage#setRecoverCorruptedIndex(boolean) */ public void setRecoverCorruptedIndex(boolean recover) { log.debug("Recover Indexes not implemented, yet"); } private static class SizeAction implements FileUtils.RecurseAction { private long size = 0; private long totalSize = 0; private long lastUpdate = 0; /** * @see org.sakaiproject.search.util.FileUtils.RecurseAction#doAfterFile(java.io.File) */ public void doAfterFile(File f) { } /** * @param lastUpdated * @return */ public String dateToString(long lastUpdated) { return new Date(lastUpdated).toString(); } /** * @return */ public long getLastUpdated() { return lastUpdate; } /** * @see org.sakaiproject.search.util.FileUtils.RecurseAction#doBeforeFile(java.io.File) */ public void doBeforeFile(File f) { } /** * @see org.sakaiproject.search.util.FileUtils.RecurseAction#doFile(java.io.File) */ public void doFile(File file) throws IOException { size += file.length(); lastUpdate = Math.max(lastUpdate, file.lastModified()); totalSize += file.length(); } /** * @return the size */ public long getSize() { return size; } /** * @return the size */ public long getTotalSize() { return totalSize; } /** * @param size * the size to set */ public void reset() { lastUpdate = 0; size = 0; } public String sizeToString(long tsize) { if (tsize > 1024 * 1024 * 10) { return String.valueOf(tsize / (1024 * 1024)) + "MB"; } else if (tsize >= 1024 * 1024) { return String.valueOf(tsize / (1024 * 1024)) + "." + String.valueOf(tsize / (102 * 1024) + "MB"); } else { return String.valueOf(tsize / (1024)) + "KB"; } } } /** * @return the datasource */ public DataSource getDatasource() { return datasource; } /** * @param datasource * the datasource to set */ public void setDatasource(DataSource datasource) { this.datasource = datasource; } /** * @return the journalMonitor */ public JournalManager getJournalManager() { return journalManager; } /** * @param journalMonitor * the journalMonitor to set */ public void setJournalManager(JournalManager journalManager) { this.journalManager = journalManager; } /** * @return the serverConfigurationService */ public ServerConfigurationService getServerConfigurationService() { return serverConfigurationService; } /** * @param serverConfigurationService * the serverConfigurationService to set */ public void setServerConfigurationService(ServerConfigurationService serverConfigurationService) { this.serverConfigurationService = serverConfigurationService; } /* * * * * @see org.sakaiproject.search.journal.api.JournaledIndex#setJournalIndexEntry(long) */ public void setJournalIndexEntry(long journalEntry) { if (journalSavePoint != journalEntry) { Connection connection = null; PreparedStatement updateJournalSavePointPst = null; PreparedStatement insertJournalSavePointPst = null; try { connection = datasource.getConnection(); updateJournalSavePointPst = connection .prepareStatement("update search_node_status set jid = ?, jidts = ? where serverid = ? "); updateJournalSavePointPst.clearParameters(); updateJournalSavePointPst.setLong(1, journalEntry); updateJournalSavePointPst.setLong(2, System.currentTimeMillis()); updateJournalSavePointPst.setString(3, serverId); if (updateJournalSavePointPst.executeUpdate() != 1) { insertJournalSavePointPst = connection.prepareStatement( "insert into search_node_status (jid,jidts,serverid) values (?,?,?) "); insertJournalSavePointPst.clearParameters(); insertJournalSavePointPst.setLong(1, journalEntry); insertJournalSavePointPst.setLong(2, System.currentTimeMillis()); insertJournalSavePointPst.setString(3, serverId); if (insertJournalSavePointPst.executeUpdate() != 1) { throw new SQLException("Unable to update journal entry for some reason "); } } connection.commit(); journalSavePoint = journalEntry; } catch (Exception ex) { try { if (connection != null) { connection.rollback(); } } catch (Exception ex2) { log.debug(ex); } log.warn("Unable to get Search Jorunal SavePoint ", ex); } finally { try { if (updateJournalSavePointPst != null) updateJournalSavePointPst.close(); } catch (Exception ex) { log.debug(ex); } try { if (insertJournalSavePointPst != null) insertJournalSavePointPst.close(); } catch (Exception ex) { log.debug(ex); } try { if (connection != null) connection.close(); } catch (Exception ex) { log.debug(ex); } } } } /** * @param oldMultiReader * @throws IOException */ protected void fireIndexReaderClose(IndexReader oldMultiReader) throws IOException { if (oldMultiReader == multiReader) { multiReader = null; } for (Iterator<IndexListener> itl = getIndexListeners().iterator(); itl.hasNext();) { IndexListener tl = itl.next(); tl.doIndexReaderClose(oldMultiReader); } } /** * @param newMultiReader */ protected void fireIndexReaderOpen(IndexReader newMultiReader) { for (Iterator<IndexListener> itl = getIndexListeners().iterator(); itl.hasNext();) { IndexListener tl = itl.next(); tl.doIndexReaderOpen(newMultiReader); } } /** * @param indexSearcher2 * @throws IOException */ void fireIndexSearcherClose(IndexSearcher indexSearcher) throws IOException { for (Iterator<IndexListener> itl = getIndexListeners().iterator(); itl.hasNext();) { IndexListener tl = itl.next(); tl.doIndexSearcherClose(indexSearcher); } } /** * @param indexSearcher2 * @throws IOException */ private void fireIndexSearcherOpen(IndexSearcher indexSearcher) throws IOException { for (Iterator<IndexListener> itl = getIndexListeners().iterator(); itl.hasNext();) { IndexListener tl = itl.next(); tl.doIndexSearcherOpen(indexSearcher); } } /** * @return */ private List<IndexListener> getIndexListeners() { return indexListeners; } public void addIndexListener(IndexListener indexListener) { List<IndexListener> tl = new ArrayList<IndexListener>(); tl.addAll(this.indexListeners); tl.add(indexListener); this.indexListeners = tl; } public void setIndexListener(List<IndexListener> indexListeners) { List<IndexListener> tl = new ArrayList<IndexListener>(); tl.addAll(indexListeners); this.indexListeners = tl; } /** * Get an index writer to the permanent index, a write lock should have been * taken before doing this * * @throws IndexTransactionException * @see org.sakaiproject.search.journal.api.JournaledIndex#getPermanentIndexWriter() */ public IndexWriter getPermanentIndexWriter() throws IndexTransactionException { if (!rwlock.isWriteLockedByCurrentThread()) { throw new JournalErrorException( "Thread Does not hold a write lock, permanent index cannot be written to "); } if (permanentIndexWriter == null) { try { File f = new File(journalSettings.getSearchIndexDirectory()); Directory d = null; if (!f.exists()) { if (!f.mkdirs()) { throw new IOException("can't create index folder " + f.getPath()); } log.debug("Indexing in " + f.getAbsolutePath()); d = FSDirectory.open(new File(journalSettings.getSearchIndexDirectory())); } else { d = FSDirectory.open(new File(journalSettings.getSearchIndexDirectory())); } if (!d.fileExists("segments.gen")) { permanentIndexWriter = new IndexWriter(FSDirectory.open(f), getAnalyzer(), true, MaxFieldLength.UNLIMITED); permanentIndexWriter.setUseCompoundFile(true); permanentIndexWriter.setMaxMergeDocs(journalSettings.getLocalMaxMergeDocs()); permanentIndexWriter.setMaxBufferedDocs(journalSettings.getLocalMaxBufferedDocs()); permanentIndexWriter.setMergeFactor(journalSettings.getLocalMaxMergeFactor()); Document doc = new Document(); doc.add(new Field("indexcreated", (new Date()).toString(), Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.NO)); permanentIndexWriter.addDocument(doc); permanentIndexWriter.close(); } permanentIndexWriter = new MonitoredIndexWriter(f, getAnalyzer(), false); ((MonitoredIndexWriter) permanentIndexWriter).addMonitorIndexListener(new IndexMonitorListener() { public void doIndexMonitorClose(IndexWriter writer) { permanentIndexWriter = null; } }); permanentIndexWriter.setUseCompoundFile(true); permanentIndexWriter.setMaxMergeDocs(100000); permanentIndexWriter.setMaxBufferedDocs(50); permanentIndexWriter.setMergeFactor(50); } catch (IOException ioex) { throw new JournalErrorException("Failed to open permanent Index ", ioex); } } return permanentIndexWriter; } /** * @see org.sakaiproject.search.journal.api.JournaledIndex#getSegments() */ public File[] getSegments() { return segments.toArray(new File[0]); } /** * @see org.sakaiproject.search.journal.api.JournaledIndex#setSegments(java.util.List) */ public void setSegments(List<File> keep) { segments = keep; } public Analyzer getAnalyzer() { return analyzerFactory.newAnalyzer(); } /** * @return Returns the analzyserFactory. */ public AnalyzerFactory getAnalyzerFactory() { return analyzerFactory; } /** * @param analzyserFactory * The analzyserFactory to set. */ public void setAnalyzerFactory(AnalyzerFactory analzyserFactory) { this.analyzerFactory = analzyserFactory; } /* * (non-Javadoc) * * @see org.sakaiproject.search.journal.api.JournaledIndex#saveSegmentList() */ public void saveSegmentList() throws IOException { File f = new File(journalSettings.getSearchIndexDirectory(), SEGMENT_LIST_NAME); SegmentListWriter sw = new SegmentListWriter(f); sw.write(segments); } public void loadSegmentList() throws IOException { File f = new File(journalSettings.getSearchIndexDirectory(), SEGMENT_LIST_NAME); SegmentListReader sr = new SegmentListReader(f); segments = sr.read(); } /** * @return the clusterService */ public ClusterService getClusterService() { return clusterService; } /** * @param clusterService * the clusterService to set */ public void setClusterService(ClusterService clusterService) { this.clusterService = clusterService; } /* * (non-Javadoc) * * @see org.sakaiproject.search.journal.api.JournaledIndex#loadIndexReader() */ public void loadIndexReader() throws IOException { loadIndexSearcherInternal(); } /* * (non-Javadoc) * * @see org.sakaiproject.search.journal.api.JournaledObject#debugLock() */ public void debugLock() { log.info(this + " Locks Waiting " + rwlock.getQueueLength()); log.info(this + " Read Locks Locks " + rwlock.getReadLockCount()); log.info(this + " Write Holds this thread " + rwlock.getWriteHoldCount()); log.info(this + " is Write Locked " + rwlock.isWriteLocked()); } /* * (non-Javadoc) * * @see org.sakaiproject.search.journal.api.IndexStorageProvider#getLastLoad() */ public long getLastLoad() { return lastLoad; } /* * (non-Javadoc) * * @see org.sakaiproject.search.journal.api.IndexStorageProvider#getLastLoadTime() */ public long getLastLoadTime() { return lastLoadTime; } /** * @return the journalSettings */ public JournalSettings getJournalSettings() { return journalSettings; } /** * @param journalSettings * the journalSettings to set */ public void setJournalSettings(JournalSettings journalSettings) { this.journalSettings = journalSettings; } public void markModified() throws IOException { modified = true; if (multiReader != null) { multiReader.close(); multiReader = null; } } /** * @return the threadLocalManager */ public ThreadLocalManager getThreadLocalManager() { return threadLocalManager; } /** * @param threadLocalManager * the threadLocalManager to set */ public void setThreadLocalManager(ThreadLocalManager threadLocalManager) { this.threadLocalManager = threadLocalManager; } Directory spellIndexDirectory = null; private void createSpellIndex(IndexReader indexReader) { if (!serverConfigurationService.getBoolean("search.experimental.didyoumean", false)) { return; } log.info("create Spell Index"); Long start = System.currentTimeMillis(); try { log.info("main index is in: " + journalSettings.getSearchIndexDirectory()); log.info("local base is: " + journalSettings.getLocalIndexBase()); spellIndexDirectory = new NIOFSDirectory(new File(journalSettings.getLocalIndexBase() + "/spellindex")); if (indexReader == null) { log.info("unable to get index reader aborting spellindex creation"); return; } Dictionary dictionary = new LuceneDictionary(indexReader, SearchService.FIELD_CONTENTS); SpellChecker spellChecker = new SpellChecker(spellIndexDirectory); spellChecker.clearIndex(); spellChecker.indexDictionary(dictionary); log.info("New Spell dictionary constructed in " + (System.currentTimeMillis() - start)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } log.info("All done in " + (System.currentTimeMillis() - start)); } public Directory getSpellDirectory() { return spellIndexDirectory; } }