Java tutorial
/********************************************************************************** * $URL: https://source.sakaiproject.org/svn/search/trunk/search-impl/impl/src/java/org/sakaiproject/search/component/dao/impl/SearchIndexBuilderWorkerDaoJdbcImpl.java $ * $Id: SearchIndexBuilderWorkerDaoJdbcImpl.java 111640 2012-08-20 12:58:11Z david.horwitz@uct.ac.za $ *********************************************************************************** * * Copyright (c) 2003, 2004, 2005, 2006, 2007, 2008 The Sakai Foundation * * Licensed under the Educational Community License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.opensource.org/licenses/ECL-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * **********************************************************************************/ package org.sakaiproject.search.component.dao.impl; import java.io.File; import java.io.IOException; import java.io.Reader; import java.sql.Connection; import java.sql.Date; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.UUID; import javax.sql.DataSource; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.lucene.document.CompressionTools; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import org.hibernate.HibernateException; import org.sakaiproject.component.api.ServerConfigurationService; import org.sakaiproject.search.api.EntityContentProducer; import org.sakaiproject.search.api.SearchIndexBuilder; import org.sakaiproject.search.api.SearchIndexBuilderWorker; import org.sakaiproject.search.api.SearchService; import org.sakaiproject.search.api.rdf.RDFIndexException; import org.sakaiproject.search.api.rdf.RDFSearchService; import org.sakaiproject.search.component.Messages; import org.sakaiproject.search.dao.SearchIndexBuilderWorkerDao; import org.sakaiproject.search.index.IndexStorage; import org.sakaiproject.search.model.SearchBuilderItem; import org.sakaiproject.search.model.impl.SearchBuilderItemImpl; import org.sakaiproject.search.util.DigestStorageUtil; import org.sakaiproject.search.util.DocumentIndexingUtils; import org.sakaiproject.site.api.Site; import org.sakaiproject.site.api.SiteService.SelectionType; import org.sakaiproject.site.api.SiteService.SortType; import org.sakaiproject.site.api.ToolConfiguration; import org.sakaiproject.site.cover.SiteService; public class SearchIndexBuilderWorkerDaoJdbcImpl implements SearchIndexBuilderWorkerDao { private static final String SEARCH_BUILDER_ITEM_FIELDS = " name, context, searchaction, searchstate, version, itemscope, id "; //$NON-NLS-1$ private static final String SEARCH_BUILDER_ITEM_T = "searchbuilderitem"; //$NON-NLS-1$ private static final String SEARCH_BUILDER_ITEM_FIELDS_PARAMS = " ?, ?, ?, ?, ?, ?, ? "; //$NON-NLS-1$ private static final String SEARCH_BUILDER_ITEM_FIELDS_UPDATE = " name = ?, context = ?, searchaction = ?, searchstate = ?, version = ?, itemscope = ? where id = ? "; //$NON-NLS-1$ private static Log log = LogFactory.getLog(SearchIndexBuilderWorkerDaoJdbcImpl.class); /** * sync object */ // private Object threadStartLock = new Object(); /** * dependency: the search index builder that is accepting new items */ private SearchIndexBuilder searchIndexBuilder = null; private boolean enabled = false; // private EntityManager entityManager; private RDFSearchService rdfSearchService = null; /** * injected to abstract the storage impl */ private IndexStorage indexStorage = null; private DataSource dataSource = null; private ServerConfigurationService serverConfigurationService; private SearchService searchService = null; public void setSearchService(SearchService searchService) { this.searchService = searchService; } public void init() { enabled = "true".equals(serverConfigurationService.getString("search.enable", "false")); try { if (rdfSearchService == null) { log.info("No RDFSearchService has been defined, RDF Indexing not enabled"); //$NON-NLS-1$ } else { log.warn("Experimental RDF Search Service is enabled using implementation " //$NON-NLS-1$ + rdfSearchService); } if (enabled) { Connection con = null; Statement stmt = null; ResultSet rs = null; try { con = dataSource.getConnection(); stmt = con.createStatement(); int nglobal = 0; try { rs = stmt.executeQuery("select count(*) from " + SEARCH_BUILDER_ITEM_T + " where itemscope = " + SearchBuilderItem.ITEM_GLOBAL_MASTER); if (rs.next()) { nglobal = rs.getInt(1); } } catch (Exception ex) { log.error( "Schema Has not been updated to include itemscope column, please run SAK-9865.sql script on this database "); System.exit(-1); } if (nglobal == 0) { // no global items found, almost certainly because data // has not been indexed with itemscope stmt.execute("update searchbuilderitem set itemscope= " + SearchBuilderItem.ITEM); stmt.execute("update searchbuilderitem set itemscope= " + SearchBuilderItem.ITEM_SITE_MASTER + " where name like '" + SearchBuilderItem.SITE_MASTER_PATTERN + "'"); stmt.execute( "update searchbuilderitem set itemscope= " + SearchBuilderItem.ITEM_GLOBAL_MASTER + " where context = '" + SearchBuilderItem.GLOBAL_CONTEXT + "'"); log.info("Exiting search item builder records have been optimized for itemscope access "); } con.commit(); } catch (Exception ex) { try { if (con != null) { con.rollback(); } } catch (Exception e) { log.debug(e); } log.error( "Failed to migrate indexes to itemscope based storage, search cant startup, please investigate immediately "); System.exit(-1); } finally { try { rs.close(); } catch (Exception e) { log.debug(e); } try { stmt.close(); } catch (Exception e) { log.debug(e); } try { if (con != null) { con.close(); } } catch (Exception e) { log.debug(e); } } } } catch (Throwable t) { log.error("Failed to init ", t); //$NON-NLS-1$ } } private void processDeletes(SearchIndexBuilderWorker worker, Connection connection, List<SearchBuilderItem> runtimeToDo) throws SQLException, IOException { if (indexStorage.indexExists()) { IndexReader indexReader = null; try { indexReader = indexStorage.getIndexReader(); // Open the index for (Iterator<SearchBuilderItem> tditer = runtimeToDo.iterator(); worker.isRunning() && tditer.hasNext();) { SearchBuilderItem sbi = (SearchBuilderItem) tditer.next(); if (!SearchBuilderItem.STATE_LOCKED.equals(sbi.getSearchstate())) { // should only be getting pending // items log.warn(" Found Item that was not pending " //$NON-NLS-1$ + sbi.getName()); continue; } if (SearchBuilderItem.ACTION_UNKNOWN.equals(sbi.getSearchaction())) { sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED); updateOrSave(connection, sbi); continue; } // remove document // if this is mult segment it might not work. try { indexReader.deleteDocuments(new Term(SearchService.FIELD_REFERENCE, sbi.getName())); if (SearchBuilderItem.ACTION_DELETE.equals(sbi.getSearchaction())) { sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED); updateOrSave(connection, sbi); } else { sbi.setSearchstate(SearchBuilderItem.STATE_PENDING_2); } } catch (IOException ex) { log.warn("Failed to delete Page ", ex); //$NON-NLS-1$ } } } finally { if (indexReader != null) { indexStorage.closeIndexReader(indexReader); indexReader = null; } } } } private void processAdd(SearchIndexBuilderWorker worker, Connection connection, List<SearchBuilderItem> runtimeToDo) throws Exception { IndexWriter indexWrite = null; try { if (worker.isRunning()) { indexWrite = indexStorage.getIndexWriter(false); } long last = System.currentTimeMillis(); for (Iterator<SearchBuilderItem> tditer = runtimeToDo.iterator(); worker.isRunning() && tditer.hasNext();) { Reader contentReader = null; try { SearchBuilderItem sbi = (SearchBuilderItem) tditer.next(); // only add adds, that have been deleted or are locked // sucessfully if (!SearchBuilderItem.STATE_PENDING_2.equals(sbi.getSearchstate()) && !SearchBuilderItem.STATE_LOCKED.equals(sbi.getSearchstate())) { continue; } // Reference ref = // entityManager.newReference(sbi.getName()); String ref = sbi.getName(); if (ref == null) { log.error("Unrecognised trigger object presented to index builder " //$NON-NLS-1$ + sbi); } long startDocIndex = System.currentTimeMillis(); worker.setStartDocIndex(startDocIndex); worker.setNowIndexing(ref); try { try { // Entity entity = ref.getEntity(); EntityContentProducer sep = searchIndexBuilder.newEntityContentProducer(ref); boolean indexDoc = true; if (searchIndexBuilder.isOnlyIndexSearchToolSites()) { try { String siteId = sep.getSiteId(sbi.getName()); Site s = SiteService.getSite(siteId); ToolConfiguration t = s.getToolForCommonId("sakai.search"); //$NON-NLS-1$ if (t == null) { indexDoc = false; log.debug("Not indexing " //$NON-NLS-1$ + sbi.getName() + " as it has no search tool"); //$NON-NLS-1$ } } catch (Exception ex) { indexDoc = false; log.debug("Not indexing " + sbi.getName() //$NON-NLS-1$ + " as it has no site", ex); //$NON-NLS-1$ } } if (indexDoc && sep != null && sep.isForIndex(ref) && sep.getSiteId(ref) != null) { DigestStorageUtil digestStorageUtil = new DigestStorageUtil(searchService); //Reader contentReader = null; Document doc = DocumentIndexingUtils.createIndexDocument(ref, digestStorageUtil, sep, serverConfigurationService.getServerUrl(), contentReader); //indexDocTMP(ref, sep); log.debug("Indexing Document " + doc); //$NON-NLS-1$ indexWrite.addDocument(doc); log.debug("Done Indexing Document " + doc); //$NON-NLS-1$ processRDF(ref, sep); } else { if (log.isDebugEnabled()) { if (!indexDoc) { log.debug("Ignored Document: Fileteed out by site " + ref); //$NON-NLS-1$ } else if (sep == null) { log.debug("Ignored Document: No EntityContentProducer " + ref); //$NON-NLS-1$ } else if (!sep.isForIndex(ref)) { log.debug("Ignored Document: Marked as Ignore " + ref); //$NON-NLS-1$ } else if (sep.getSiteId(ref) == null) { log.debug("Ignored Document: No Site ID " + ref); //$NON-NLS-1$ } else { log.debug("Ignored Document: Reason Unknown " + ref); //$NON-NLS-1$ } } } } catch (Exception e1) { log.debug(" Failed to index document for " + ref + " cause: " //$NON-NLS-1$ + e1.getMessage(), e1); } sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED); updateOrSave(connection, sbi); } catch (Exception e1) { log.debug(" Failed to index document cause: " //$NON-NLS-1$ + e1.getMessage()); } long endDocIndex = System.currentTimeMillis(); worker.setLastIndex(endDocIndex - startDocIndex); if ((endDocIndex - startDocIndex) > 60000L) { log.warn("Slow index operation " //$NON-NLS-1$ + String.valueOf((endDocIndex - startDocIndex) / 1000) + " seconds to index " //$NON-NLS-1$ + ref); } // update this node lock to indicate its // still alove, no document should // take more than 2 mins to process // ony do this check once every minute long now = System.currentTimeMillis(); if ((now - last) > (60L * 1000L)) { last = System.currentTimeMillis(); if (!worker.getLockTransaction(15L * 60L * 1000L, true)) { throw new Exception("Transaction Lock Expired while indexing " //$NON-NLS-1$ + ref); } } } finally { if (contentReader != null) { try { contentReader.close(); } catch (IOException ioex) { log.debug(ioex); } } } } worker.setStartDocIndex(System.currentTimeMillis()); worker.setNowIndexing(Messages.getString("SearchIndexBuilderWorkerDaoJdbcImpl.33")); //$NON-NLS-1$ } catch (Exception ex) { log.error("Failed to Add Documents ", ex); throw new Exception(ex); } finally { if (indexWrite != null) { if (log.isDebugEnabled()) { log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents"); Directory d = indexWrite.getDirectory(); String[] s = d.listAll(); log.debug("Directory Contains "); for (int i = 0; i < s.length; i++) { File f = new File(s[i]); log.debug("\t" + String.valueOf(f.length()) + "\t" + new Date(f.lastModified()) + "\t" + s[i]); } } indexStorage.closeIndexWriter(indexWrite); } } } private int completeUpdate(SearchIndexBuilderWorker worker, Connection connection, List runtimeToDo) throws Exception { try { for (Iterator tditer = runtimeToDo.iterator(); worker.isRunning() && tditer.hasNext();) { SearchBuilderItem sbi = (SearchBuilderItem) tditer.next(); if (SearchBuilderItem.STATE_COMPLETED.equals(sbi.getSearchstate())) { if (SearchBuilderItem.ACTION_DELETE.equals(sbi.getSearchaction())) { delete(connection, sbi); connection.commit(); } else { updateOrSave(connection, sbi); connection.commit(); } } } return runtimeToDo.size(); } catch (Exception ex) { log.warn("Failed to update state in database due to " //$NON-NLS-1$ + ex.getMessage() + " this will be corrected on the next run of the IndexBuilder, no cause for alarm"); //$NON-NLS-1$ } return 0; } /* * (non-Javadoc) * * @see org.sakaiproject.search.component.dao.impl.SearchIndexBuilderWorkerDao#processToDoListTransaction() */ public void processToDoListTransaction(SearchIndexBuilderWorker worker, int indexBatchSize) { Connection connection = null; try { connection = dataSource.getConnection(); long startTime = System.currentTimeMillis(); int totalDocs = 0; // Load the list List runtimeToDo = findPending(indexBatchSize, connection, worker); totalDocs = runtimeToDo.size(); log.debug("Processing " + totalDocs + " documents"); //$NON-NLS-1$ //$NON-NLS-2$ if (totalDocs > 0) { log.debug("Preupdate Start"); indexStorage.doPreIndexUpdate(); log.debug("Preupdate End"); // get lock // this needs to be exclusive log.debug("Process Deletes Start"); processDeletes(worker, connection, runtimeToDo); log.debug("Process Deletes End"); // upate and release lock // after a process Deletes the index needs to updated // can be parallel log.debug("Process Add Start"); processAdd(worker, connection, runtimeToDo); log.debug("Process Add End"); log.debug("Complete Update Start"); completeUpdate(worker, connection, runtimeToDo); log.debug("Complete Update End"); // get lock try { log.debug("Post update Start"); indexStorage.doPostIndexUpdate(); log.debug("Post update End"); } catch (IOException e) { log.error("Failed to do Post Index Update", e); //$NON-NLS-1$ } // release lock } if (worker.isRunning()) { long endTime = System.currentTimeMillis(); float totalTime = endTime - startTime; float ndocs = totalDocs; if (totalDocs > 0) { float docspersec = 1000 * ndocs / totalTime; log.info("Completed Process List of " + totalDocs + " at " //$NON-NLS-1$ //$NON-NLS-2$ + docspersec + " documents/per second "); //$NON-NLS-1$ } } } catch (Exception ex) { log.warn("Failed to perform index cycle " + ex.getMessage(), ex); //$NON-NLS-1$ //rollback any uncommitted transactions try { connection.rollback(); } catch (SQLException e) { log.debug(e); } } finally { try { connection.close(); } catch (Exception ex) { log.debug(ex); } } } /* * (non-Javadoc) * * @see org.sakaiproject.search.dao.SearchIndexBuilderWorkerDao#createIndexTransaction(org.sakaiproject.search.api.SearchIndexBuilderWorker) */ public void createIndexTransaction(SearchIndexBuilderWorker worker) { Connection connection = null; try { connection = dataSource.getConnection(); long startTime = System.currentTimeMillis(); int totalDocs = 0; log.debug("Preupdate Start"); indexStorage.doPreIndexUpdate(); log.debug("Preupdate End"); createIndex(worker, connection); // get lock try { log.debug("Post update Start"); indexStorage.doPostIndexUpdate(); log.debug("Post update End"); } catch (IOException e) { log.error("Failed to do Post Index Update", e); //$NON-NLS-1$ } log.info("Created Index"); //$NON-NLS-1$ } catch (Exception ex) { log.warn("Failed to create Index " + ex.getMessage()); //$NON-NLS-1$ log.debug("Traceback is ", ex); //$NON-NLS-1$ } finally { try { connection.close(); } catch (Exception ex) { log.debug(ex); } } } /** * @param worker * @param connection * @throws Exception */ private void createIndex(SearchIndexBuilderWorker worker, Connection connection) throws Exception { IndexWriter indexWrite = null; try { if (worker.isRunning()) { indexWrite = indexStorage.getIndexWriter(false); } if (indexWrite != null) { Document doc = new Document(); //The date of indexing String timeStamp = String.valueOf(System.currentTimeMillis()); doc.add(new Field(SearchService.DATE_STAMP, timeStamp, Field.Store.NO, Field.Index.NOT_ANALYZED)); doc.add(new Field(SearchService.DATE_STAMP, CompressionTools.compressString(timeStamp), Field.Store.YES)); String ref = "---INDEX-CREATED---"; doc.add(new Field(SearchService.FIELD_REFERENCE, CompressionTools.compressString(ref), Field.Store.YES)); doc.add(new Field(SearchService.FIELD_REFERENCE, ref, Field.Store.NO, Field.Index.NOT_ANALYZED)); indexWrite.addDocument(doc); } else { log.error("Couldn't get indexWriter to add document!"); } } catch (Exception ex) { log.error("Failed to Add Documents ", ex); throw new Exception(ex); } finally { if (indexWrite != null) { if (log.isDebugEnabled()) { log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents"); Directory d = indexWrite.getDirectory(); String[] s = d.listAll(); log.debug("Directory Contains "); for (int i = 0; i < s.length; i++) { File f = new File(s[i]); log.debug("\t" + String.valueOf(f.length()) + "\t" + new Date(f.lastModified()) + "\t" + s[i]); } } indexStorage.closeIndexWriter(indexWrite); } } } private void processRDF(String ref, EntityContentProducer sep) throws RDFIndexException { if (rdfSearchService != null) { String s = sep.getCustomRDF(ref); if (s != null) { rdfSearchService.addData(s); } } } private List getSiteMasterItems(Connection connection) throws SQLException { PreparedStatement pst = null; ResultSet rst = null; try { pst = connection.prepareStatement("select " //$NON-NLS-1$ + SEARCH_BUILDER_ITEM_FIELDS + " from " //$NON-NLS-1$ + SEARCH_BUILDER_ITEM_T + " where itemscope = ? "); //$NON-NLS-1$ pst.clearParameters(); pst.setInt(1, SearchBuilderItem.ITEM_SITE_MASTER.intValue()); rst = pst.executeQuery(); ArrayList<SearchBuilderItemImpl> a = new ArrayList<SearchBuilderItemImpl>(); while (rst.next()) { SearchBuilderItemImpl sbi = new SearchBuilderItemImpl(); populateSearchBuilderItem(rst, sbi); a.add(sbi); } return a; } finally { try { rst.close(); } catch (Exception ex) { log.debug(ex); } try { pst.close(); } catch (Exception ex) { log.debug(ex); } } } /** * get the Instance Master * * @return * @throws HibernateException */ private SearchBuilderItem getMasterItem(Connection connection) throws SQLException { log.debug("get Master Items with " + connection); //$NON-NLS-1$ PreparedStatement pst = null; ResultSet rst = null; try { pst = connection.prepareStatement("select " //$NON-NLS-1$ + SEARCH_BUILDER_ITEM_FIELDS + " from " //$NON-NLS-1$ + SEARCH_BUILDER_ITEM_T + " where itemscope = ? "); //$NON-NLS-1$ pst.clearParameters(); pst.setInt(1, SearchBuilderItem.ITEM_GLOBAL_MASTER.intValue()); rst = pst.executeQuery(); SearchBuilderItemImpl sbi = new SearchBuilderItemImpl(); if (rst.next()) { populateSearchBuilderItem(rst, sbi); } else { sbi.setName(SearchBuilderItem.INDEX_MASTER); sbi.setContext(SearchBuilderItem.GLOBAL_CONTEXT); sbi.setSearchaction(SearchBuilderItem.ACTION_UNKNOWN); sbi.setSearchstate(SearchBuilderItem.STATE_UNKNOWN); sbi.setItemscope(SearchBuilderItem.ITEM_GLOBAL_MASTER); } return sbi; } finally { try { rst.close(); } catch (Exception ex) { log.debug(ex); } try { pst.close(); } catch (Exception ex) { log.debug(ex); } } } private void populateSearchBuilderItem(ResultSet rst, SearchBuilderItemImpl sbi) throws SQLException { sbi.setName(rst.getString(1)); sbi.setContext(rst.getString(2)); sbi.setSearchaction(Integer.valueOf(rst.getInt(3))); sbi.setSearchstate(Integer.valueOf(rst.getInt(4))); sbi.setVersion(rst.getDate(5)); sbi.setItemscope(Integer.valueOf(rst.getInt(6))); sbi.setId(rst.getString(7)); } private int populateStatement(PreparedStatement pst, SearchBuilderItem sbi) throws SQLException { pst.setString(1, sbi.getName()); pst.setString(2, sbi.getContext()); pst.setInt(3, sbi.getSearchaction().intValue()); pst.setInt(4, sbi.getSearchstate().intValue()); pst.setDate(5, new Date(sbi.getVersion().getTime())); pst.setInt(6, sbi.getItemscope().intValue()); pst.setString(7, sbi.getId()); return 7; } /** * Update a search builder item on success the item will be committed * @param connection * @param sbi * @throws SQLException */ private void updateOrSave(Connection connection, SearchBuilderItem sbi) throws SQLException { PreparedStatement pst = null; try { try { save(connection, sbi); } catch (SQLException sqlex) { pst = connection.prepareStatement("update " //$NON-NLS-1$ + SEARCH_BUILDER_ITEM_T + " set " //$NON-NLS-1$ + SEARCH_BUILDER_ITEM_FIELDS_UPDATE); populateStatement(pst, sbi); pst.executeUpdate(); connection.commit(); } } catch (SQLException ex) { log.warn("Failed ", ex); //$NON-NLS-1$ connection.rollback(); throw ex; } finally { try { if (pst != null) { pst.close(); } } catch (Exception ex) { log.debug(ex); } } } private void save(Connection connection, SearchBuilderItem sbi) throws SQLException { PreparedStatement pst = null; try { pst = connection.prepareStatement(" insert into " //$NON-NLS-1$ + SEARCH_BUILDER_ITEM_T + " ( " //$NON-NLS-1$ + SEARCH_BUILDER_ITEM_FIELDS + " ) values ( " //$NON-NLS-1$ + SEARCH_BUILDER_ITEM_FIELDS_PARAMS + " ) "); //$NON-NLS-1$ pst.clearParameters(); populateStatement(pst, sbi); pst.executeUpdate(); } finally { try { pst.close(); } catch (Exception ex) { log.debug(ex); } } } private void delete(Connection connection, SearchBuilderItem sbi) throws SQLException { PreparedStatement pst = null; try { pst = connection.prepareStatement(" delete from " //$NON-NLS-1$ + SEARCH_BUILDER_ITEM_T + " where id = ? "); //$NON-NLS-1$ pst.clearParameters(); pst.setString(1, sbi.getId()); pst.execute(); } catch (SQLException ex) { log.warn("Failed ", ex); //$NON-NLS-1$ throw ex; } finally { try { pst.close(); } catch (Exception ex) { log.debug(ex); } } } /** * get the action for the site master * * @param siteMaster * @return */ private Integer getSiteMasterAction(SearchBuilderItem siteMaster) { if (siteMaster.getName().startsWith(SearchBuilderItem.INDEX_MASTER) && !SearchBuilderItem.GLOBAL_CONTEXT.equals(siteMaster.getContext())) { if (SearchBuilderItem.STATE_PENDING.equals(siteMaster.getSearchstate())) { return siteMaster.getSearchaction(); } } return SearchBuilderItem.STATE_UNKNOWN; } /** * Get the site that the siteMaster references * * @param siteMaster * @return */ private String getSiteMasterSite(SearchBuilderItem siteMaster) { if (siteMaster.getName().startsWith(SearchBuilderItem.INDEX_MASTER) && !SearchBuilderItem.GLOBAL_CONTEXT.equals(siteMaster.getContext())) { // this depends on the pattern, perhapse it should be a parse return siteMaster.getName().substring(SearchBuilderItem.INDEX_MASTER.length() + 1); } return null; } /** * get the master action of known master item * * @param master * @return */ private Integer getMasterAction(SearchBuilderItem master) { if (master.getName().equals(SearchBuilderItem.GLOBAL_MASTER)) { if (SearchBuilderItem.STATE_PENDING.equals(master.getSearchstate())) { return master.getSearchaction(); } } return SearchBuilderItem.STATE_UNKNOWN; } private List findPending(int batchSize, Connection connection, SearchIndexBuilderWorker worker) throws SQLException { // Pending is the first 100 items // State == PENDING // Action != Unknown long start = System.currentTimeMillis(); try { log.debug("TXFind pending with " + connection); //$NON-NLS-1$ SearchBuilderItem masterItem = getMasterItem(connection); Integer masterAction = getMasterAction(masterItem); log.debug(" Master Item is " + masterItem.getName() + ":" //$NON-NLS-1$ //$NON-NLS-2$ + masterItem.getSearchaction() + ":" //$NON-NLS-1$ + masterItem.getSearchstate() + "::" //$NON-NLS-1$ + masterItem.getVersion()); if (SearchBuilderItem.ACTION_REFRESH.equals(masterAction)) { log.debug(" Master Action is " + masterAction); //$NON-NLS-1$ log.debug(" REFRESH = " + SearchBuilderItem.ACTION_REFRESH); //$NON-NLS-1$ log.debug(" RELOAD = " + SearchBuilderItem.ACTION_REBUILD); //$NON-NLS-1$ // get a complete list of all items, before the master // action version // if there are none, update the master action action to // completed // and return a blank list refreshIndex(connection, masterItem); } else if (SearchBuilderItem.ACTION_REBUILD.equals(masterAction)) { rebuildIndex(connection, masterItem, worker); } else { // get all site masters and perform the required action. List siteMasters = getSiteMasterItems(connection); for (Iterator i = siteMasters.iterator(); i.hasNext();) { SearchBuilderItem siteMaster = (SearchBuilderItem) i.next(); Integer action = getSiteMasterAction(siteMaster); if (SearchBuilderItem.ACTION_REBUILD.equals(action)) { rebuildIndex(connection, siteMaster, worker); } else if (SearchBuilderItem.ACTION_REFRESH.equals(action)) { refreshIndex(connection, siteMaster); } } } PreparedStatement pst = null; PreparedStatement lockedPst = null; ResultSet rst = null; try { pst = connection.prepareStatement("select " //$NON-NLS-1$ + SEARCH_BUILDER_ITEM_FIELDS + " from " //$NON-NLS-1$ + SEARCH_BUILDER_ITEM_T + " where searchstate = ? and " //$NON-NLS-1$ + " itemscope = ? order by version "); //$NON-NLS-1$ lockedPst = connection.prepareStatement("update " //$NON-NLS-1$ + SEARCH_BUILDER_ITEM_T + " set searchstate = ? " //$NON-NLS-1$ + " where id = ? and searchstate = ? "); //$NON-NLS-1$ pst.clearParameters(); pst.setInt(1, SearchBuilderItem.STATE_PENDING.intValue()); pst.setInt(2, SearchBuilderItem.ITEM.intValue()); rst = pst.executeQuery(); ArrayList<SearchBuilderItemImpl> a = new ArrayList<SearchBuilderItemImpl>(); while (rst.next() && a.size() < batchSize) { SearchBuilderItemImpl sbi = new SearchBuilderItemImpl(); populateSearchBuilderItem(rst, sbi); if (!SearchBuilderItem.ACTION_UNKNOWN.equals(sbi.getSearchaction())) { lockedPst.clearParameters(); lockedPst.setInt(1, SearchBuilderItem.STATE_LOCKED.intValue()); lockedPst.setString(2, sbi.getId()); lockedPst.setInt(3, SearchBuilderItem.STATE_PENDING.intValue()); if (lockedPst.executeUpdate() == 1) { sbi.setSearchstate(SearchBuilderItem.STATE_LOCKED); a.add(sbi); } connection.commit(); } } return a; } finally { try { rst.close(); } catch (Exception ex) { log.debug(ex); } try { pst.close(); } catch (Exception ex) { log.debug(ex); } } } finally { long finish = System.currentTimeMillis(); log.debug(" findPending took " + (finish - start) + " ms"); //$NON-NLS-1$ //$NON-NLS-2$ } } public int countPending(Connection connection) { PreparedStatement pst = null; ResultSet rst = null; try { pst = connection.prepareStatement("select count(*) from " //$NON-NLS-1$ + SEARCH_BUILDER_ITEM_T + " where searchstate = ? "); //$NON-NLS-1$ pst.clearParameters(); pst.setInt(1, SearchBuilderItem.STATE_PENDING.intValue()); rst = pst.executeQuery(); if (rst.next()) { return rst.getInt(1); } return 0; } catch (SQLException sqlex) { return 0; } finally { try { rst.close(); } catch (Exception ex) { log.debug(ex); } try { pst.close(); } catch (Exception ex) { log.debug(ex); } } } private void rebuildIndex(Connection connection, SearchBuilderItem controlItem, SearchIndexBuilderWorker worker) throws SQLException { // delete all and return the master action only // the caller will then rebuild the index from scratch log.info("DELETE ALL RECORDS =========================================================="); //$NON-NLS-1$ Statement stm = null; try { stm = connection.createStatement(); if (SearchBuilderItem.GLOBAL_CONTEXT.equals(controlItem.getContext())) { stm.execute("delete from searchbuilderitem where itemscope = " + SearchBuilderItem.ITEM + " or itemscope = " + SearchBuilderItem.ITEM_SITE_MASTER); //$NON-NLS-1$ } else { stm.execute("delete from searchbuilderitem where itemscope = " + SearchBuilderItem.ITEM); stm.execute("delete from searchbuilderitem where context = '" //$NON-NLS-1$ + controlItem.getContext() + "' and name <> '" //$NON-NLS-1$ + controlItem.getName() + "' "); //$NON-NLS-1$ } log.debug("DONE DELETE ALL RECORDS ==========================================================="); //$NON-NLS-1$ connection.commit(); log.debug("ADD ALL RECORDS ==========================================================="); //$NON-NLS-1$ long lastupdate = System.currentTimeMillis(); List<String> contextList = new ArrayList<String>(); if (SearchBuilderItem.GLOBAL_CONTEXT.equals(controlItem.getContext())) { for (Iterator<Site> i = SiteService .getSites(SelectionType.ANY, null, null, null, SortType.NONE, null).iterator(); i .hasNext();) { Site s = (Site) i.next(); if (!SiteService.isSpecialSite(s.getId())) { if (searchIndexBuilder.isOnlyIndexSearchToolSites()) { ToolConfiguration t = s.getToolForCommonId("sakai.search"); //$NON-NLS-1$ if (t != null) { contextList.add(s.getId()); } } else if (!(searchIndexBuilder.isExcludeUserSites() && SiteService.isUserSite(s.getId()))) { contextList.add(s.getId()); } } } } else { contextList.add(controlItem.getContext()); } for (Iterator<String> c = contextList.iterator(); c.hasNext();) { String siteContext = (String) c.next(); log.debug("Rebuild for " + siteContext); //$NON-NLS-1$ for (Iterator<EntityContentProducer> i = searchIndexBuilder.getContentProducers().iterator(); i .hasNext();) { EntityContentProducer ecp = (EntityContentProducer) i.next(); Iterator<String> contentIterator = null; contentIterator = ecp.getSiteContentIterator(siteContext); log.debug("Using ECP " + ecp); //$NON-NLS-1$ int added = 0; for (; contentIterator.hasNext();) { if ((System.currentTimeMillis() - lastupdate) > 60000L) { lastupdate = System.currentTimeMillis(); if (!worker.getLockTransaction(15L * 60L * 1000L, true)) { throw new RuntimeException("Transaction Lock Expired while Rebuilding Index "); //$NON-NLS-1$ } } String resourceName = (String) contentIterator.next(); log.debug("Checking " + resourceName); //$NON-NLS-1$ if (resourceName == null || resourceName.length() > 255) { log.warn("Entity Reference Longer than 255 characters, ignored: Reference=" //$NON-NLS-1$ + resourceName); continue; } SearchBuilderItem sbi = new SearchBuilderItemImpl(); sbi.setName(resourceName); sbi.setSearchaction(SearchBuilderItem.ACTION_ADD); sbi.setSearchstate(SearchBuilderItem.STATE_PENDING); sbi.setId(UUID.randomUUID().toString()); sbi.setVersion(new Date(System.currentTimeMillis())); sbi.setItemscope(SearchBuilderItem.ITEM); String context = null; try { context = ecp.getSiteId(resourceName); } catch (Exception ex) { log.debug("No context for resource " + resourceName //$NON-NLS-1$ + " defaulting to none"); //$NON-NLS-1$ } if (context == null || context.length() == 0) { context = "none"; //$NON-NLS-1$ } sbi.setContext(context); try { updateOrSave(connection, sbi); } catch (SQLException sqlex) { log.error("Failed to update " + sqlex.getMessage()); //$NON-NLS-1$ } connection.commit(); } log.debug(" Added " + added); //$NON-NLS-1$ } } log.info("DONE ADD ALL RECORDS ==========================================================="); //$NON-NLS-1$ controlItem.setSearchstate(SearchBuilderItem.STATE_COMPLETED); updateOrSave(connection, controlItem); connection.commit(); } finally { try { stm.close(); } catch (Exception ex) { log.debug(ex); } } } private void refreshIndex(Connection connection, SearchBuilderItem controlItem) throws SQLException { // delete all and return the master action only // the caller will then rebuild the index from scratch log.debug("UPDATE ALL RECORDS =========================================================="); //$NON-NLS-1$ Statement stm = null; try { stm = connection.createStatement(); if (SearchBuilderItem.GLOBAL_CONTEXT.equals(controlItem.getContext())) { stm.execute("update searchbuilderitem set searchstate = " //$NON-NLS-1$ + SearchBuilderItem.STATE_PENDING + " where itemscope = " + SearchBuilderItem.ITEM); //$NON-NLS-1$ } else { stm.execute("update searchbuilderitem set searchstate = " //$NON-NLS-1$ + SearchBuilderItem.STATE_PENDING + " where itemscope = " //$NON-NLS-1$ + SearchBuilderItem.ITEM_SITE_MASTER + " and context = '" + controlItem.getContext() + "' and name <> '" + controlItem.getName() + "'"); //$NON-NLS-2$ //$NON-NLS-3$ } controlItem.setSearchstate(SearchBuilderItem.STATE_COMPLETED); updateOrSave(connection, controlItem); connection.commit(); } finally { try { stm.close(); } catch (Exception ex) { log.debug(ex); } ; } } /** * @return Returns the indexStorage. */ public IndexStorage getIndexStorage() { return indexStorage; } /** * @param indexStorage * The indexStorage to set. */ public void setIndexStorage(IndexStorage indexStorage) { this.indexStorage = indexStorage; } /** * @return Returns the dataSource. */ public DataSource getDataSource() { return dataSource; } /** * @param dataSource * The dataSource to set. */ public void setDataSource(DataSource dataSource) { this.dataSource = dataSource; } public boolean isLockRequired() { return !indexStorage.isMultipleIndexers(); } /* * (non-Javadoc) * * @see org.sakaiproject.search.dao.SearchIndexBuilderWorkerDao#indexExists() */ public boolean indexExists() { return indexStorage.centralIndexExists(); } /** * @return the rdfSearchService */ public RDFSearchService getRdfSearchService() { return rdfSearchService; } /** * @param rdfSearchService the rdfSearchService to set */ public void setRdfSearchService(RDFSearchService rdfSearchService) { this.rdfSearchService = rdfSearchService; } /** * @return the searchIndexBuilder */ public SearchIndexBuilder getSearchIndexBuilder() { return searchIndexBuilder; } /** * @param searchIndexBuilder the searchIndexBuilder to set */ public void setSearchIndexBuilder(SearchIndexBuilder searchIndexBuilder) { this.searchIndexBuilder = searchIndexBuilder; } /** * @return the serverConfigurationService */ public ServerConfigurationService getServerConfigurationService() { return serverConfigurationService; } /** * @param serverConfigurationService the serverConfigurationService to set */ public void setServerConfigurationService(ServerConfigurationService serverConfigurationService) { this.serverConfigurationService = serverConfigurationService; } }