Java tutorial
/* =============================================================================== * * Part of the InfoGlue Content Management Platform (www.infoglue.org) * * =============================================================================== * * Copyright (C) * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License version 2, as published by the * Free Software Foundation. See the file LICENSE.html for more information. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY, including the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License along with * this program; if not, write to the Free Software Foundation, Inc. / 59 Temple * Place, Suite 330 / Boston, MA 02111-1307 / USA. * * =============================================================================== */ package org.infoglue.cms.controllers.kernel.impl.simple; import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.log4j.Logger; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.ClassicAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericField; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.queryParser.MultiFieldQueryParser; import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.search.Filter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.Version; import org.infoglue.cms.entities.kernel.BaseEntityVO; import org.infoglue.cms.security.InfoGlueGroup; import org.infoglue.cms.security.InfoGluePrincipal; import org.infoglue.cms.security.InfoGlueRole; import org.infoglue.cms.util.NotificationListener; import org.infoglue.cms.util.NotificationMessage; import org.infoglue.deliver.util.Timer; public class LuceneUsersController extends BaseController implements NotificationListener { private final static Logger logger = Logger.getLogger(LuceneUsersController.class.getName()); private static Directory idx = null; private static long idxCreatedDateTime = 0L; private static AtomicBoolean idxReindexing = new AtomicBoolean(false); /** * Default Constructor */ public static LuceneUsersController getController() { return new LuceneUsersController(); } private Analyzer getAnalyzer() throws Exception { return new ClassicAnalyzer(Version.LUCENE_34); //return new KeywordAnalyzer(); //return new WhitespaceAnalyzer(Version.LUCENE_34); //return new StandardAnalyzer(Version.LUCENE_34); } private synchronized void validateIndex() throws Exception { long indexTimeout = 3600000; if (idx == null) { reIndex(); } else { long diff = System.currentTimeMillis() - idxCreatedDateTime; if (idx == null || (diff > indexTimeout && !idxReindexing.getAndSet(true))) { new Thread() { public void run() { logger.info("Reindex..."); try { reIndex(); } catch (Exception e) { logger.warn("Error reindexing users: " + e.getMessage()); } finally { idxReindexing.set(false); } } }.start(); } } logger.info("" + idx); } /** * Call this method directly if you have a threaded implementation which syncs and caches the users on a regular basis. * * @throws Exception */ public synchronized void reIndex() throws Exception { Directory newIndex = new RAMDirectory(); logger.info("creating index...."); createIndex(newIndex); idx = newIndex; idxCreatedDateTime = System.currentTimeMillis(); } /** * Call this method directly if you must have a forced reset right away. * * @throws Exception */ public synchronized void reset() throws Exception { idx = null; } /** * This method creates the index by indexing all users. */ private Directory createIndex(Directory directory) throws CorruptIndexException, LockObtainFailedException, IOException, Exception { Analyzer analyzer = getAnalyzer(); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_34, analyzer); IndexWriter iw = new IndexWriter(directory, config); Timer t = new Timer(); addDocuments(iw, UserControllerProxy.getController().getAllUsers()); t.printElapsedTime("Getting and indexing all users took", 50); iw.optimize(); iw.close(); return idx; } /** * This method is used to fetch all or a subset of sorted users either filtered on a text or not. * If the index is not created or older than set interval the index is created. */ public List<InfoGluePrincipal> getFilteredUsers(Integer offset, Integer limit, String sortProperty, String direction, String searchString, boolean populateRolesAndGroups) throws Exception { Timer t = new Timer(); if (logger.isInfoEnabled()) { logger.info("offset: " + offset); logger.info("limit: " + limit); logger.info("sortProperty: " + sortProperty); logger.info("direction: " + direction); logger.info("searchString: " + searchString); logger.info("populateRolesAndGroups: " + populateRolesAndGroups); } validateIndex(); t.printElapsedTime("Validate index took", 50); List<InfoGluePrincipal> users = new ArrayList<InfoGluePrincipal>(); try { List<Document> docs; logger.info("sortProperty:" + sortProperty + ":" + direction); if (searchString == null || searchString.equals("")) docs = queryDocuments(offset, limit, sortProperty, direction); else docs = queryDocuments(offset, limit, sortProperty, direction, searchString); logger.info("docs:" + docs.size()); for (Document doc : docs) { String userName = doc.get("userName"); InfoGluePrincipal user = new InfoGluePrincipal(userName, doc.get("userName"), doc.get("firstName"), doc.get("lastName"), doc.get("email"), doc.get("source"), true, new Date(), new ArrayList(), new ArrayList(), new HashMap(), false, null); users.add(user); } } catch (Exception e) { logger.warn("Error getting filtered users:" + e.getMessage(), e); } logger.info("Users took:" + t.getElapsedTime()); return users; } /** * This method returns the number of users in the system (optionally filtered by text search). */ public Integer getUserCount(String searchString) throws Exception { List<Document> docs = null; validateIndex(); if (searchString == null || searchString.equals("")) docs = queryDocuments(0, 1000000, "userName", "asc"); else docs = queryDocuments(0, 1000000, "userName", "asc", searchString); return docs.size(); } /** * This method adds all users to the. */ private void addDocuments(IndexWriter iw, List<InfoGluePrincipal> users) throws CorruptIndexException, IOException { logger.info("Indexing users:" + users.size()); Timer t = new Timer(); for (InfoGluePrincipal user : users) { Document doc = new Document(); // Add user String contentToIndex = "" + user.getName().toLowerCase() + " " + user.getFirstName() + " " + user.getLastName() + " " + user.getEmail().toLowerCase() + " " + user.getSource(); doc.add(new Field("contents", new StringReader(contentToIndex))); doc.add(new Field("userName", user.getName().toLowerCase(), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); doc.add(new Field("firstName", user.getFirstName(), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); doc.add(new Field("lastName", user.getLastName(), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); doc.add(new Field("email", user.getEmail().toLowerCase(), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); doc.add(new Field("source", user.getSource(), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); doc.add(new NumericField("lastModified", Field.Store.YES, true) .setLongValue(user.getModifiedDateTime().getTime())); for (InfoGlueRole role : user.getRoles()) { if (!role.getName().equals("anonymous")) logger.info("role:" + role.getName()); doc.add(new Field("roles", new StringReader("" + role.getName()))); } for (InfoGlueGroup group : user.getGroups()) { if (!group.getName().equals("anonymous")) logger.info("group:" + group.getName()); doc.add(new Field("groups", new StringReader("" + group.getName()))); } iw.addDocument(doc); } if (logger.isInfoEnabled()) t.printElapsedTime("Indexing users took"); } /** * Getting the index searcher */ private IndexSearcher getIndexSearcher() throws Exception { IndexReader reader = IndexReader.open(idx, true); return new IndexSearcher(reader); } /** * A query api for getting all users sorted and paged */ public List<Document> queryDocuments(Integer offset, Integer limit, String sortProperty, String direction) throws Exception { IndexSearcher searcher = getIndexSearcher(); List<Document> docs = new ArrayList<Document>(); if (offset == null) offset = 0; if (limit == null) limit = 10; if (sortProperty == null) sortProperty = "userName"; boolean reverse = false; if (direction != null && direction.equalsIgnoreCase("desc")) reverse = true; logger.info("sortProperty:" + sortProperty); logger.info("reverse:" + reverse); SortField sf = new SortField(sortProperty, SortField.STRING, reverse); Query query = new MatchAllDocsQuery(); TopFieldDocs topDocs = searcher.search(query, (Filter) null, 1000000, new Sort(sf)); logger.info("offset:" + offset); logger.info("limit:" + limit); logger.info("topDocs.totalHits:" + topDocs.totalHits); int start = offset; int end = offset + limit; if (end > topDocs.totalHits) end = topDocs.totalHits; logger.info("start:" + start); logger.info("end:" + end); for (int i = start; i < end; i++) { ScoreDoc scoreDoc = topDocs.scoreDocs[i]; Document doc = searcher.doc(scoreDoc.doc); docs.add(doc); } searcher.close(); return docs; } /** * A query api for getting all users sorted and paged filtered on search text */ public List<Document> queryDocuments(Integer offset, Integer limit, String sortProperty, String direction, String searchText) throws Exception { Analyzer analyzer = getAnalyzer(); IndexSearcher searcher = getIndexSearcher(); List<Document> docs = new ArrayList<Document>(); if (offset == null) offset = 0; if (limit == null) limit = 10; if (sortProperty == null) sortProperty = "userName"; boolean reverse = false; if (direction != null && direction.equalsIgnoreCase("desc")) reverse = true; SortField sf = new SortField(sortProperty, SortField.STRING, reverse); logger.info("searchText:" + searchText); Query query = new QueryParser(Version.LUCENE_34, "contents", analyzer).parse(searchText + "*"); TopFieldDocs topDocs = searcher.search(query, (Filter) null, 1000000, new Sort(sf)); logger.info("offset:" + offset); logger.info("limit:" + limit); logger.info("topDocs.totalHits:" + topDocs.totalHits); int start = offset; int end = offset + limit; if (end > topDocs.totalHits) end = topDocs.totalHits; logger.info("start:" + start); logger.info("end:" + end); for (int i = start; i < end; i++) { ScoreDoc scoreDoc = topDocs.scoreDocs[i]; Document doc = searcher.doc(scoreDoc.doc); //System.out.println("doc:" + doc); docs.add(doc); } searcher.close(); return docs; } /** * A query api for getting user docs found by role or group mainly sorted and paged and optinally filtered on search text */ public List<Document> getDocuments(String field, String entityName, Integer offset, Integer limit, String sortProperty, String direction, String searchText) throws Exception { Analyzer analyzer = getAnalyzer(); IndexSearcher searcher = getIndexSearcher(); List<Document> docs = new ArrayList<Document>(); if (offset == null) offset = 0; if (limit == null) limit = 10; if (sortProperty == null) sortProperty = "userName"; boolean reverse = false; if (direction != null && direction.equalsIgnoreCase("desc")) reverse = true; SortField sf = new SortField(sortProperty, SortField.STRING, reverse); logger.info("searchText:" + searchText); Query query = new QueryParser(Version.LUCENE_34, field, analyzer).parse(entityName); if (searchText != null && !searchText.equals("")) { MultiFieldQueryParser mfqp = new MultiFieldQueryParser(Version.LUCENE_34, new String[] { field, "contents" }, analyzer); mfqp.setDefaultOperator(MultiFieldQueryParser.AND_OPERATOR); query = mfqp.parse(entityName + " " + searchText + "*"); } logger.info("query" + query); TopFieldDocs topDocs = searcher.search(query, (Filter) null, 1000000, new Sort(sf)); logger.info("offset:" + offset); logger.info("limit:" + limit); logger.info("topDocs.totalHits:" + topDocs.totalHits); int start = offset; int end = offset + limit; if (end > topDocs.totalHits) end = topDocs.totalHits; logger.info("start:" + start); logger.info("end:" + end); for (int i = start; i < end; i++) { ScoreDoc scoreDoc = topDocs.scoreDocs[i]; Document doc = searcher.doc(scoreDoc.doc); docs.add(doc); } searcher.close(); return docs; } @Override public void setContextParameters(Map map) { // TODO Auto-generated method stub } @Override public void notify(NotificationMessage message) { // TODO Auto-generated method stub } @Override public void process() throws Exception { // TODO Auto-generated method stub } @Override public BaseEntityVO getNewVO() { // TODO Auto-generated method stub return null; } }