List of usage examples for org.apache.lucene.search.highlight Highlighter setTextFragmenter
public void setTextFragmenter(Fragmenter fragmenter)
From source file:com.thinkgem.jeesite.common.persistence.BaseDaoImpl.java
License:Open Source License
/** * //from ww w. j a v a 2 s .c om * * @param query * * @param list * * @param fields * ?? */ public List<T> keywordsHighlight(BooleanQuery query, List<T> list, String... fields) { Analyzer analyzer = new IKAnalyzer(); Formatter formatter = new SimpleHTMLFormatter("<span class=\"highlight\">", "</span>"); Highlighter highlighter = new Highlighter(formatter, new QueryScorer(query)); highlighter.setTextFragmenter(new SimpleFragmenter(100)); for (T entity : list) { try { for (String field : fields) { String text = StringUtils.replaceHtml((String) Reflections.invokeGetter(entity, field)); String desciption = highlighter.getBestFragment(analyzer, field, text); if (desciption != null) { Reflections.invokeSetter(entity, fields[0], desciption); break; } Reflections.invokeSetter(entity, fields[0], StringUtils.abbreviate(text, 100)); } // Reflections.invokeSetter(entity, fields[1], // "sdfkjsdlkfjklsdjf"); } catch (IOException e) { e.printStackTrace(); } catch (InvalidTokenOffsetsException e) { e.printStackTrace(); } } return list; }
From source file:de.blizzy.documentr.search.GetSearchHitTask.java
License:Open Source License
@Override public SearchHit call() throws IOException { Formatter formatter = new SimpleHTMLFormatter("<strong>", "</strong>"); //$NON-NLS-1$ //$NON-NLS-2$ Scorer scorer = new QueryScorer(query); Highlighter highlighter = new Highlighter(formatter, scorer); highlighter.setTextFragmenter(new SimpleFragmenter(FRAGMENT_SIZE)); highlighter.setEncoder(new SimpleHTMLEncoder()); Document doc = reader.document(docId); String projectName = doc.get(PageIndex.PROJECT); String branchName = doc.get(PageIndex.BRANCH); String path = doc.get(PageIndex.PATH); String title = doc.get(PageIndex.TITLE); String text = doc.get(PageIndex.TEXT); String[] tagsArray = doc.getValues(PageIndex.TAG); List<String> tags = Lists.newArrayList(tagsArray); Collections.sort(tags);/* w w w . ja v a 2 s . co m*/ TokenStream tokenStream = null; String highlightedText = StringUtils.EMPTY; try { tokenStream = TokenSources.getAnyTokenStream(reader, docId, PageIndex.TEXT, doc, analyzer); String[] fragments = highlighter.getBestFragments(tokenStream, text, NUM_FRAGMENTS); cleanupFragments(fragments); highlightedText = Util.join(fragments, " <strong>...</strong> "); //$NON-NLS-1$ } catch (InvalidTokenOffsetsException e) { // ignore } finally { Closeables.closeQuietly(tokenStream); } return new SearchHit(projectName, branchName, path, title, highlightedText, tags); }
From source file:de.elbe5.cms.search.SearchData.java
License:Open Source License
public String getContext(Highlighter highlighter, Analyzer analyzer, String fieldName, int contextLength) { highlighter.setTextFragmenter(new SimpleFragmenter(contextLength)); TokenStream tokenStream = null;//from w ww . j a v a2s .c o m String text = getDoc().get(fieldName); if (text != null && text.length() > 0) tokenStream = analyzer.tokenStream(fieldName, new StringReader(text)); try { text = tokenStream == null ? "" : highlighter.getBestFragments(tokenStream, text, 1, "..."); } catch (Exception ignore) { } return text; }
From source file:de.fhg.iais.cortex.search.highlight.SolrTermRememberingHighlighter.java
License:Apache License
private Highlighter createNewHighlighter(String fieldName, SolrQueryRequest request, Highlighter originalHighlighter) { Formatter localDecoratedFormatter = createDecoratedFormatter(fieldName, request); Highlighter newHighlighter = new Highlighter(localDecoratedFormatter, originalHighlighter.getFragmentScorer()); newHighlighter.setTextFragmenter(originalHighlighter.getTextFragmenter()); return newHighlighter; }
From source file:de.innovationgate.wga.server.api.Lucene.java
License:Open Source License
@CodeCompletion public List<String> highlightLuceneField(String field, String originalText, String prefix, String suffix) throws WGException { WGACore core = _wga.getCore();//from w w w. ja va 2 s . c o m if (!core.isLuceneEnabled()) { _wga.getLog().warn("Unable to highlight text bc. lucene is not enabled."); return Collections.singletonList(originalText); } // try to retrieve last lucene query for highlighting org.apache.lucene.search.Query query = (org.apache.lucene.search.Query) _wga.getRequest().getSession() .getAttribute(Query.SESSION_ATTRIBUTE_SIMPLIFIED_LUCENEQUERY); if (query == null) { // no query in session - highlighting not possible return Collections.singletonList(originalText); } // create htmlformatter to highlight fragments with "$HIGHLIGHT_PREFIX$", "$HIGHLIGHT_SUFFIX$" // these placeholders are later on replaced by the given prefix and suffix // this additional step is necessary to encode the fragment text properly String prefixPlaceholder = "$HIGHLIGHT_PREFIX$"; String suffixPlaceholder = "$HIGHLIGHT_SUFFIX$"; SimpleHTMLFormatter formatter = new SimpleHTMLFormatter(prefixPlaceholder, suffixPlaceholder); // create highlighter Highlighter highlighter = core.getLuceneManager().createHighlighter(field, query, formatter); // create tokenstream TokenStream tokenStream = core.getLuceneManager().createTokenStream(originalText, _cx.content()); // create fragmenter and set fragmentsize to metaText.length to ensure only one fragments with the whole metaText is returned Fragmenter fragmenter = new SimpleFragmenter(originalText.length() + 1); // +1 is necessary here highlighter.setTextFragmenter(fragmenter); try { String highlighted = highlighter.getBestFragment(tokenStream, originalText); if (highlighted != null) { // replace highlight placeholders with correct prefix and suffix highlighted = WGUtils.strReplace(highlighted, prefixPlaceholder, prefix, true); highlighted = WGUtils.strReplace(highlighted, suffixPlaceholder, suffix, true); return Collections.singletonList(highlighted); } else { return Collections.singletonList(originalText); } } catch (IOException e) { _wga.getLog().warn("Unable to highlight text bc. of exception '" + e.getMessage() + "'."); return Collections.singletonList(originalText); } catch (InvalidTokenOffsetsException e) { _wga.getLog().warn("Unable to highlight meta text bc. of exception '" + e.getMessage() + "'."); return Collections.singletonList(originalText); } }
From source file:de.innovationgate.wga.server.api.Lucene.java
License:Open Source License
public List<String> bestFileFragments(int fragmentSize, int maxFragments, String prefix, String suffix, String encode) throws WGException { if (!_wga.getCore().isLuceneEnabled()) { _cx.addwarning("Unable to retrieve best file fragments - lucene is not enabled."); return Collections.emptyList(); }// w w w . j a va 2 s . c om if (_wga.database().db().getContentStoreVersion() < WGDatabase.CSVERSION_WGA5 || (_wga.database().db().getContentStoreVersion() == WGDatabase.CSVERSION_WGA5 && _wga.database().db().getContentStorePatchLevel() < 5)) { _cx.addwarning("bestFileFragments() is not supported on this content store version."); return Collections.emptyList(); } org.apache.lucene.search.Query query = (org.apache.lucene.search.Query) _cx.gethttpsession() .getAttribute(Query.SESSION_ATTRIBUTE_SIMPLIFIED_LUCENEQUERY); if (query == null) { // no query in session return Collections.emptyList(); } String filename = null; SearchDetails sd = _cx.getcontent().getSearchDetails(); if (sd != null && sd instanceof LuceneSearchDetails) { filename = ((LuceneSearchDetails) sd).getFilename(); } if (filename == null) { return Collections.emptyList(); } if (encode == null) { encode = _wga.design().getTmlDefaultEncoding(); } String prefixPlaceholder = "$HIGHLIGHT_PREFIX$"; String suffixPlaceholder = "$HIGHLIGHT_SUFFIX$"; SimpleHTMLFormatter formatter = new SimpleHTMLFormatter(prefixPlaceholder, suffixPlaceholder); // create highlighter Highlighter highlighter = _wga.getCore().getLuceneManager() .createHighlighter(LuceneManager.INDEXFIELD_ALLCONTENT, query, formatter); // retrieve attachment text WGFileMetaData md = _cx.content().getFileMetaData(filename); if (md == null) { return Collections.emptyList(); } BinaryFieldData textData = md.getPlainText(); if (textData == null) { return Collections.emptyList(); } try { // TODO highlighter does not support streams - should we limit plaintext size here? Reader textReader = new InputStreamReader(textData.getInputStream()); String text = IOUtils.toString(textReader); textReader.close(); // create tokenstream TokenStream tokenStream = _wga.getCore().getLuceneManager().createTokenStream(text, _cx.content()); // create fragmenter Fragmenter fragmenter = new SimpleFragmenter(fragmentSize); highlighter.setTextFragmenter(fragmenter); String[] highlighted = highlighter.getBestFragments(tokenStream, text, maxFragments); if (highlighted != null) { List<String> list = new ArrayList<String>(); for (int i = 0; i < highlighted.length; i++) { String fragment = highlighted[i]; if (encode != null) { try { fragment = _cx.multiencode(encode, fragment); } catch (FormattingException e) { _cx.addwarning("Unable to retrieve best fragments for file '" + filename + "' bc. of formating exception '" + e.getMessage() + "'."); return Collections.emptyList(); } } fragment = WGUtils.strReplace(fragment, prefixPlaceholder, prefix, true); fragment = WGUtils.strReplace(fragment, suffixPlaceholder, suffix, true); list.add(fragment); } return list; } else { return Collections.emptyList(); } } catch (Exception e) { _cx.addwarning("Unable to retrieve best fragments for file '" + filename + "' bc. of exception '" + e.getMessage() + "'."); return Collections.emptyList(); } }
From source file:de.innovationgate.wgpublisher.webtml.utils.TMLContext.java
License:Open Source License
@CodeCompletion public String highlightitem(String name, String prefix, String suffix, String encode) throws WGAPIException { if (name == null) { return null; }/* w ww . jav a 2s . c o m*/ // lowercase name name = name.toLowerCase(); // retrieve itemtext String originalText = itemTextValue(name, encode); if (originalText == null) { return null; } if (!getwgacore().isLuceneEnabled()) { addwarning("Unable to highlight item '" + name + "' bc. lucene is not enabled."); return originalText; } // try to retrieve last lucene query for highlighting org.apache.lucene.search.Query query = (org.apache.lucene.search.Query) getrequest().getSession() .getAttribute(Query.SESSION_ATTRIBUTE_SIMPLIFIED_LUCENEQUERY); if (query == null) { // no query in session - highlighting not possible addwarning( "Lucene highlighting not possible because there is no query with enabled highlighting support"); return originalText; } // create htmlformatter to highlight fragments with "$HIGHLIGHT_PREFIX$", "$HIGHLIGHT_SUFFIX$" // these placeholders are later on replaced by the given prefix and suffix // this additional step is necessary to encode the fragment text properly // see F00004C66 String prefixPlaceholder = "$HIGHLIGHT_PREFIX$"; String suffixPlaceholder = "$HIGHLIGHT_SUFFIX$"; SimpleHTMLFormatter formatter = new SimpleHTMLFormatter(prefixPlaceholder, suffixPlaceholder); // create highlighter Highlighter highlighter = getwgacore().getLuceneManager().createHighlighter(name, query, formatter); // create text to analyze LuceneConfiguration config = getwgacore().getLuceneManager() .retrieveLuceneConfig(content().getDatabase().getDbReference()); LuceneIndexItemRule rule = config.getMatchingItemRule(name); String analyzeText = rule.parseItemValue(originalText); // create tokenstream TokenStream tokenStream = getwgacore().getLuceneManager().createTokenStream(analyzeText, content()); // create fragmenter and set fragmentsize to itemText.length to ensure only one fragments with the whole itemText is returned Fragmenter fragmenter = new SimpleFragmenter(originalText.length() + 1); // if analyzeText.length == originalText.length we might get two fragments from lucene without the +1 (possible lucene bug) highlighter.setTextFragmenter(fragmenter); try { String highlighted = highlighter.getBestFragment(tokenStream, originalText); if (highlighted != null) { // replace highlight placeholders with correct prefix and suffix highlighted = WGUtils.strReplace(highlighted, prefixPlaceholder, prefix, true); highlighted = WGUtils.strReplace(highlighted, suffixPlaceholder, suffix, true); return highlighted; } } catch (IOException e) { addwarning("Unable to highlight item '" + name + "' bc. of exception '" + e.getMessage() + "'."); } catch (InvalidTokenOffsetsException e) { addwarning("Unable to highlight item '" + name + "' bc. of exception '" + e.getMessage() + "'."); } return originalText; }
From source file:de.innovationgate.wgpublisher.webtml.utils.TMLContext.java
License:Open Source License
/** * returns a singleton list with metavalues highlighted (surrounded with given <prefix> and <suffix>) based uppon the last lucene query with highlight attribute set to true * if highlighting is not possible this method returns metalist(<name>); * @param name//from w w w . j av a 2 s . com * @param prefix * @param suffix * @param encode * @return list * @throws WGAPIException */ @CodeCompletion public List highlightMeta(String name, String prefix, String suffix, String encode) throws WGAPIException { if (name == null) { return null; } String originalText = metaTextValue(name, encode); List<String> originalTextAsList = Collections.singletonList(originalText); if (!getwgacore().isLuceneEnabled()) { addwarning("Unable to highlight meta '" + name + "' bc. lucene is not enabled."); return originalTextAsList; } // try to retrieve last lucene query for highlighting org.apache.lucene.search.Query query = (org.apache.lucene.search.Query) getrequest().getSession() .getAttribute(Query.SESSION_ATTRIBUTE_SIMPLIFIED_LUCENEQUERY); if (query == null) { // no query in session - highlighting not possible return originalTextAsList; } // create htmlformatter to highlight fragments with "$HIGHLIGHT_PREFIX$", "$HIGHLIGHT_SUFFIX$" // these placeholders are later on replaced by the given prefix and suffix // this additional step is necessary to encode the fragment text properly String prefixPlaceholder = "$HIGHLIGHT_PREFIX$"; String suffixPlaceholder = "$HIGHLIGHT_SUFFIX$"; SimpleHTMLFormatter formatter = new SimpleHTMLFormatter(prefixPlaceholder, suffixPlaceholder); // create highlighter Highlighter highlighter = getwgacore().getLuceneManager().createHighlighter(name.toUpperCase(), query, formatter); // create tokenstream TokenStream tokenStream = getwgacore().getLuceneManager().createTokenStream(originalText, content()); // create fragmenter and set fragmentsize to metaText.length to ensure only one fragments with the whole metaText is returned Fragmenter fragmenter = new SimpleFragmenter(originalText.length() + 1); // +1 is necessary here highlighter.setTextFragmenter(fragmenter); try { String highlighted = highlighter.getBestFragment(tokenStream, originalText); if (highlighted != null) { // replace highlight placeholders with correct prefix and suffix highlighted = WGUtils.strReplace(highlighted, prefixPlaceholder, prefix, true); highlighted = WGUtils.strReplace(highlighted, suffixPlaceholder, suffix, true); return Collections.singletonList(highlighted); } else { return originalTextAsList; } } catch (IOException e) { addwarning("Unable to highlight meta '" + name + "' bc. of exception '" + e.getMessage() + "'."); return originalTextAsList; } catch (InvalidTokenOffsetsException e) { addwarning("Unable to highlight meta '" + name + "' bc. of exception '" + e.getMessage() + "'."); return originalTextAsList; } }
From source file:de.innovationgate.wgpublisher.webtml.utils.TMLContext.java
License:Open Source License
/** * retrieves a list of the best fragments from the given contentItem based upon the last lucene query with highlight attribute set to true * query hits are highlighted (surrounded by given <prefix> and <suffix>) in fragments * @param itemname the item fragements should be retrieved from * @param fragmentSize the number of characters for each fragment * @param maxFragments the maximum number of fragements returned * @param prefix the prefix for highlighting the search term in fragements * @param suffix the suffix for highlighting the search term in fragements * @param encode encode each fragment in the given encoding - prefix and suffix will not be encoded * @return list of fragements (Strings) - if no lucene query present returns EMPTY_LIST * @throws WGAPIException//from ww w.j a v a2 s .c o m */ @CodeCompletion public List bestfragments(String itemname, int fragmentSize, int maxFragments, String prefix, String suffix, String encode) throws WGAPIException { // check preconditions for highlighting if (itemname == null) { throw new WGIllegalArgumentException("Unable to retrieve best fragments for item 'null'."); } if (!getwgacore().isLuceneEnabled()) { addwarning("Unable to highlight item '" + itemname + "' bc. lucene is not enabled."); return Collections.EMPTY_LIST; } // try to retrieve last lucene query for highlighting org.apache.lucene.search.Query query = (org.apache.lucene.search.Query) getrequest().getSession() .getAttribute(Query.SESSION_ATTRIBUTE_SIMPLIFIED_LUCENEQUERY); if (query == null) { // no query in session - highlighting not possible return Collections.EMPTY_LIST; } // lowercase name itemname = itemname.toLowerCase(); // create htmlformatter to highlight fragments with "$HIGHLIGHT_PREFIX$", "$HIGHLIGHT_SUFFIX$" // these placeholders are later on replaced by the given prefix and suffix // this additional step is necessary to encode the fragment text properly // see B00004BBE String prefixPlaceholder = "$HIGHLIGHT_PREFIX$"; String suffixPlaceholder = "$HIGHLIGHT_SUFFIX$"; SimpleHTMLFormatter formatter = new SimpleHTMLFormatter(prefixPlaceholder, suffixPlaceholder); // create highlighter Highlighter highlighter = getwgacore().getLuceneManager().createHighlighter(itemname, query, formatter); // retrieve itemtext String text = itemTextValue(itemname, "none"); if (text == null) { return Collections.EMPTY_LIST; } // remove html/xml from text // fragments should not contain html/xml bc. of design issues try { text = WGUtils.toPlainText(text, " ", false); // B000049EA // if the item value contains encoded html entities these entities has been converted to their characters // we should do an html encode for sure // text = WGUtils.encodeHTML(text); --> has side effects @see B00004BBE } catch (IOException e) { addwarning("Unable to highlight item '" + itemname + "' bc. of exception '" + e.getMessage() + "'."); return Collections.EMPTY_LIST; } // create tokenstream TokenStream tokenStream = getwgacore().getLuceneManager().createTokenStream(text, content()); // create fragmenter Fragmenter fragmenter = new SimpleFragmenter(fragmentSize); highlighter.setTextFragmenter(fragmenter); try { String[] highlighted = highlighter.getBestFragments(tokenStream, text, maxFragments); if (highlighted != null) { ArrayList list = new ArrayList(); for (int i = 0; i < highlighted.length; i++) { // B00004BBE // evtl. encode fragment String fragment = highlighted[i]; if (encode != null) { try { fragment = multiencode(encode, fragment); } catch (FormattingException e) { addwarning("Unable to highlight item '" + itemname + "' bc. of formating exception '" + e.getMessage() + "'."); return Collections.EMPTY_LIST; } } // B00004BBE // replace highlight placeholders with correct prefix and suffix fragment = WGUtils.strReplace(fragment, prefixPlaceholder, prefix, true); fragment = WGUtils.strReplace(fragment, suffixPlaceholder, suffix, true); list.add(fragment); } return list; } else { return Collections.EMPTY_LIST; } } catch (IOException e) { addwarning("Unable to highlight item '" + itemname + "' bc. of exception '" + e.getMessage() + "'."); return Collections.EMPTY_LIST; } catch (InvalidTokenOffsetsException e) { addwarning("Unable to highlight item '" + itemname + "' bc. of exception '" + e.getMessage() + "'."); return Collections.EMPTY_LIST; } }
From source file:de.iteratec.iteraplan.businesslogic.service.SearchServiceImpl.java
License:Open Source License
/** * Execute the hibernate search/*from w w w . java2s .c om*/ * * @param searchMap * Multimap of SearhRowDTOs contains the results * @param queryString * the query string which the user entered */ private void executeSearchQuery(Multimap<String, SearchRowDTO> searchMap, String queryString, String buildingBlockTypeFilter) { // reader provider is required to close the reader after search has finished ReaderProvider readerProvider = searchDAO.getReaderProvider(); IndexReader reader = searchDAO.openReader(readerProvider, getClassArray()); // if the reader is null (i.e. the user has no functional permissions to search any one of the // building blocks) return without executing a search if (reader == null) { return; } try { // index fields that will be searched String[] productFields = { "attributeValueAssignments.attributeValue.valueString", "name", "version", "description", "informationSystem.name", "technicalComponent.name", "runtimePeriod.start", "runtimePeriod.end", "informationSystemReleaseA.informationSystem.name", "informationSystemReleaseA.version", "informationSystemReleaseB.informationSystem.name", "informationSystemReleaseB.version" }; QueryParser parser = new MultiFieldQueryParser(Version.LUCENE_31, productFields, new StandardAnalyzer(Version.LUCENE_31)); // allow wildcard * at the beginning of a query string parser.setAllowLeadingWildcard(true); // automatically put quotes around the search term for phrase search, because that's what most people expect parser.setAutoGeneratePhraseQueries(true); // workaround for known issue with highlighter and wildcard queries parser.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE); Query luceneQuery = null; try { // parse the query string luceneQuery = parser.parse(queryString); // rewrite luceneQuery for highlighting luceneQuery = luceneQuery.rewrite(reader); } catch (TooManyClauses tmcEx) { throw new IteraplanBusinessException(IteraplanErrorMessages.LUCENE_QUERY_TOO_COMPLEX, tmcEx); } catch (ParseException e) { throw new IteraplanBusinessException(IteraplanErrorMessages.LUCENE_QUERY_PARSE_FAILED, e); } catch (IOException e) { throw new IteraplanTechnicalException(IteraplanErrorMessages.LUCENE_QUERY_REWRITE_FAILED, e); } // the found content is being highlighted SimpleHTMLFormatter sHtmlF = new SimpleHTMLFormatter("<span class=\"highlighted\">", "</span>"); Highlighter highlighter = new Highlighter(sHtmlF, new QueryScorer(luceneQuery)); highlighter.setTextFragmenter(new SimpleFragmenter(40)); // sorted by Building block types SortField sortField = new SortField("buildingBlockType.name", SortField.STRING); Sort sort = new Sort(sortField); String[] projections = { "id", "buildingBlockType.name", FullTextQuery.DOCUMENT, "name", "informationSystem.name", "technicalComponent.name", "version", "description", "runtimePeriod.start", "runtimePeriod.end", "informationSystemReleaseA.informationSystem.name", "informationSystemReleaseA.version", "informationSystemReleaseB.informationSystem.name", "informationSystemReleaseB.version" }; List<Object[]> results = searchDAO.search(luceneQuery, sort, getClassArray(), projections); addResultsToSearchMap(searchMap, results, buildingBlockTypeFilter, highlighter); } finally { readerProvider.closeReader(reader); } }