List of usage examples for org.apache.lucene.index.memory MemoryIndex addField
public void addField(String fieldName, TokenStream stream, int positionIncrementGap)
From source file:org.elasticsearch.percolator.MultiDocumentPercolatorIndex.java
License:Apache License
MemoryIndex indexDoc(ParseContext.Document d, Analyzer analyzer) {
MemoryIndex memoryIndex = new MemoryIndex(true);
for (IndexableField field : d.getFields()) {
if (!field.fieldType().indexed() && field.name().equals(UidFieldMapper.NAME)) {
continue;
}// w w w . j ava2 s. c o m
try {
TokenStream tokenStream = field.tokenStream(analyzer);
if (tokenStream != null) {
memoryIndex.addField(field.name(), tokenStream, field.boost());
}
} catch (IOException e) {
throw new ElasticsearchException("Failed to create token stream", e);
}
}
return memoryIndex;
}
From source file:org.elasticsearch.percolator.PercolateQueryTests.java
License:Apache License
public void testPercolateQuery() throws Exception { List<Iterable<? extends IndexableField>> docs = new ArrayList<>(); List<Query> queries = new ArrayList<>(); PercolateQuery.QueryStore queryStore = ctx -> queries::get; queries.add(new TermQuery(new Term("field", "fox"))); docs.add(Collections.singleton(new StringField("select", "a", Field.Store.NO))); SpanNearQuery.Builder snp = new SpanNearQuery.Builder("field", true); snp.addClause(new SpanTermQuery(new Term("field", "jumps"))); snp.addClause(new SpanTermQuery(new Term("field", "lazy"))); snp.addClause(new SpanTermQuery(new Term("field", "dog"))); snp.setSlop(2);// w w w. ja v a2s . c om queries.add(snp.build()); docs.add(Collections.singleton(new StringField("select", "b", Field.Store.NO))); PhraseQuery.Builder pq1 = new PhraseQuery.Builder(); pq1.add(new Term("field", "quick")); pq1.add(new Term("field", "brown")); pq1.add(new Term("field", "jumps")); pq1.setSlop(1); queries.add(pq1.build()); docs.add(Collections.singleton(new StringField("select", "b", Field.Store.NO))); BooleanQuery.Builder bq1 = new BooleanQuery.Builder(); bq1.add(new TermQuery(new Term("field", "quick")), BooleanClause.Occur.MUST); bq1.add(new TermQuery(new Term("field", "brown")), BooleanClause.Occur.MUST); bq1.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.MUST); queries.add(bq1.build()); docs.add(Collections.singleton(new StringField("select", "b", Field.Store.NO))); indexWriter.addDocuments(docs); indexWriter.close(); directoryReader = DirectoryReader.open(directory); IndexSearcher shardSearcher = newSearcher(directoryReader); MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer()); IndexSearcher percolateSearcher = memoryIndex.createSearcher(); // no scoring, wrapping it in a constant score query: Query query = new ConstantScoreQuery(new PercolateQuery("type", queryStore, new BytesArray("a"), new TermQuery(new Term("select", "a")), percolateSearcher, new MatchNoDocsQuery(""))); TopDocs topDocs = shardSearcher.search(query, 10); assertThat(topDocs.totalHits, equalTo(1)); assertThat(topDocs.scoreDocs.length, equalTo(1)); assertThat(topDocs.scoreDocs[0].doc, equalTo(0)); Explanation explanation = shardSearcher.explain(query, 0); assertThat(explanation.isMatch(), is(true)); assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[0].score)); query = new ConstantScoreQuery(new PercolateQuery("type", queryStore, new BytesArray("b"), new TermQuery(new Term("select", "b")), percolateSearcher, new MatchNoDocsQuery(""))); topDocs = shardSearcher.search(query, 10); assertThat(topDocs.totalHits, equalTo(3)); assertThat(topDocs.scoreDocs.length, equalTo(3)); assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); explanation = shardSearcher.explain(query, 1); assertThat(explanation.isMatch(), is(true)); assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[0].score)); assertThat(topDocs.scoreDocs[1].doc, equalTo(2)); explanation = shardSearcher.explain(query, 2); assertThat(explanation.isMatch(), is(true)); assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[1].score)); assertThat(topDocs.scoreDocs[2].doc, equalTo(3)); explanation = shardSearcher.explain(query, 2); assertThat(explanation.isMatch(), is(true)); assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[2].score)); query = new ConstantScoreQuery(new PercolateQuery("type", queryStore, new BytesArray("c"), new MatchAllDocsQuery(), percolateSearcher, new MatchAllDocsQuery())); topDocs = shardSearcher.search(query, 10); assertThat(topDocs.totalHits, equalTo(4)); query = new PercolateQuery("type", queryStore, new BytesArray("{}"), new TermQuery(new Term("select", "b")), percolateSearcher, new MatchNoDocsQuery("")); topDocs = shardSearcher.search(query, 10); assertThat(topDocs.totalHits, equalTo(3)); assertThat(topDocs.scoreDocs.length, equalTo(3)); assertThat(topDocs.scoreDocs[0].doc, equalTo(3)); explanation = shardSearcher.explain(query, 3); assertThat(explanation.isMatch(), is(true)); assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[0].score)); assertThat(explanation.getDetails(), arrayWithSize(1)); assertThat(topDocs.scoreDocs[1].doc, equalTo(2)); explanation = shardSearcher.explain(query, 2); assertThat(explanation.isMatch(), is(true)); assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[1].score)); assertThat(explanation.getDetails(), arrayWithSize(1)); assertThat(topDocs.scoreDocs[2].doc, equalTo(1)); explanation = shardSearcher.explain(query, 1); assertThat(explanation.isMatch(), is(true)); assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[2].score)); assertThat(explanation.getDetails(), arrayWithSize(1)); }
From source file:org.elasticsearch.percolator.PercolatorFieldMapperTests.java
License:Apache License
public void testCreateCandidateQuery() throws Exception { addQueryMapping();/*from w w w. ja v a2 s . c o m*/ MemoryIndex memoryIndex = new MemoryIndex(false); memoryIndex.addField("field1", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer()); memoryIndex.addField("field2", "some more text", new WhitespaceAnalyzer()); memoryIndex.addField("_field3", "unhide me", new WhitespaceAnalyzer()); memoryIndex.addField("field4", "123", new WhitespaceAnalyzer()); memoryIndex.addField(new LongPoint("number_field", 10L), new WhitespaceAnalyzer()); IndexReader indexReader = memoryIndex.createSearcher().getIndexReader(); TermsQuery termsQuery = (TermsQuery) fieldType.createCandidateQuery(indexReader); PrefixCodedTerms terms = termsQuery.getTermData(); assertThat(terms.size(), equalTo(15L)); PrefixCodedTerms.TermIterator termIterator = terms.iterator(); assertTermIterator(termIterator, "_field3\u0000me", fieldType.queryTermsField.name()); assertTermIterator(termIterator, "_field3\u0000unhide", fieldType.queryTermsField.name()); assertTermIterator(termIterator, "field1\u0000brown", fieldType.queryTermsField.name()); assertTermIterator(termIterator, "field1\u0000dog", fieldType.queryTermsField.name()); assertTermIterator(termIterator, "field1\u0000fox", fieldType.queryTermsField.name()); assertTermIterator(termIterator, "field1\u0000jumps", fieldType.queryTermsField.name()); assertTermIterator(termIterator, "field1\u0000lazy", fieldType.queryTermsField.name()); assertTermIterator(termIterator, "field1\u0000over", fieldType.queryTermsField.name()); assertTermIterator(termIterator, "field1\u0000quick", fieldType.queryTermsField.name()); assertTermIterator(termIterator, "field1\u0000the", fieldType.queryTermsField.name()); assertTermIterator(termIterator, "field2\u0000more", fieldType.queryTermsField.name()); assertTermIterator(termIterator, "field2\u0000some", fieldType.queryTermsField.name()); assertTermIterator(termIterator, "field2\u0000text", fieldType.queryTermsField.name()); assertTermIterator(termIterator, "field4\u0000123", fieldType.queryTermsField.name()); assertTermIterator(termIterator, EXTRACTION_FAILED, fieldType.extractionResultField.name()); }
From source file:org.elasticsearch.percolator.PercolatorQueryTests.java
License:Apache License
public void testVariousQueries() throws Exception { addPercolatorQuery("1", new TermQuery(new Term("field", "brown"))); addPercolatorQuery("2", new TermQuery(new Term("field", "monkey"))); addPercolatorQuery("3", new TermQuery(new Term("field", "fox"))); BooleanQuery.Builder bq1 = new BooleanQuery.Builder(); bq1.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.SHOULD); bq1.add(new TermQuery(new Term("field", "monkey")), BooleanClause.Occur.SHOULD); addPercolatorQuery("4", bq1.build()); BooleanQuery.Builder bq2 = new BooleanQuery.Builder(); bq2.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.MUST); bq2.add(new TermQuery(new Term("field", "monkey")), BooleanClause.Occur.MUST); addPercolatorQuery("5", bq2.build()); BooleanQuery.Builder bq3 = new BooleanQuery.Builder(); bq3.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.MUST); bq3.add(new TermQuery(new Term("field", "apes")), BooleanClause.Occur.MUST_NOT); addPercolatorQuery("6", bq3.build()); BooleanQuery.Builder bq4 = new BooleanQuery.Builder(); bq4.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.MUST_NOT); bq4.add(new TermQuery(new Term("field", "apes")), BooleanClause.Occur.MUST); addPercolatorQuery("7", bq4.build()); PhraseQuery.Builder pq1 = new PhraseQuery.Builder(); pq1.add(new Term("field", "lazy")); pq1.add(new Term("field", "dog")); addPercolatorQuery("8", pq1.build()); indexWriter.close();//from w w w . j a v a2s . c o m directoryReader = DirectoryReader.open(directory); IndexSearcher shardSearcher = newSearcher(directoryReader); MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer()); IndexSearcher percolateSearcher = memoryIndex.createSearcher(); PercolatorQuery.Builder builder = new PercolatorQuery.Builder(percolateSearcher, queries, new MatchAllDocsQuery()); builder.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME); TopDocs topDocs = shardSearcher.search(builder.build(), 10); assertThat(topDocs.totalHits, equalTo(5)); assertThat(topDocs.scoreDocs.length, equalTo(5)); assertThat(topDocs.scoreDocs[0].doc, equalTo(0)); assertThat(topDocs.scoreDocs[1].doc, equalTo(2)); assertThat(topDocs.scoreDocs[2].doc, equalTo(3)); assertThat(topDocs.scoreDocs[3].doc, equalTo(5)); assertThat(topDocs.scoreDocs[4].doc, equalTo(7)); }
From source file:org.elasticsearch.percolator.PercolatorQueryTests.java
License:Apache License
public void testWithScoring() throws Exception { addPercolatorQuery("1", new TermQuery(new Term("field", "brown")), "field", "value1"); indexWriter.close();//from w w w . ja v a 2 s .c om directoryReader = DirectoryReader.open(directory); IndexSearcher shardSearcher = newSearcher(directoryReader); MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer()); IndexSearcher percolateSearcher = memoryIndex.createSearcher(); PercolatorQuery.Builder builder = new PercolatorQuery.Builder(percolateSearcher, queries, new MatchAllDocsQuery()); builder.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME); builder.setPercolateQuery(new TermQuery(new Term("field", "value1"))); PercolatorQuery percolatorQuery = builder.build(); TopDocs topDocs = shardSearcher.search(percolatorQuery, 1); assertThat(topDocs.totalHits, equalTo(1)); assertThat(topDocs.scoreDocs.length, equalTo(1)); assertThat(topDocs.scoreDocs[0].doc, equalTo(0)); assertThat(topDocs.scoreDocs[0].score, not(1f)); Explanation explanation = shardSearcher.explain(percolatorQuery, 0); assertThat(explanation.isMatch(), is(true)); assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[0].score)); }
From source file:org.elasticsearch.percolator.PercolatorQueryTests.java
License:Apache License
public void testDuel() throws Exception { int numQueries = scaledRandomIntBetween(32, 256); for (int i = 0; i < numQueries; i++) { String id = Integer.toString(i); Query query;//from w w w . j a v a2s . c o m if (randomBoolean()) { query = new PrefixQuery(new Term("field", id)); } else if (randomBoolean()) { query = new WildcardQuery(new Term("field", id + "*")); } else if (randomBoolean()) { query = new CustomQuery(new Term("field", id + "*")); } else { query = new TermQuery(new Term("field", id)); } addPercolatorQuery(id, query); } indexWriter.close(); directoryReader = DirectoryReader.open(directory); IndexSearcher shardSearcher = newSearcher(directoryReader); for (int i = 0; i < numQueries; i++) { MemoryIndex memoryIndex = new MemoryIndex(); String id = Integer.toString(i); memoryIndex.addField("field", id, new WhitespaceAnalyzer()); IndexSearcher percolateSearcher = memoryIndex.createSearcher(); PercolatorQuery.Builder builder1 = new PercolatorQuery.Builder(percolateSearcher, queries, new MatchAllDocsQuery()); // enables the optimization that prevents queries from being evaluated that don't match builder1.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME); TopDocs topDocs1 = shardSearcher.search(builder1.build(), 10); PercolatorQuery.Builder builder2 = new PercolatorQuery.Builder(percolateSearcher, queries, new MatchAllDocsQuery()); TopDocs topDocs2 = shardSearcher.search(builder2.build(), 10); assertThat(topDocs1.totalHits, equalTo(topDocs2.totalHits)); assertThat(topDocs1.scoreDocs.length, equalTo(topDocs2.scoreDocs.length)); for (int j = 0; j < topDocs1.scoreDocs.length; j++) { assertThat(topDocs1.scoreDocs[j].doc, equalTo(topDocs2.scoreDocs[j].doc)); } } }
From source file:org.elasticsearch.percolator.PercolatorServiceTests.java
License:Apache License
public void testCount() throws Exception { PercolateContext context = mock(PercolateContext.class); when(context.shardTarget()).thenReturn(new SearchShardTarget("_id", new Index("_index", "_na_"), 0)); when(context.percolatorTypeFilter()).thenReturn(new MatchAllDocsQuery()); when(context.isOnlyCount()).thenReturn(true); IndexShard shard = mock(IndexShard.class); when(shard.shardId()).thenReturn(new ShardId("_index", "_na_", 0)); when(context.indexShard()).thenReturn(shard); PercolatorQueriesRegistry registry = createRegistry(); addPercolatorQuery("1", new TermQuery(new Term("field", "brown")), indexWriter, registry); addPercolatorQuery("2", new TermQuery(new Term("field", "fox")), indexWriter, registry); addPercolatorQuery("3", new TermQuery(new Term("field", "monkey")), indexWriter, registry); indexWriter.close();//from w w w . j a v a 2s . com directoryReader = DirectoryReader.open(directory); IndexSearcher shardSearcher = newSearcher(directoryReader); when(context.searcher()).thenReturn(new ContextIndexSearcher(new Engine.Searcher("test", shardSearcher), shardSearcher.getQueryCache(), shardSearcher.getQueryCachingPolicy())); MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer()); IndexSearcher percolateSearcher = memoryIndex.createSearcher(); when(context.docSearcher()).thenReturn(percolateSearcher); PercolateShardResponse response = PercolatorService.doPercolate(context, registry, null, null, null); assertThat(response.topDocs().totalHits, equalTo(2)); }
From source file:org.elasticsearch.percolator.PercolatorServiceTests.java
License:Apache License
public void testTopMatching() throws Exception { PercolateContext context = mock(PercolateContext.class); when(context.shardTarget()).thenReturn(new SearchShardTarget("_id", new Index("_index", "_na_"), 0)); when(context.percolatorTypeFilter()).thenReturn(new MatchAllDocsQuery()); when(context.size()).thenReturn(10); IndexShard shard = mock(IndexShard.class); when(shard.shardId()).thenReturn(new ShardId("_index", "_na_", 0)); when(context.indexShard()).thenReturn(shard); PercolatorQueriesRegistry registry = createRegistry(); addPercolatorQuery("1", new TermQuery(new Term("field", "brown")), indexWriter, registry); addPercolatorQuery("2", new TermQuery(new Term("field", "monkey")), indexWriter, registry); addPercolatorQuery("3", new TermQuery(new Term("field", "fox")), indexWriter, registry); indexWriter.close();/* w ww . jav a 2s. c o m*/ directoryReader = DirectoryReader.open(directory); IndexSearcher shardSearcher = newSearcher(directoryReader); when(context.searcher()).thenReturn(new ContextIndexSearcher(new Engine.Searcher("test", shardSearcher), shardSearcher.getQueryCache(), shardSearcher.getQueryCachingPolicy())); MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer()); IndexSearcher percolateSearcher = memoryIndex.createSearcher(); when(context.docSearcher()).thenReturn(percolateSearcher); PercolateShardResponse response = PercolatorService.doPercolate(context, registry, null, null, null); TopDocs topDocs = response.topDocs(); assertThat(topDocs.totalHits, equalTo(2)); assertThat(topDocs.scoreDocs.length, equalTo(2)); assertThat(topDocs.scoreDocs[0].doc, equalTo(0)); assertThat(topDocs.scoreDocs[1].doc, equalTo(2)); }
From source file:org.elasticsearch.percolator.SingleDocumentPercolatorIndex.java
License:Apache License
@Override public void prepare(PercolateContext context, ParsedDocument parsedDocument) { MemoryIndex memoryIndex = cache.get(); for (IndexableField field : parsedDocument.rootDoc().getFields()) { if (!field.fieldType().indexed() && field.name().equals(UidFieldMapper.NAME)) { continue; }/*w w w . j av a2 s . c o m*/ try { TokenStream tokenStream = field.tokenStream(parsedDocument.analyzer()); if (tokenStream != null) { memoryIndex.addField(field.name(), tokenStream, field.boost()); } } catch (IOException e) { throw new ElasticsearchException("Failed to create token stream", e); } } context.initialize(new DocEngineSearcher(memoryIndex), parsedDocument); }
From source file:org.gridkit.coherence.search.lucene.LuceneSearchPlugin.java
License:Apache License
@Override public boolean evaluate(Query query, Object document) { Field[] fields = (Field[]) document; MemoryIndex memIndex = new MemoryIndex(); for (Field field : fields) { memIndex.addField(field.name(), field.tokenStreamValue(), field.getBoost()); }/*from w ww. j a v a 2s . c o m*/ return memIndex.search(query) > 0.0f; }