List of usage examples for org.apache.solr.common SolrDocumentList SolrDocumentList
SolrDocumentList
From source file:net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector.java
License:Open Source License
/** * conversion from a SolrQueryResponse (which is a solr-internal data format) to SolrDocumentList (which is a solrj-format) * The conversion is done inside the solrj api using the BinaryResponseWriter and a very complex unfolding process * via org.apache.solr.common.util.JavaBinCodec.marshal. * @param request/* w w w .j a v a2 s. co m*/ * @param sqr * @return */ public SolrDocumentList SolrQueryResponse2SolrDocumentList(final SolrQueryRequest req, final SolrQueryResponse rsp) { SolrDocumentList sdl = new SolrDocumentList(); NamedList<?> nl = rsp.getValues(); ResultContext resultContext = (ResultContext) nl.get("response"); DocList response = resultContext == null ? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f) : resultContext.docs; sdl.setNumFound(response == null ? 0 : response.matches()); sdl.setStart(response == null ? 0 : response.offset()); String originalName = Thread.currentThread().getName(); if (response != null) { try { SolrIndexSearcher searcher = req.getSearcher(); final int responseCount = response.size(); DocIterator iterator = response.iterator(); for (int i = 0; i < responseCount; i++) { int docid = iterator.nextDoc(); Thread.currentThread() .setName("EmbeddedSolrConnector.SolrQueryResponse2SolrDocumentList: " + docid); Document responsedoc = searcher.doc(docid, (Set<String>) null); SolrDocument sordoc = doc2SolrDoc(responsedoc); sdl.add(sordoc); } } catch (IOException e) { ConcurrentLog.logException(e); } } Thread.currentThread().setName(originalName); return sdl; }
From source file:net.yacy.cora.federate.solr.connector.MirrorSolrConnector.java
License:Open Source License
/** * get a query result from solr//from w ww.j av a2 s .c o m * to get all results set the query String to "*:*" * @param querystring * @throws IOException */ @Override public SolrDocumentList getDocumentListByQuery(final String querystring, final String sort, final int offset, final int count, final String... fields) throws IOException { if (this.solr0 == null && this.solr1 == null) return new SolrDocumentList(); if (offset == 0 && count == 1 && querystring.startsWith("id:") && ((querystring.length() == 17 && querystring.charAt(3) == '"' && querystring.charAt(16) == '"') || querystring.length() == 15)) { final SolrDocumentList list = new SolrDocumentList(); SolrDocument doc = getDocumentById( querystring.charAt(3) == '"' ? querystring.substring(4, querystring.length() - 1) : querystring.substring(3), fields); list.add(doc); // no addToCache(list) here because that was already handlet in get(); return list; } if (this.solr0 != null && this.solr1 == null) { SolrDocumentList list = this.solr0.getDocumentListByQuery(querystring, sort, offset, count, fields); return list; } if (this.solr1 != null && this.solr0 == null) { SolrDocumentList list = this.solr1.getDocumentListByQuery(querystring, sort, offset, count, fields); return list; } // combine both lists SolrDocumentList l; l = this.solr0.getDocumentListByQuery(querystring, sort, offset, count, fields); if (l.size() >= count) return l; // at this point we need to know how many results are in solr0 // compute this with a very bad hack; replace with better method later int size0 = 0; { //bad hack - TODO: replace SolrDocumentList lHack = this.solr0.getDocumentListByQuery(querystring, sort, 0, Integer.MAX_VALUE, fields); size0 = lHack.size(); } // now use the size of the first query to do a second query final SolrDocumentList list = new SolrDocumentList(); for (final SolrDocument d : l) list.add(d); l = this.solr1.getDocumentListByQuery(querystring, sort, offset + l.size() - size0, count - l.size(), fields); for (final SolrDocument d : l) list.add(d); return list; }
From source file:net.yacy.cora.federate.solr.connector.MirrorSolrConnector.java
License:Open Source License
@Override public SolrDocumentList getDocumentListByParams(ModifiableSolrParams query) throws IOException, SolrException { Integer count0 = query.getInt(CommonParams.ROWS); int count = count0 == null ? 10 : count0.intValue(); Integer start0 = query.getInt(CommonParams.START); int start = start0 == null ? 0 : start0.intValue(); if (this.solr0 == null && this.solr1 == null) return new SolrDocumentList(); if (this.solr0 != null && this.solr1 == null) { SolrDocumentList list = this.solr0.getDocumentListByParams(query); return list; }// w ww . jav a 2 s . co m if (this.solr1 != null && this.solr0 == null) { SolrDocumentList list = this.solr1.getDocumentListByParams(query); return list; } // combine both lists final SolrDocumentList l = this.solr0.getDocumentListByParams(query); if (l.size() >= count) return l; // at this point we need to know how many results are in solr0 // compute this with a very bad hack; replace with better method later int size0 = 0; { //bad hack - TODO: replace query.set(CommonParams.START, 0); query.set(CommonParams.ROWS, Integer.MAX_VALUE); final SolrDocumentList lHack = this.solr0.getDocumentListByParams(query); query.set(CommonParams.START, start); query.set(CommonParams.ROWS, count); size0 = lHack.size(); } // now use the size of the first query to do a second query query.set(CommonParams.START, start + l.size() - size0); query.set(CommonParams.ROWS, count - l.size()); final SolrDocumentList l1 = this.solr1.getDocumentListByParams(query); query.set(CommonParams.START, start); query.set(CommonParams.ROWS, count); // TODO: combine both return l1; }
From source file:net.yacy.cora.federate.solr.instance.ResponseAccumulator.java
License:Open Source License
public ResponseAccumulator() { this.fieldsAcc = new SimpleOrderedMap<Object>(); this.index_countsAcc = new SimpleOrderedMap<Object>(); this.facet_countsAcc = new SimpleOrderedMap<Object>(); this.highlightingAcc = new SimpleOrderedMap<Object>(); this.headerAcc = new SimpleOrderedMap<Object>(); this.resultsAcc = new SolrDocumentList(); }
From source file:net.yacy.cora.federate.solr.logic.AbstractTerm.java
License:Open Source License
/** * create a hit subset of the given SolrDocumentList according to the conjunction defined * in this object/* w w w .j a v a2 s .c om*/ * @param sdl the SolrDocumentList * @return a manufactured subset-clone of the given SolrDocumentList where document match with the term as given in this object */ @Override public SolrDocumentList apply(SolrDocumentList sdl) { SolrDocumentList r = new SolrDocumentList(); int numFound = 0; for (SolrDocument d : sdl) { if (matches(d)) { r.add(d); numFound++; } } r.setNumFound(numFound); return r; }
From source file:net.yacy.search.schema.CollectionConfiguration.java
License:Open Source License
public void postprocessing_doublecontent(Segment segment, Set<String> uniqueURLs, SolrDocument doc, final SolrInputDocument sid, final DigestURL url) { // FIND OUT IF THIS IS A DOUBLE DOCUMENT // term to describe documents which are indexable: // - no noindex in meta oder x-robots // - no canonical-tag Conjunction ValidDocTermTemplate = new Conjunction(); ValidDocTermTemplate.addOperand(new LongLiteral(CollectionSchema.httpstatus_i, 200)); ValidDocTermTemplate.addOperand(//from ww w. j a v a2 s . com new Disjunction(new Negation(new CatchallLiteral(CollectionSchema.canonical_equal_sku_b)), new BooleanLiteral(CollectionSchema.canonical_equal_sku_b, true))); ValidDocTermTemplate.addOperand(new Negation(new LongLiteral(CollectionSchema.robots_i, 8))); // bit 3 (noindex) ValidDocTermTemplate.addOperand(new Negation(new LongLiteral(CollectionSchema.robots_i, 24))); // bit 3 + 4 (noindex + nofollow) ValidDocTermTemplate.addOperand(new Negation(new LongLiteral(CollectionSchema.robots_i, 512))); // bit 9 (noindex) ValidDocTermTemplate.addOperand(new Negation(new LongLiteral(CollectionSchema.robots_i, 1536))); // bit 9 + 10 (noindex + nofollow) String urlhash = ASCII.String(url.hash()); String hostid = url.hosthash(); Disjunction dnf = new Disjunction(); CollectionSchema[][] doccheckschema = new CollectionSchema[][] { { CollectionSchema.exact_signature_l, CollectionSchema.exact_signature_unique_b, CollectionSchema.exact_signature_copycount_i }, { CollectionSchema.fuzzy_signature_l, CollectionSchema.fuzzy_signature_unique_b, CollectionSchema.fuzzy_signature_copycount_i } }; uniquecheck: for (CollectionSchema[] checkfields : doccheckschema) { CollectionSchema signaturefield = checkfields[0]; CollectionSchema uniquefield = checkfields[1]; CollectionSchema countfield = checkfields[2]; if (this.contains(signaturefield) && this.contains(uniquefield) && this.contains(countfield)) { // lookup the document with the same signature Long signature = (Long) doc.getFieldValue(signaturefield.getSolrFieldName()); if (signature == null) continue uniquecheck; //con.addOperand(new Negation(new Literal(CollectionSchema.id, urlhash))); //con.addOperand(new Literal(CollectionSchema.host_id_s, hostid)); dnf.addOperand(new LongLiteral(signaturefield, signature)); } } Conjunction con = (Conjunction) ValidDocTermTemplate.clone(); con.addOperand(dnf); con.addOperand(new Negation(new StringLiteral(CollectionSchema.id, urlhash))); con.addOperand(new StringLiteral(CollectionSchema.host_id_s, hostid)); String query = con.toString(); SolrDocumentList docsAkk; try { docsAkk = segment.fulltext().getDefaultConnector().getDocumentListByQuery(query, null, 0, 1000, CollectionSchema.id.getSolrFieldName(), CollectionSchema.exact_signature_l.getSolrFieldName(), CollectionSchema.fuzzy_signature_l.getSolrFieldName()); } catch (final IOException e) { ConcurrentLog.logException(e); docsAkk = new SolrDocumentList(); } if (docsAkk.getNumFound() > 0) uniquecheck: for (CollectionSchema[] checkfields : doccheckschema) { CollectionSchema signaturefield = checkfields[0]; CollectionSchema uniquefield = checkfields[1]; CollectionSchema countfield = checkfields[2]; if (this.contains(signaturefield) && this.contains(uniquefield) && this.contains(countfield)) { // lookup the document with the same signature Long signature = (Long) doc.getFieldValue(signaturefield.getSolrFieldName()); if (signature == null) continue uniquecheck; SolrDocumentList docs = new StringLiteral(signaturefield, signature.toString()).apply(docsAkk); if (docs.getNumFound() == 0) { sid.setField(uniquefield.getSolrFieldName(), true); sid.setField(countfield.getSolrFieldName(), 1); } else { boolean firstappearance = true; for (SolrDocument d : docs) { if (uniqueURLs.contains(d.getFieldValue(CollectionSchema.id.getSolrFieldName()))) firstappearance = false; break; } sid.setField(uniquefield.getSolrFieldName(), firstappearance); sid.setField(countfield.getSolrFieldName(), docs.getNumFound() + 1); // the current url was excluded from search but is included in count } } } // CHECK IF TITLE AND DESCRIPTION IS UNIQUE (this is by default not switched on) // in case that the document has no status code 200, has a noindex attribute // or a canonical tag which does not point to the document itself, // then the unique-field is not written at all! Integer robots_i = this.contains(CollectionSchema.robots_i) ? (Integer) doc.getFieldValue(CollectionSchema.robots_i.getSolrFieldName()) : null; Integer httpstatus_i = this.contains(CollectionSchema.httpstatus_i) ? (Integer) doc.getFieldValue(CollectionSchema.httpstatus_i.getSolrFieldName()) : null; String canonical_s = this.contains(CollectionSchema.canonical_s) ? (String) doc.getFieldValue(CollectionSchema.canonical_s.getSolrFieldName()) : null; Boolean canonical_equal_sku_b = this.contains(CollectionSchema.canonical_equal_sku_b) ? (Boolean) doc.getFieldValue(CollectionSchema.canonical_equal_sku_b.getSolrFieldName()) : null; CollectionSchema[][] metadatacheckschema = new CollectionSchema[][] { { CollectionSchema.title, CollectionSchema.title_exact_signature_l, CollectionSchema.title_unique_b }, { CollectionSchema.description_txt, CollectionSchema.description_exact_signature_l, CollectionSchema.description_unique_b } }; if (segment.fulltext().getDefaultConfiguration().contains(CollectionSchema.host_id_s) && (robots_i == null || (robots_i.intValue() & (1 << 9)) == 0 /*noindex in http X-ROBOTS*/ && (robots_i.intValue() & (1 << 3)) == 0 /*noindex in html metas*/ ) && (canonical_s == null || canonical_s.length() == 0 || (canonical_equal_sku_b != null && canonical_equal_sku_b.booleanValue()) || url.toNormalform(true).equals(canonical_s)) && (httpstatus_i == null || httpstatus_i.intValue() == 200)) { uniquecheck: for (CollectionSchema[] checkfields : metadatacheckschema) { CollectionSchema checkfield = checkfields[0]; CollectionSchema signaturefield = checkfields[1]; CollectionSchema uniquefield = checkfields[2]; if (this.contains(checkfield) && this.contains(signaturefield) && this.contains(uniquefield)) { // lookup in the index within the same hosts for the same title or description //String checkstring = checkfield == CollectionSchema.title ? document.dc_title() : document.dc_description(); Long signature = (Long) doc.getFieldValue(signaturefield.getSolrFieldName()); if (signature == null) { continue uniquecheck; } try { Conjunction doccountterm = (Conjunction) ValidDocTermTemplate.clone(); doccountterm.addOperand(new Negation(new StringLiteral(CollectionSchema.id, urlhash))); doccountterm.addOperand(new StringLiteral(CollectionSchema.host_id_s, hostid)); doccountterm.addOperand(new LongLiteral(signaturefield, signature)); long doccount = segment.fulltext().getDefaultConnector() .getCountByQuery("{!cache=false}" + doccountterm.toString()); sid.setField(uniquefield.getSolrFieldName(), doccount == 0); } catch (final IOException e) { } } } } uniqueURLs.add(urlhash); }
From source file:nl.knaw.huygens.timbuctoo.index.solr.SolrIndexTest.java
License:Open Source License
private void setupQueryResponseForQueryWithResults(SolrQueryMatcher query, Map<String, Object>... results) throws SolrServerException { QueryResponse queryResponse = mock(QueryResponse.class); SolrDocumentList solrDocuments = new SolrDocumentList(); for (Map<String, Object> result : results) { solrDocuments.add(createDoc(result)); }// w w w .ja v a2s . c o m when(queryResponse.getResults()).thenReturn(solrDocuments); when(solrServerMock.search(argThat(query))).thenReturn(queryResponse); }
From source file:opennlp.tools.similarity.apps.solr.IterativeSearchRequestHandler.java
License:Apache License
public void handleRequestBody1(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { // extract params from request SolrParams params = req.getParams(); String q = params.get(CommonParams.Q); String[] fqs = params.getParams(CommonParams.FQ); int start = 0; try {/*from w ww. j a v a2 s .co m*/ start = Integer.parseInt(params.get(CommonParams.START)); } catch (Exception e) { /* default */ } int rows = 0; try { rows = Integer.parseInt(params.get(CommonParams.ROWS)); } catch (Exception e) { /* default */ } //SolrPluginUtils.setReturnFields(req, rsp); // build initial data structures SolrDocumentList results = new SolrDocumentList(); SolrIndexSearcher searcher = req.getSearcher(); Map<String, SchemaField> fields = req.getSchema().getFields(); int ndocs = start + rows; Filter filter = buildFilter(fqs, req); Set<Integer> alreadyFound = new HashSet<Integer>(); // invoke the various sub-handlers in turn and return results doSearch1(results, searcher, q, filter, ndocs, req, fields, alreadyFound); // ... more sub-handler calls here ... // build and write response float maxScore = 0.0F; int numFound = 0; List<SolrDocument> slice = new ArrayList<SolrDocument>(); for (Iterator<SolrDocument> it = results.iterator(); it.hasNext();) { SolrDocument sdoc = it.next(); Float score = (Float) sdoc.getFieldValue("score"); if (maxScore < score) { maxScore = score; } if (numFound >= start && numFound < start + rows) { slice.add(sdoc); } numFound++; } results.clear(); results.addAll(slice); results.setNumFound(numFound); results.setMaxScore(maxScore); results.setStart(start); rsp.add("response", results); }
From source file:org.apache.blur.slur.BlurResultHelper.java
License:Apache License
public static SolrDocumentList from(BlurResults results) { SolrDocumentList docResults = new SolrDocumentList(); convertMetadata(results, docResults); convertRows(results.getResults(), docResults); return docResults; }
From source file:org.apache.hadoop.hive.solr.SolrDAO.java
License:Apache License
SolrDAO(String nodeURL, String shardName, String collectionName, SolrQuery query) {
this.nodeURL = nodeURL;
this.shardName = shardName;
this.collectionName = collectionName;
this.solrServer = new HttpSolrServer(this.nodeURL + "/" + this.shardName);
this.currentPosition = 0;
this.query = query;
this.start = 0;
this.window = 1000;
if (query != null) {
initSize();//from w w w.j a va2 s . c o m
this.inputBuffer = new SolrDocumentList();
this.inputDocs = new SolrDocumentList();
this.readerCB = new CyclicBarrier(2);
this.nextCursorMark = new StringBuffer();
this.nextCursorMark.append("*");
LOG.debug("Starting the SolrBatchReader thread for start = " + start + ", window = " + window);
T = new Thread(new SolrBatchReader(start, window, size, query, solrServer, inputBuffer, readerCB,
nextCursorMark));
T.start();
} else {
this.outputBuffer = new ArrayList<SolrInputDocument>();
this.outputDocs = new ArrayList<SolrInputDocument>();
this.writerCB = new CyclicBarrier(2);
this.isWriterThreadInitiated = false;
}
}