List of usage examples for org.apache.solr.common SolrDocumentList getNumFound
public long getNumFound()
From source file:jp.aegif.nemaki.cmis.aspect.query.solr.SolrQueryProcessor.java
License:Open Source License
@Override public ObjectList query(CallContext callContext, String repositoryId, String statement, Boolean searchAllVersions, Boolean includeAllowableActions, IncludeRelationships includeRelationships, String renditionFilter, BigInteger maxItems, BigInteger skipCount, ExtensionsData extension) { SolrServer solrServer = solrUtil.getSolrServer(); // replacing backslashed for TIMESTAMP only Pattern time_p = Pattern.compile("(TIMESTAMP\\s?'[\\-\\d]*T\\d{2})\\\\:(\\d{2})\\\\:([\\.\\d]*Z')", Pattern.CASE_INSENSITIVE); Matcher time_m = time_p.matcher(statement); statement = time_m.replaceAll("$1:$2:$3"); // TODO walker is required? QueryUtilStrict util = new QueryUtilStrict(statement, new CmisTypeManager(repositoryId, typeManager), null); QueryObject queryObject = util.getQueryObject(); // Get where caluse as Tree Tree whereTree = null;//from www . ja v a 2 s .c o m try { util.processStatement(); Tree tree = util.parseStatement(); whereTree = extractWhereTree(tree); } catch (Exception e) { e.printStackTrace(); } // Build solr statement of WHERE String whereQueryString = ""; if (whereTree == null || whereTree.isNil()) { whereQueryString = "*:*"; } else { try { SolrPredicateWalker solrPredicateWalker = new SolrPredicateWalker(repositoryId, queryObject, solrUtil, contentService); Query whereQuery = solrPredicateWalker.walkPredicate(whereTree); whereQueryString = whereQuery.toString(); } catch (Exception e) { e.printStackTrace(); // TODO Output more detailed exception exceptionService.invalidArgument("Invalid CMIS SQL statement!"); } } // Build solr query of FROM String fromQueryString = ""; String repositoryQuery = "repository_id:" + repositoryId; fromQueryString += repositoryQuery + " AND "; TypeDefinition td = null; td = queryObject.getMainFromName(); // includedInSupertypeQuery List<TypeDefinitionContainer> typeDescendants = typeManager.getTypesDescendants(repositoryId, td.getId(), BigInteger.valueOf(-1), false); Iterator<TypeDefinitionContainer> iterator = typeDescendants.iterator(); List<String> tables = new ArrayList<String>(); while (iterator.hasNext()) { TypeDefinition descendant = iterator.next().getTypeDefinition(); if (td.getId() != descendant.getId()) { boolean isq = (descendant.isIncludedInSupertypeQuery() == null) ? false : descendant.isIncludedInSupertypeQuery(); if (!isq) continue; } String table = descendant.getQueryName(); tables.add(table.replaceAll(":", "\\\\:")); } // Term t = new Term( // solrUtil.getPropertyNameInSolr(PropertyIds.OBJECT_TYPE_ID), // StringUtils.join(tables, " ")); // fromQueryString += new TermQuery(t).toString(); fromQueryString += "(" + solrUtil.getPropertyNameInSolr(repositoryId, PropertyIds.OBJECT_TYPE_ID) + ":" + StringUtils.join(tables, " " + solrUtil.getPropertyNameInSolr(repositoryId, PropertyIds.OBJECT_TYPE_ID) + ":") + ")"; // Execute query SolrQuery solrQuery = new SolrQuery(); solrQuery.setQuery(whereQueryString); solrQuery.setFilterQueries(fromQueryString); logger.info(solrQuery.toString()); logger.info("statement: " + statement); logger.info("skipCount: " + skipCount); logger.info("maxItems: " + maxItems); if (skipCount == null) { solrQuery.set(CommonParams.START, 0); } else { solrQuery.set(CommonParams.START, skipCount.intValue()); } if (maxItems == null) { solrQuery.set(CommonParams.ROWS, 50); } else { solrQuery.set(CommonParams.ROWS, maxItems.intValue()); } QueryResponse resp = null; try { resp = solrServer.query(solrQuery); } catch (SolrServerException e) { e.printStackTrace(); } long numFound = 0; // Output search results to ObjectList if (resp != null && resp.getResults() != null && resp.getResults().getNumFound() != 0) { SolrDocumentList docs = resp.getResults(); numFound = docs.getNumFound(); List<Content> contents = new ArrayList<Content>(); for (SolrDocument doc : docs) { String docId = (String) doc.getFieldValue("object_id"); Content c = contentService.getContent(repositoryId, docId); // When for some reason the content is missed, pass through if (c == null) { logger.warn("[objectId=" + docId + "]It is missed in DB but still rests in Solr."); } else { contents.add(c); } } List<Lock> locks = threadLockService.readLocks(repositoryId, contents); try { threadLockService.bulkLock(locks); // Filter out by permissions List<Content> permitted = permissionService.getFiltered(callContext, repositoryId, contents); // Filter return value with SELECT clause Map<String, String> requestedWithAliasKey = queryObject.getRequestedPropertiesByAlias(); String filter = null; if (!requestedWithAliasKey.keySet().contains("*")) { // Create filter(queryNames) from query aliases filter = StringUtils.join(requestedWithAliasKey.values(), ","); } // Build ObjectList String orderBy = orderBy(queryObject); ObjectList result = compileService.compileObjectDataListForSearchResult(callContext, repositoryId, permitted, filter, includeAllowableActions, includeRelationships, renditionFilter, false, maxItems, skipCount, false, orderBy, numFound); return result; } finally { threadLockService.bulkUnlock(locks); } } else { ObjectListImpl nullList = new ObjectListImpl(); nullList.setHasMoreItems(false); nullList.setNumItems(BigInteger.ZERO); return nullList; } }
From source file:jp.aegif.nemaki.query.solr.SolrPredicateWalker.java
License:Open Source License
/** * Get all subfolder ids by connecting to Solr recursively * * @param folderId/*w ww .j ava2 s . c o m*/ * @param solrServer * @return */ private List<String> getDescendantFolderId(String folderId, SolrServer solrServer) { List<String> list = new ArrayList<String>(); list.add(folderId); // Add oneself to the list in advance SolrQuery query = new SolrQuery(); query.setQuery(solrUtil.getPropertyNameInSolr(PropertyIds.PARENT_ID) + ":" + folderId + " AND " + solrUtil.getPropertyNameInSolr(PropertyIds.BASE_TYPE_ID) + ":cmis\\:folder"); // only "folder" nodes // Connect to SolrServer and add subfolder ids to the list try { QueryResponse resp = solrServer.query(query); SolrDocumentList children = resp.getResults(); // END NODE case: Do nothing but return oneself if (children.getNumFound() == 0) { return list; // Other than END NODE case: collect descendants values // recursively } else { Iterator<SolrDocument> iterator = resp.getResults().iterator(); while (iterator.hasNext()) { SolrDocument child = iterator.next(); String childId = (String) child.getFieldValue("id"); // Recursive call to this method List<String> l = getDescendantFolderId(childId, solrServer); list.addAll(l); } return list; } } catch (SolrServerException e) { e.printStackTrace(); return null; } }
From source file:jp.sf.fess.solr.plugin.suggest.SuggestUpdateControllerTest.java
License:Apache License
public void test_incrementUpdate() { final SuggestSolrServer suggestSolrServer = TestUtils.createSuggestSolrServer(); try {/* ww w .j av a 2 s. c om*/ suggestSolrServer.deleteAll(); suggestSolrServer.commit(); final SuggestUpdateConfig config = TestUtils.getSuggestUpdateConfig(); final SuggestUpdateController controller = new SuggestUpdateController(config, getSuggestFieldInfoList(config, false), new SolrResourceLoader(SolrResourceLoader.locateSolrHome())); controller.addLabelFieldName("label"); controller.addRoleFieldName("role"); controller.start(); final SolrInputDocument doc = new SolrInputDocument(); for (int i = 0; i < 5; i++) { doc.setField("content", "??"); doc.setField("label", "label" + i); doc.setField("role", "role" + i); doc.setField(config.getExpiresField(), DateUtil.getThreadLocalDateFormat().format(new Date())); controller.add(doc); Thread.sleep(5 * 1000); } controller.commit(); Thread.sleep(5 * 1000); final SolrDocumentList solrDocuments = suggestSolrServer.select("*:*"); assertEquals(1, solrDocuments.getNumFound()); final SolrDocument solrDocument = solrDocuments.get(0); final Object count = solrDocument.getFieldValue(SuggestConstants.SuggestFieldNames.COUNT); assertEquals("5", count.toString()); final Collection<Object> labels = solrDocument .getFieldValues(SuggestConstants.SuggestFieldNames.LABELS); assertEquals(5, labels.size()); for (int i = 0; i < 5; i++) { assertTrue(labels.contains("label" + i)); } final Collection<Object> roles = solrDocument.getFieldValues(SuggestConstants.SuggestFieldNames.ROLES); assertEquals(5, roles.size()); for (int i = 0; i < 5; i++) { assertTrue(roles.contains("role" + i)); } } catch (final Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
From source file:kbSRU.kbSRU.java
License:Open Source License
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { response.setContentType(XML_RESPONSE_HEADER); // Talkback happens in XML form. response.setCharacterEncoding("UTF-8"); // Unicode++ request.setCharacterEncoding("UTF-8"); PrintWriter out = null; // The talkback buffer. // handle startrecord Integer startRecord = 0;/* w ww . j av a 2s. c om*/ if (!(request.getParameter("startRecord") == null)) { try { startRecord = Integer.parseInt(request.getParameter("startRecord")) - 1; } catch (NumberFormatException e) { startRecord = 0; } } // maximumrecords Integer maximumRecords = Integer.parseInt(this.config.getProperty("default_maximumRecords")); if (!(request.getParameter("maximumRecords") == null)) { maximumRecords = Integer.parseInt(request.getParameter("maximumRecords")); } // operation String operation = request.getParameter("operation"); // x_collection String x_collection = request.getParameter("x-collection"); if (x_collection == null) x_collection = this.config.getProperty("default_x_collection"); if (x_collection == null) operation = null; // sortkeys String sortKeys = request.getParameter("sortKeys"); // sortorder String sortOrder = request.getParameter("sortOrder"); // recordschema String recordSchema = request.getParameter("recordSchema"); if (recordSchema == null) recordSchema = "dc"; if (recordSchema.equalsIgnoreCase("dcx")) { recordSchema = "dcx"; } if (recordSchema.equalsIgnoreCase("solr")) { recordSchema = "solr"; } // query request String query = request.getParameter("query"); String q = request.getParameter("q"); // who is requestor ? String remote_ip = request.getHeader("X-FORWARDED-FOR"); if (remote_ip == null) { remote_ip = request.getRemoteAddr().trim(); } else { remote_ip = request.getHeader("X-FORWARDED-FOR"); } // handle debug Boolean debug = Boolean.parseBoolean(request.getParameter("debug")); if (!debug) { out = new PrintWriter(new OutputStreamWriter(response.getOutputStream(), "UTF8"), true); } // handle query if ((query == null) && (q != null)) { query = q; } else { if ((query != null) && (q == null)) { q = query; } else { operation = null; } } // handle operation if (operation == null) { if (query != null) { operation = "searchRetrieve"; } else { operation = "explain"; } } // searchRetrieve if (operation.equalsIgnoreCase("searchRetrieve")) { if (query == null) { operation = "explain"; log.debug(operation + ":" + query); } } // start talking back. String[] sq = { "" }; String solrquery = ""; // facet String facet = null; List<FacetField> fct = null; if (request.getParameter("facet") != null) { facet = request.getParameter("facet"); log.debug("facet : " + facet); } if (operation == null) { operation = "searchretrieve"; } else { // explain response if (operation.equalsIgnoreCase("explain")) { log.debug("operation = explain"); out.write("<srw:explainResponse xmlns:srw=\"http://www.loc.gov/zing/srw/\">"); out.write("</srw:explainResponse>"); } else { // DEBUG routine operation = "searchretrieve"; String triplequery = null; if (query.matches(".*?\\[.+?\\].*?")) { // New symantic syntax triplequery = symantic_query(query); query = query.split("\\[")[0] + " " + triplequery; log.fatal(triplequery); solrquery = CQLtoLucene.translate(query, log, config); } else { solrquery = CQLtoLucene.translate(query, log, config); } log.debug(solrquery); if (debug == true) { response.setContentType(HTML_RESPONSE_HEADER); out = new PrintWriter(new OutputStreamWriter(response.getOutputStream(), "UTF8"), true); out.write("<html><body>\n\n"); out.write("'" + remote_ip + "'<br>\n"); out.write("<form action='http://www.kbresearch.nl/kbSRU'>"); out.write("<input type=text name=q value='" + query + "' size=120>"); out.write("<input type=hidden name=debug value=True>"); out.write("<input type=submit>"); out.write("<table border=1><tr><td>"); out.write("q</td><td>" + query + "</td></tr><tr>"); out.write("<td>query out</td><td>" + URLDecoder.decode(solrquery) + "</td></tr>"); out.write("<tr><td>SOLR_URL</td><td> <a href='" + this.config.getProperty("collection." + x_collection.toLowerCase() + ".solr_baseurl") + "/?q=" + solrquery + "'>" + this.config.getProperty("collection." + x_collection.toLowerCase() + ".solr_baseurl") + "/select/?q=" + solrquery + "</a><br>" + this.config.getProperty("solr_url") + solrquery + "</td></tr>"); out.write("<b>SOLR_QUERY</b> : <BR> <iframe width=900 height=400 src='" + this.config.getProperty("collection." + x_collection.toLowerCase() + ".solr_baseurl") + "/../?q=" + solrquery + "'></iframe><BR>"); out.write("<b>SRU_QUERY</b> : <BR> <a href=" + this.config.getProperty("baseurl") + "?q=" + query + "'>" + this.config.getProperty("baseurl") + "?q=" + query + "</a><br><iframe width=901 height=400 src='http://www.kbresearch.nl/kbSRU/?q=" + query + "'></iframe><BR>"); out.write("<br><b>JSRU_QUERY</b> : <BR><a href='http://jsru.kb.nl/sru/?query=" + query + "&x-collection=" + x_collection + "'>http://jsru.kb.nl/sru/?query=" + query + "&x-collection=GGC</a><br><iframe width=900 height=400 src='http://jsru.kb.nl/sru/?query=" + query + "&x-collection=GGC'></iframe>"); } else { // XML SearchRetrieve response String url = this.config .getProperty("collection." + x_collection.toLowerCase() + ".solr_baseurl"); String buffer = ""; CommonsHttpSolrServer server = null; server = new CommonsHttpSolrServer(url); log.fatal("URSING " + url); server.setParser(new XMLResponseParser()); int numfound = 0; try { SolrQuery do_query = new SolrQuery(); do_query.setQuery(solrquery); do_query.setRows(maximumRecords); do_query.setStart(startRecord); if ((sortKeys != null) && (sortKeys.length() > 1)) { if (sortOrder != null) { if (sortOrder.equals("asc")) { do_query.setSortField(sortKeys, SolrQuery.ORDER.asc); } if (sortOrder.equals("desc")) { do_query.setSortField(sortKeys, SolrQuery.ORDER.desc); } } else { for (String str : sortKeys.trim().split(",")) { str = str.trim(); if (str.length() > 1) { if (str.equals("date")) { do_query.setSortField("date_date", SolrQuery.ORDER.desc); log.debug("SORTORDERDEBUG | DATE! " + str + " | "); break; } else { do_query.setSortField(str + "_str", SolrQuery.ORDER.asc); log.debug("SORTORDERDEBUG | " + str + " | "); break; } } } } } if (facet != null) { if (facet.indexOf(",") > 1) { for (String str : facet.split(",")) { if (str.indexOf("date") > 1) { do_query.addFacetField(str); } else { do_query.addFacetField(str); } //do_query.setParam("facet.method", "enum"); } //q.setFacetSort(false); } else { do_query.addFacetField(facet); } do_query.setFacet(true); do_query.setFacetMinCount(1); do_query.setFacetLimit(-1); } log.fatal(solrquery); QueryResponse rsp = null; boolean do_err = false; boolean do_sugg = false; SolrDocumentList sdl = null; String diag = ""; StringBuffer suggest = new StringBuffer(""); String content = "1"; SolrQuery spellq = do_query; try { rsp = server.query(do_query); } catch (SolrServerException e) { String header = this.SRW_HEADER.replaceAll("\\$numberOfRecords", "0"); out.write(header); diag = this.SRW_DIAG.replaceAll("\\$error", e.getMessage()); do_err = true; rsp = null; } log.fatal("query done.."); if (!(do_err)) { // XML dc response SolrDocumentList docs = rsp.getResults(); numfound = (int) docs.getNumFound(); int count = startRecord; String header = this.SRW_HEADER.replaceAll("\\$numberOfRecords", Integer.toString(numfound)); out.write(header); out.write("<srw:records>"); Iterator<SolrDocument> iter = rsp.getResults().iterator(); while (iter.hasNext()) { count += 1; if (recordSchema.equalsIgnoreCase("dc")) { SolrDocument resultDoc = iter.next(); content = (String) resultDoc.getFieldValue("id"); out.write("<srw:record>"); out.write("<srw:recordPacking>xml</srw:recordPacking>"); out.write("<srw:recordSchema>info:srw/schema/1/dc-v1.1</srw:recordSchema>"); out.write( "<srw:recordData xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:mods=\"http://www.loc.gov/mods\" xmlns:dcterms=\"http://purl.org/dc/terms/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:dcx=\"http://krait.kb.nl/coop/tel/handbook/telterms.html\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:marcrel=\"http://www.loc.gov/loc.terms/relators/OTH\" xmlns:facets=\"info:srw/extension/4/facets\" >"); StringBuffer result = new StringBuffer(""); construct_lucene_dc(result, resultDoc); out.write(result.toString()); out.write("</srw:recordData>"); out.write("<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>"); out.write("</srw:record>"); } if (recordSchema.equalsIgnoreCase("solr")) { SolrDocument resultDoc = iter.next(); content = (String) resultDoc.getFieldValue("id"); out.write("<srw:record>"); out.write("<srw:recordPacking>xml</srw:recordPacking>"); out.write("<srw:recordSchema>info:srw/schema/1/solr</srw:recordSchema>"); out.write("<srw:recordData xmlns:expand=\"http://www.kbresearch.nl/expand\">"); StringBuffer result = new StringBuffer(""); construct_lucene_solr(result, resultDoc); out.write(result.toString()); out.write("</srw:recordData>"); out.write("<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>"); out.write("</srw:record>"); } if (recordSchema.equalsIgnoreCase("dcx")) { // XML dcx response out.write("<srw:record>"); out.write("<srw:recordPacking>xml</srw:recordPacking>"); out.write("<srw:recordSchema>info:srw/schema/1/dc-v1.1</srw:recordSchema>"); out.write( "<srw:recordData><srw_dc:dc xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:mods=\"http://www.loc.gov/mods\" xmlns:dcterms=\"http://purl.org/dc/terms/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:dcx=\"http://krait.kb.nl/coop/tel/handbook/telterms.html\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:marcrel=\"http://www.loc.gov/marc.relators/\" xmlns:expand=\"http://www.kbresearch.nl/expand\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" >"); SolrDocument resultDoc = iter.next(); content = (String) resultDoc.getFieldValue("id"); String dcx_data = helpers.getOAIdcx( "http://services.kb.nl/mdo/oai?verb=GetRecord&identifier=" + content, log); if (x_collection.equalsIgnoreCase("ggc-thes")) { dcx_data = helpers.getOAIdcx( "http://serviceso.kb.nl/mdo/oai?verb=GetRecord&identifier=" + content, log); } if (!(dcx_data.length() == 0)) { out.write(dcx_data); } else { // Should not do this!! out.write("<srw:record>"); out.write("<srw:recordPacking>xml</srw:recordPacking>"); out.write("<srw:recordSchema>info:srw/schema/1/dc-v1.1</srw:recordSchema>"); out.write( "<srw:recordData xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:mods=\"http://www.loc.gov/mods\" xmlns:dcterms=\"http://purl.org/dc/terms/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:dcx=\"http://krait.kb.nl/coop/tel/handbook/telterms.html\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:marcrel=\"http://www.loc.gov/loc.terms/relators/OTH\" >"); StringBuffer result = new StringBuffer(""); construct_lucene_dc(result, resultDoc); out.write(result.toString()); out.write("</srw:recordData>"); out.write("<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>"); out.write("</srw:record>"); } out.write("</srw_dc:dc>"); StringBuffer expand_data; boolean expand = false; if (content.startsWith("GGC-THES:AC:")) { String tmp_content = ""; tmp_content = content.replaceFirst("GGC-THES:AC:", ""); log.fatal("calling get"); expand_data = new StringBuffer( helpers.getExpand("http://www.kbresearch.nl/general/lod_new/get/" + tmp_content + "?format=rdf", log)); log.fatal("get finini"); if (expand_data.toString().length() > 4) { out.write( "<srw_dc:expand xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:expand=\"http://www.kbresearch.nl/expand\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" >"); out.write(expand_data.toString()); expand = true; } } else { expand_data = new StringBuffer(helpers .getExpand("http://www.kbresearch.nl/ANP.cgi?q=" + content, log)); if (expand_data.toString().length() > 0) { if (!expand) { out.write( "<srw_dc:expand xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:expand=\"http://www.kbresearch.nl/expand\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" >"); expand = true; } out.write(expand_data.toString()); } } if (expand) { out.write("</srw_dc:expand>"); } out.write("</srw:recordData>"); out.write("<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>"); out.write("</srw:record>"); } } } if ((do_err) || (numfound == 0)) { log.fatal("I haz suggestions"); try { spellq.setParam("spellcheck", true); spellq.setQueryType("/spell"); server = new CommonsHttpSolrServer(url); rsp = server.query(spellq); sdl = rsp.getResults(); SpellCheckResponse spell; spell = rsp.getSpellCheckResponse(); List<SpellCheckResponse.Suggestion> suggestions = spell.getSuggestions(); if (suggestions.isEmpty() == false) { suggest.append("<srw:extraResponseData>"); suggest.append("<suggestions>"); for (SpellCheckResponse.Suggestion sugg : suggestions) { suggest.append("<suggestionfor>" + sugg.getToken() + "</suggestionfor>"); for (String item : sugg.getSuggestions()) { suggest.append("<suggestion>" + item + "</suggestion>"); } suggest.append("</suggestions>"); suggest.append("</srw:extraResponseData>"); } do_sugg = true; } } catch (Exception e) { rsp = null; //log.fatal(e.toString()); } ; } ; if (!do_err) { if (facet != null) { try { fct = rsp.getFacetFields(); out.write("<srw:facets>"); for (String str : facet.split(",")) { out.write("<srw:facet>"); out.write("<srw:facetType>"); out.write(str); out.write("</srw:facetType>"); for (FacetField f : fct) { log.debug(f.getName()); //if (f.getName().equals(str+"_str") || (f.getName().equals(str+"_date")) ) { List<FacetField.Count> facetEnties = f.getValues(); for (FacetField.Count fcount : facetEnties) { out.write("<srw:facetValue>"); out.write("<srw:valueString>"); out.write(helpers.xmlEncode(fcount.getName())); out.write("</srw:valueString>"); out.write("<srw:count>"); out.write(Double.toString(fcount.getCount())); out.write("</srw:count>"); out.write("</srw:facetValue>"); // } } } out.write("</srw:facet>"); } out.write("</srw:facets>"); startRecord += 1; } catch (Exception e) { } //log.fatal(e.toString()); } } } else { out.write(diag); } out.write("</srw:records>"); // SearchRetrieve response footer String footer = this.SRW_FOOTER.replaceAll("\\$query", helpers.xmlEncode(query)); footer = footer.replaceAll("\\$startRecord", (startRecord).toString()); footer = footer.replaceAll("\\$maximumRecords", maximumRecords.toString()); footer = footer.replaceAll("\\$recordSchema", recordSchema); if (do_sugg) { out.write(suggest.toString()); } out.write(footer); } catch (MalformedURLException e) { out.write(e.getMessage()); } catch (IOException e) { out.write("TO ERR is Human"); } } } } out.close(); }
From source file:kesako.search.Search.java
License:Apache License
/** * Make a research. Return the status of the search and store the results in vDoc.<br> * The syntax of the different parameters is the SOLR syntax. * @param query query-string of the research * @param sortingString string to sort results. If sortingString is empty or null, the default sort order (by date) is done. * @param filter implementation of the facet mechanism * @param start The method doSearch return 20 results beginning from start to start+20 or nbFound if nbFound < start+20 * @return RESULTS if there is results, NO_RESULT if there is no result, RESULT_ERROR if an error occurs during the search *//* ww w. j a v a 2 s.com*/ public int doSearch(String query, String filter, String sortingString, int start) { NamedList<String> q = new NamedList<String>(); String sortingString2; if (query.trim().equals("") || query.trim().equalsIgnoreCase("AllDoc")) { q.add("q", "*:*"); } else { q.add("q", query); } q.add("fl", "*,score"); q.add("rows", "20"); q.add("start", Integer.toString(start)); q.add("hl", "on"); q.add("hl.fl", "text"); sortingString2 = sortingString.trim(); if (!sortingString2.equals("")) { q.add("sort", sortingString2); } else { q.add("sort", "score desc"); } if (!filter.trim().equals("")) { q.add("fq", filter.trim()); } /* * liste des champs afficher : * titre_f : arrayList * titre_doc :string * score : float * extract_date : date * content_type : arraylist * author :string */ logger.debug("query : " + q); QueryResponse r; SolrDocumentList listDoc; ResultDoc rDoc; try { r = SOLRUtilities.getSOLRServer().query(SolrParams.toSolrParams(q), METHOD.POST); listDoc = r.getResults(); nbFound = listDoc.getNumFound(); logger.debug("Nb Results : " + nbFound); Map<String, Map<String, List<String>>> listHighlight = r.getHighlighting(); vDoc.removeAllElements(); if (listDoc.size() > 0) { for (int i = 0; i < listDoc.size(); i++) { rDoc = new ResultDoc(listDoc.get(i)); if (listHighlight.get(rDoc.getFileURI()).get("text") != null) { logger.debug("Highlight : " + listHighlight.get(rDoc.getFileURI()).get("text").get(0)); rDoc.setHighlight(listHighlight.get(rDoc.getFileURI()).get("text").get(0)); } else { logger.debug("Highlight : No Highlight"); rDoc.setHighlight(""); } vDoc.add(rDoc); } } else { return NO_RESULT; } } catch (SolrServerException e) { logger.fatal("doSearch", e); return RESULT_ERROR; } return RESULTS; }
From source file:kesako.watcher.runnable.IndexFileProcess.java
License:Apache License
/** * Verify that all files in the index are referenced in the data-base. <br> * If a file in the index is not referenced in the data-base, it is suppressed from the index. *//*from ww w. ja v a 2 s.c om*/ private void cleanIndex(Connection cn, long start) { logger.debug("cleanIndex"); long nextStart = 0; long nbResults = 0; String query = "*:*"; String uri; NamedList<String> q = new NamedList<String>(); q.add("fl", "*"); q.add("q", query); q.add("rows", Integer.toString(this.nbFiles)); q.add("start", Long.toString(start)); logger.debug("query : " + q); QueryResponse r; SolrDocumentList listDoc; try { r = SOLRUtilities.getSOLRServer().query(SolrParams.toSolrParams(q)); listDoc = r.getResults(); nbResults = listDoc.getNumFound(); if (listDoc != null && listDoc.size() > 0) { nextStart = start; try { for (SolrDocument solrDoc : listDoc) { if (solrDoc.getFieldValue("file_uri") != null) { uri = solrDoc.getFieldValue("file_uri").toString(); if (DBUtilities.getFileId(cn, uri) < 0) { //suppression du fichier de l'index SOLRUtilities.getSOLRServer() .deleteByQuery("file_uri:\"" + uri.replace("\\", "\\\\") + "\""); } else { //on dcalle l'index du dbut de la prochaine recherche nextStart++; } } else { solrDoc.setField("file_uri", "FILEWATCHER_TO_DELETE"); //suppression du fichier de l'index SOLRUtilities.getSOLRServer().deleteByQuery("file_uri:\"FILEWATCHER_TO_DELETE\""); } } SOLRUtilities.getSOLRServer().commit(); SOLRUtilities.getSOLRServer().optimize(); SOLRUtilities.getSOLRServer().commit(); } catch (Exception e) { logger.fatal("cleanindex : ", e); } } } catch (SolrServerException e) { logger.fatal("cleanindex : ", e); } if (nextStart < nbResults) { cleanIndex(cn, nextStart); } logger.debug("END Clean Index"); }
From source file:net.cloudfree.apps.shop.internal.app.ListingServlet.java
License:Open Source License
@Override protected void doGet(final HttpServletRequest req, final HttpServletResponse resp) throws ServletException, IOException { final IListingManager manager = ModelUtil.getManager(IListingManager.class, getContext()); final ISolrQueryExecutor queryExecutor = (ISolrQueryExecutor) manager.getAdapter(ISolrQueryExecutor.class); if (null == queryExecutor) { resp.sendError(404);/*from w w w. j a v a2s . c o m*/ return; } final List<String> selectedFacets = new ArrayList<String>(); final SolrQuery query = new SolrQuery(); boolean facet = true; boolean checkVariations = false; final String path = req.getPathInfo(); if ((null != path) && (path.length() > 1)) { query.setFilterQueries(Document.URI_PATH + ":" + path.substring(1)); query.setFields("id", "title", "price", "name", "score", "img480", "uripath", "description"); facet = false; checkVariations = true; } else { final String q = req.getParameter("q"); if (StringUtils.isNotBlank(q)) { query.setQuery(q); } query.setFields("id", "title", "price", "name", "score", "img48", "uripath"); // ignore variations final String f = req.getParameter("f"); if (StringUtils.isNotBlank(f)) { query.addFilterQuery(f); } else { query.addFilterQuery("-type:variation"); } // facet narrowing? final String[] narrows = req.getParameterValues("narrow"); if (null != narrows) { for (final String narrow : narrows) { if (StringUtils.isBlank(narrow)) { continue; } final String[] split = StringUtils.split(narrow, ':'); if (split.length != 2) { continue; } final String name = split[0]; final String value = split[1]; if (StringUtils.isBlank(name) || StringUtils.isBlank(value)) { continue; } final FacetFilter filter = facetFilters.get(name); if (null != filter) { if (filter.addFilter(query, value)) { selectedFacets.add(name + ":" + value); } } } } } query.setQueryType("dismax"); query.set("debug", true); // facet fields if (facet) { query.setFacet(true); for (final FacetFilter filter : facetFilters.values()) { filter.defineFilter(query); } } final QueryResponse response = queryExecutor.query(query); final SolrDocumentList results = response.getResults(); resp.setContentType("text/html"); resp.setCharacterEncoding("UTF-8"); final PrintWriter writer = resp.getWriter(); writer.println("<html><head>"); writer.println("<title>"); writer.println(path); writer.println(" - Shop Listings</title>"); writer.println("</head><body>"); writer.println("<h1>Found Listings</h1>"); writer.println("<p>"); writer.println("CloudFree found <strong>" + results.getNumFound() + "</strong> products in " + (response.getQTime() < 1000 ? "less than a second." : (response.getQTime() + "ms."))); if (results.size() < results.getNumFound()) { writer.println("<br/>"); if (results.getStart() == 0) { writer.println("Only the first " + results.size() + " products are shown."); } else { writer.println("Only products " + results.getStart() + " till " + (results.getStart() + results.size()) + " will be shown."); } } writer.println("</p>"); final List<FacetField> facetFields = response.getFacetFields(); if ((null != facetFields) && !facetFields.isEmpty()) { writer.println("<p>"); writer.println("You can filter the results by: </br>"); for (final FacetField facetField : facetFields) { final List<Count> values = facetField.getValues(); if ((null != values) && !values.isEmpty()) { writer.println("<div style=\"float:left;\">"); writer.print("<em>"); writer.print(facetField.getName()); writer.print("</em>"); writer.println("<ul style=\"margin:0;\">"); int filters = 0; for (final Count count : values) { if (count.getCount() == 0) { continue; } writer.print("<li>"); writer.print(count.getName()); writer.print(" ("); writer.print(count.getCount()); writer.print(")"); writer.print("</li>"); filters++; } if (filters == 0) { writer.print("<li>none</li>"); } writer.println("</ul>"); writer.println("</div>"); } } writer.println("<div style=\"clear:both;\"> </div>"); writer.println("</p>"); } writer.println("<p>"); if (!results.isEmpty()) { for (final SolrDocument listing : results) { writeListing(listing, writer, req); if (checkVariations) { final SolrQuery query2 = new SolrQuery(); query2.setQuery("parentid:" + listing.getFirstValue("id")); query2.setFields("id", "title", "price", "name", "score", "img48", "uripath", "color", "size"); final QueryResponse response2 = queryExecutor.query(query2); final SolrDocumentList results2 = response2.getResults(); if ((null != results2) && !results2.isEmpty()) { writer.println("There are " + results2.size() + " variations available."); for (final SolrDocument variation : results2) { writeListing(variation, writer, req); } } } } } else { writer.println("No listings found!"); } writer.println("</p>"); writer.println("</body>"); }
From source file:net.hasor.search.server.rsf.service.SorlSearchService.java
License:Apache License
@Override public QuerySearchResult query(SearchQuery searchQuery) throws Throwable { SolrQuery solrQuery = new SolrQuery(); solrQuery.add(new MultiMapSolrParams(searchQuery.toMap())); QueryResponse response = getSolrClient().query(solrQuery); SolrDocumentList docList = response.getResults(); ////from w ww . jav a 2 s. c o m List<SearchDocument> documentList = new ArrayList<SearchDocument>(); if (docList != null) { for (SolrDocument solrDocument : docList) { SearchDocument document = convetTo(solrDocument); documentList.add(document); } } // QuerySearchResult searchResult = new QuerySearchResult(documentList); searchResult.setElapsedTime(response.getElapsedTime()); searchResult.setMaxScore(docList.getMaxScore()); searchResult.setNumFound(docList.getNumFound()); searchResult.setStart(docList.getStart()); searchResult.setStatus(response.getStatus()); searchResult.setQueryTime(response.getQTime()); return searchResult; }
From source file:net.yacy.cora.federate.FederateSearchManager.java
License:Open Source License
/** * Discover opensearch description links from local (embedded) Solr index * using meta data field 'outboundlinks_tag_txt' and add found systems to * the config file/*from w w w . j a va2 s .c o m*/ * * @return true if background discover job was started, false if job not * started */ public boolean discoverFromSolrIndex(Switchboard sb) { if (sb == null) { return false; } // check if needed Solr fields are available (selected) if (!sb.index.fulltext().useWebgraph()) { ConcurrentLog.severe("FederateSearchManager", "Error on connecting to embedded Solr webgraph index"); return false; } final SolrConnector connector = sb.index.fulltext().getWebgraphConnector(); final boolean metafieldavailable = sb.index.fulltext().getWebgraphConfiguration() .contains(WebgraphSchema.target_rel_s.name()) && (sb.index.fulltext().getWebgraphConfiguration().contains(WebgraphSchema.target_protocol_s.name()) && sb.index.fulltext().getWebgraphConfiguration() .contains(WebgraphSchema.target_urlstub_s.name())) && sb.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false); if (!metafieldavailable) { ConcurrentLog.warn("FederateSearchManager", "webgraph option and webgraph Schema fields target_rel_s, target_protocol_s and target_urlstub_s must be switched on"); return false; } // the solr search final String webgraphquerystr = WebgraphSchema.target_rel_s.getSolrFieldName() + ":search"; final String[] webgraphqueryfields = { WebgraphSchema.target_protocol_s.getSolrFieldName(), WebgraphSchema.target_urlstub_s.getSolrFieldName() }; // alternatively target_protocol_s + "://" +target_host_s + target_path_s final long numfound; try { SolrDocumentList docList = connector.getDocumentListByQuery(webgraphquerystr, null, 0, 1, webgraphqueryfields); numfound = docList.getNumFound(); if (numfound == 0) { ConcurrentLog.info("FederateSearchManager", "no results found, abort discover job"); return true; } ConcurrentLog.info("FederateSearchManager", "start checking " + Long.toString(numfound) + " found index results"); } catch (final IOException ex) { ConcurrentLog.logException(ex); return false; } final long stoptime = System.currentTimeMillis() + 1000 * 3600; // make sure job doesn't run forever // job to iterate through Solr index to find links to opensearchdescriptions // started as background job as connect timeouts may cause it run a long time final Thread job = new Thread() { @Override public void run() { try { boolean doloop = true; int loopnr = 0; Set<String> dblmem = new HashSet<String>(); // temp memory for already checked url while (doloop) { ConcurrentLog.info("FederateSearchManager", "start Solr query loop at " + Integer.toString(loopnr * 20) + " of " + Long.toString(numfound)); SolrDocumentList docList = connector.getDocumentListByQuery(webgraphquerystr, null, loopnr * 20, 20, webgraphqueryfields); // check chunk of 20 result documents loopnr++; if (stoptime < System.currentTimeMillis()) {// stop after max 1h doloop = false; ConcurrentLog.info("FederateSearchManager", "long running discover task aborted"); } if (docList != null && docList.size() > 0) { Iterator<SolrDocument> docidx = docList.iterator(); while (docidx.hasNext()) { SolrDocument sdoc = docidx.next(); String hrefurltxt = sdoc .getFieldValue(WebgraphSchema.target_protocol_s.getSolrFieldName()) + "://" + sdoc.getFieldValue(WebgraphSchema.target_urlstub_s.getSolrFieldName()); try { URL url = new URL(hrefurltxt); //TODO: check Blacklist if (dblmem.add(url.getAuthority())) { // use only main path to detect double entries opensearchdescriptionReader os = new opensearchdescriptionReader( hrefurltxt); if (os.getRSSorAtomUrl() != null) { // add found system to config file addOpenSearchTarget(os.getShortName(), os.getRSSorAtomUrl(), false, os.getItem("LongName")); ConcurrentLog.info("FederateSearchManager", "added " + os.getShortName() + " " + hrefurltxt); } else { ConcurrentLog.info("FederateSearchManager", "osd.xml check failed (no RSS or Atom support) for " + hrefurltxt); } } } catch (final MalformedURLException ex) { } } } else { doloop = false; } } ConcurrentLog.info("FederateSearchManager", "finisched Solr query (checked " + Integer.toString(dblmem.size()) + " unique opensearchdescription links found in " + Long.toString(numfound) + " results)"); } catch (final IOException ex) { ConcurrentLog.logException(ex); } } }; job.start(); return true; }
From source file:net.yacy.cora.federate.opensearch.OpenSearchConnector.java
License:Open Source License
/** * Discover opensearch description links from local (embedded) Solr index using * meta data field 'outboundlinks_tag_txt' and add found systems to the * config file/* w w w. j av a2 s . c o m*/ * * @return true if background discover job was started, false if job not started */ public boolean discoverFromSolrIndex(final Switchboard sb) { if (sb == null) { return false; } // check if needed Solr fields are available (selected) if (!sb.index.fulltext().useWebgraph()) { ConcurrentLog.severe("OpenSearchConnector.Discover", "Error on connecting to embedded Solr webgraph index"); return false; } final SolrConnector connector = sb.index.fulltext().getWebgraphConnector(); final boolean metafieldavailable = sb.index.fulltext().getWebgraphConfiguration() .contains(WebgraphSchema.target_rel_s.name()) && (sb.index.fulltext().getWebgraphConfiguration().contains(WebgraphSchema.target_protocol_s.name()) && sb.index.fulltext().getWebgraphConfiguration() .contains(WebgraphSchema.target_urlstub_s.name())) && sb.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false); if (!metafieldavailable) { ConcurrentLog.warn("OpenSearchConnector.Discover", "webgraph option and webgraph Schema fields target_rel_s, target_protocol_s and target_urlstub_s must be switched on"); return false; } // the solr query final String webgraphquerystr = WebgraphSchema.target_rel_s.getSolrFieldName() + ":search"; final String[] webgraphqueryfields = { WebgraphSchema.target_protocol_s.getSolrFieldName(), WebgraphSchema.target_urlstub_s.getSolrFieldName() }; // alternatively target_protocol_s + "://" +target_host_s + target_path_s final long numfound; try { SolrDocumentList docList = connector.getDocumentListByQuery(webgraphquerystr, null, 0, 1, webgraphqueryfields); numfound = docList.getNumFound(); if (numfound == 0) { ConcurrentLog.info("OpenSearchConnector.Discover", "no results found, abort discover job"); return true; } ConcurrentLog.info("OpenSearchConnector.Discover", "start checking " + Long.toString(numfound) + " found index results"); } catch (final IOException ex) { ConcurrentLog.logException(ex); return false; } final long stoptime = System.currentTimeMillis() + 1000 * 3600; // make sure job doesn't run forever // job to iterate through Solr index to find links to opensearchdescriptions // started as background job as connect timeouts may cause it run a long time final Thread job = new Thread() { @Override public void run() { try { boolean doloop = true; int loopnr = 0; Set<String> dblmem = new HashSet<String>(); // temp memory for already checked url while (doloop) { ConcurrentLog.info("OpenSearchConnector.Discover", "start Solr query loop at " + Integer.toString(loopnr * 20) + " of " + Long.toString(numfound)); SolrDocumentList docList = connector.getDocumentListByQuery(webgraphquerystr, null, loopnr * 20, 20, webgraphqueryfields); // check chunk of 20 result documents loopnr++; if (stoptime < System.currentTimeMillis()) {// stop after max 1h doloop = false; ConcurrentLog.info("OpenSearchConnector.Discover", "long running discover task aborted"); } if (docList != null && docList.size() > 0) { Iterator<SolrDocument> docidx = docList.iterator(); while (docidx.hasNext()) { SolrDocument sdoc = docidx.next(); String hrefurltxt = sdoc .getFieldValue(WebgraphSchema.target_protocol_s.getSolrFieldName()) + "://" + sdoc.getFieldValue(WebgraphSchema.target_urlstub_s.getSolrFieldName()); try { URL url = new URL(hrefurltxt); //TODO: check Blacklist if (dblmem.add(url.getAuthority())) { // use only main path to detect double entries opensearchdescriptionReader os = new opensearchdescriptionReader( hrefurltxt); if (os.getRSSorAtomUrl() != null) { // add found system to config file add(os.getShortName(), os.getRSSorAtomUrl(), false, os.getItem("LongName")); ConcurrentLog.info("OpenSearchConnector.Discover", "added " + os.getShortName() + " " + hrefurltxt); } else { ConcurrentLog.info("OpenSearchConnector.Discover", "osd.xml check failed (no RSS or Atom support) for " + hrefurltxt); } } } catch (final MalformedURLException ex) { } } } else { doloop = false; } } ConcurrentLog.info("OpenSearchConnector.Discover", "finisched Solr query (checked " + Integer.toString(dblmem.size()) + " unique opensearchdescription links found in " + Long.toString(numfound) + " results)"); } catch (final IOException ex) { ConcurrentLog.logException(ex); } } }; job.start(); return true; }