Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.ltr; import java.io.File; import java.util.SortedMap; import org.apache.commons.io.FileUtils; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.embedded.JettyConfig; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.cloud.AbstractDistribZkTestBase; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.ltr.feature.SolrFeature; import org.apache.solr.ltr.feature.ValueFeature; import org.apache.solr.ltr.model.LinearModel; import org.eclipse.jetty.servlet.ServletHolder; import org.junit.AfterClass; import org.junit.Test; public class TestLTROnSolrCloud extends TestRerankBase { private MiniSolrCloudCluster solrCluster; String solrconfig = "solrconfig-ltr.xml"; String schema = "schema.xml"; SortedMap<ServletHolder, String> extraServlets = null; @Override public void setUp() throws Exception { super.setUp(); extraServlets = setupTestInit(solrconfig, schema, true); System.setProperty("enable.update.log", "true"); int numberOfShards = random().nextInt(4) + 1; int numberOfReplicas = random().nextInt(2) + 1; int maxShardsPerNode = numberOfShards + random().nextInt(4) + 1; int numberOfNodes = numberOfShards * maxShardsPerNode; setupSolrCluster(numberOfShards, numberOfReplicas, numberOfNodes, maxShardsPerNode); } @Override public void tearDown() throws Exception { restTestHarness.close(); restTestHarness = null; jetty.stop(); jetty = null; solrCluster.shutdown(); super.tearDown(); } @Test public void testSimpleQuery() throws Exception { // will randomly pick a configuration with [1..5] shards and [1..3] replicas // Test regular query, it will sort the documents by inverse // popularity (the less popular, docid == 1, will be in the first // position SolrQuery query = new SolrQuery("{!func}sub(8,field(popularity))"); query.setRequestHandler("/query"); query.setFields("*,score"); query.setParam("rows", "8"); QueryResponse queryResponse = solrCluster.getSolrClient().query(COLLECTION, query); assertEquals(8, queryResponse.getResults().getNumFound()); assertEquals("1", queryResponse.getResults().get(0).get("id").toString()); assertEquals("2", queryResponse.getResults().get(1).get("id").toString()); assertEquals("3", queryResponse.getResults().get(2).get("id").toString()); assertEquals("4", queryResponse.getResults().get(3).get("id").toString()); final String result0_features = FeatureLoggerTestUtils.toFeatureVector("powpularityS", "64.0", "c3", "2.0"); final String result1_features = FeatureLoggerTestUtils.toFeatureVector("powpularityS", "49.0", "c3", "2.0"); final String result2_features = FeatureLoggerTestUtils.toFeatureVector("powpularityS", "36.0", "c3", "2.0"); final String result3_features = FeatureLoggerTestUtils.toFeatureVector("powpularityS", "25.0", "c3", "2.0"); // Test re-rank and feature vectors returned query.setFields("*,score,features:[fv]"); query.add("rq", "{!ltr model=powpularityS-model reRankDocs=8}"); queryResponse = solrCluster.getSolrClient().query(COLLECTION, query); assertEquals(8, queryResponse.getResults().getNumFound()); assertEquals("8", queryResponse.getResults().get(0).get("id").toString()); assertEquals(result0_features, queryResponse.getResults().get(0).get("features").toString()); assertEquals("7", queryResponse.getResults().get(1).get("id").toString()); assertEquals(result1_features, queryResponse.getResults().get(1).get("features").toString()); assertEquals("6", queryResponse.getResults().get(2).get("id").toString()); assertEquals(result2_features, queryResponse.getResults().get(2).get("features").toString()); assertEquals("5", queryResponse.getResults().get(3).get("id").toString()); assertEquals(result3_features, queryResponse.getResults().get(3).get("features").toString()); } private void setupSolrCluster(int numShards, int numReplicas, int numServers, int maxShardsPerNode) throws Exception { JettyConfig jc = buildJettyConfig("/solr"); jc = JettyConfig.builder(jc).withServlets(extraServlets).build(); solrCluster = new MiniSolrCloudCluster(numServers, tmpSolrHome.toPath(), jc); File configDir = tmpSolrHome.toPath().resolve("collection1/conf").toFile(); solrCluster.uploadConfigSet(configDir.toPath(), "conf1"); solrCluster.getSolrClient().setDefaultCollection(COLLECTION); createCollection(COLLECTION, "conf1", numShards, numReplicas, maxShardsPerNode); indexDocuments(COLLECTION); createJettyAndHarness(tmpSolrHome.getAbsolutePath(), solrconfig, schema, "/solr", true, extraServlets); loadModelsAndFeatures(); } private void createCollection(String name, String config, int numShards, int numReplicas, int maxShardsPerNode) throws Exception { CollectionAdminResponse response; CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(name, config, numShards, numReplicas); create.setMaxShardsPerNode(maxShardsPerNode); response = create.process(solrCluster.getSolrClient()); if (response.getStatus() != 0 || response.getErrorMessages() != null) { fail("Could not create collection. Response" + response.toString()); } ZkStateReader zkStateReader = solrCluster.getSolrClient().getZkStateReader(); AbstractDistribZkTestBase.waitForRecoveriesToFinish(name, zkStateReader, false, true, 100); } void indexDocument(String collection, String id, String title, String description, int popularity) throws Exception { SolrInputDocument doc = new SolrInputDocument(); doc.setField("id", id); doc.setField("title", title); doc.setField("description", description); doc.setField("popularity", popularity); solrCluster.getSolrClient().add(collection, doc); } private void indexDocuments(final String collection) throws Exception { final int collectionSize = 8; for (int docId = 1; docId <= collectionSize; docId++) { final int popularity = docId; indexDocument(collection, String.valueOf(docId), "a1", "bloom", popularity); } solrCluster.getSolrClient().commit(collection); } private void loadModelsAndFeatures() throws Exception { final String featureStore = "test"; final String[] featureNames = new String[] { "powpularityS", "c3" }; final String jsonModelParams = "{\"weights\":{\"powpularityS\":1.0,\"c3\":1.0}}"; loadFeature(featureNames[0], SolrFeature.class.getCanonicalName(), featureStore, "{\"q\":\"{!func}pow(popularity,2)\"}"); loadFeature(featureNames[1], ValueFeature.class.getCanonicalName(), featureStore, "{\"value\":2}"); loadModel("powpularityS-model", LinearModel.class.getCanonicalName(), featureNames, featureStore, jsonModelParams); reloadCollection(COLLECTION); } private void reloadCollection(String collection) throws Exception { CollectionAdminRequest.Reload reloadRequest = CollectionAdminRequest.reloadCollection(collection); CollectionAdminResponse response = reloadRequest.process(solrCluster.getSolrClient()); assertEquals(0, response.getStatus()); assertTrue(response.isSuccess()); } @AfterClass public static void after() throws Exception { FileUtils.deleteDirectory(tmpSolrHome); System.clearProperty("managed.schema.mutable"); } }