Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr; import java.io.File; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Random; import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.lucene.util.IOUtils; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.JettyConfig; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.NoOpResponseParser; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; import org.apache.solr.servlet.DirectSolrConnection; import org.noggit.JSONUtil; import org.noggit.ObjectBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @SolrTestCaseJ4.SuppressSSL //@LuceneTestCase.SuppressCodecs({"Lucene3x","Lucene40","Lucene41","Lucene42","Lucene45","Appending","Asserting"}) public class SolrTestCaseHS extends SolrTestCaseJ4 { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @SafeVarargs public static <T> Set<T> set(T... a) { LinkedHashSet<T> s = new LinkedHashSet<>(); for (T t : a) { s.add(t); } return s; } public static <T> T rand(T... vals) { return vals[random().nextInt(vals.length)]; } public static ModifiableSolrParams params(SolrParams params, String... moreParams) { ModifiableSolrParams msp = new ModifiableSolrParams(params); for (int i = 0; i < moreParams.length; i += 2) { msp.add(moreParams[i], moreParams[i + 1]); } return msp; } public static Map<String, Object> toObject(Doc doc, IndexSchema schema, Collection<String> fieldNames) { Map<String, Object> result = new HashMap<>(); for (Fld fld : doc.fields) { if (fieldNames != null && !fieldNames.contains(fld.ftype.fname)) continue; SchemaField sf = schema.getField(fld.ftype.fname); if (!sf.multiValued()) { result.put(fld.ftype.fname, fld.vals.get(0)); } else { result.put(fld.ftype.fname, fld.vals); } } return result; } public static Object createDocObjects(Map<Comparable, Doc> fullModel, Comparator sort, int rows, Collection<String> fieldNames) { List<Doc> docList = new ArrayList<>(fullModel.values()); Collections.sort(docList, sort); List sortedDocs = new ArrayList(rows); for (Doc doc : docList) { if (sortedDocs.size() >= rows) break; Map<String, Object> odoc = toObject(doc, h.getCore().getLatestSchema(), fieldNames); sortedDocs.add(toObject(doc, h.getCore().getLatestSchema(), fieldNames)); } return sortedDocs; } public static void compare(SolrQueryRequest req, String path, Object model, Map<Comparable, Doc> fullModel) throws Exception { String strResponse = h.query(req); Object realResponse = ObjectBuilder.fromJSON(strResponse); String err = JSONTestUtil.matchObj(path, realResponse, model); if (err != null) { log.error("RESPONSE MISMATCH: " + err + "\n\trequest=" + req + "\n\tresult=" + strResponse + "\n\texpected=" + JSONUtil.toJSON(model) + "\n\tmodel=" + fullModel); // re-execute the request... good for putting a breakpoint here for debugging String rsp = h.query(req); fail(err); } } /** Pass "null" for the client to query the local server */ public static void assertJQ(SolrClient client, SolrParams args, String... tests) throws Exception { String resp; resp = getJSON(client, args); matchJSON(resp, tests); } public static void matchJSON(String response, String... tests) throws Exception { boolean failed = false; for (String test : tests) { if (test == null || test.length() == 0) continue; try { failed = true; String err = JSONTestUtil.match(response, test, JSONTestUtil.DEFAULT_DELTA); failed = false; if (err != null) { log.error("query failed JSON validation. error=" + err + "\n expected =" + test + "\n response = " + response); throw new RuntimeException(err); } } finally { if (failed) { log.error("JSON query validation threw an exception." + "\n expected =" + test + "\n response = " + response); } } } } /*** public static void clearNCache() { SolrQueryRequest req = req(); req.getSearcher().getnCache().clear(); // OFF-HEAP req.close(); }***/ public static void clearQueryCache() { SolrQueryRequest req = req(); req.getSearcher(); req.close(); } public static String getQueryResponse(SolrClient client, String wt, SolrParams params) throws Exception { if (client == null) { return getQueryResponse(wt, params); } ModifiableSolrParams p = new ModifiableSolrParams(params); p.set("wt", wt); String path = p.get("qt"); p.remove("qt"); p.set("indent", "true"); QueryRequest query = new QueryRequest(p); if (path != null) { query.setPath(path); } query.setResponseParser(new NoOpResponseParser(wt)); NamedList<Object> rsp = client.request(query); String raw = (String) rsp.get("response"); return raw; } public static String getQueryResponse(String wt, SolrParams params) throws Exception { ModifiableSolrParams p = new ModifiableSolrParams(params); p.set("wt", wt); String path = p.get("qt"); p.remove("qt"); p.set("indent", "true"); DirectSolrConnection connection = new DirectSolrConnection(h.getCore()); String raw = connection.request(path, p, null); return raw; } public static String getJSON(SolrClient client, SolrParams params) throws Exception { return getQueryResponse(client, "json", params); } /** Adds a document using the specific client, or to the local test core if null. * Returns the version. TODO: work in progress... version not always returned. */ public static Long add(SolrClient client, SolrInputDocument sdoc, ModifiableSolrParams params) throws Exception { if (client == null) { Long version = addAndGetVersion(sdoc, params); return version; } else { UpdateRequest updateRequest = new UpdateRequest(); if (params != null) { updateRequest.setParams(params); } updateRequest.add(sdoc); UpdateResponse rsp = updateRequest.process(client); // TODO - return version return null; } } public static class Client { ClientProvider provider; ModifiableSolrParams queryDefaults; public Tester tester = new Tester(); public static class Tester { public void assertJQ(SolrClient client, SolrParams args, String... tests) throws Exception { SolrTestCaseHS.assertJQ(client, args, tests); } } public static Client localClient = new Client(null, 1); public static Client localClient() { return new Client(null, 1); } public Client(List<SolrClient> clients, int seed) { if (clients != null) { provider = new ClientProvider(clients, seed); } } public static int hash(int x) { // from Thomas Mueller x = ((x >>> 16) ^ x) * 0x45d9f3b; x = ((x >>> 16) ^ x) * 0x45d9f3b; x = ((x >>> 16) ^ x); return x; } public ModifiableSolrParams queryDefaults() { if (queryDefaults == null) { queryDefaults = new ModifiableSolrParams(); } return queryDefaults; } public boolean local() { return provider == null; } public ClientProvider getClientProvider() { return provider; } public void testJQ(SolrParams args, String... tests) throws Exception { if (queryDefaults != null) { ModifiableSolrParams newParams = params(queryDefaults); newParams.add(args); args = newParams; } SolrClient client = provider == null ? null : provider.client(null, args); tester.assertJQ(client, args, tests); } public Long add(SolrInputDocument sdoc, ModifiableSolrParams params) throws Exception { SolrClient client = provider == null ? null : provider.client(sdoc, params); return SolrTestCaseHS.add(client, sdoc, params); } public void commit() throws IOException, SolrServerException { if (local()) { assertU(SolrTestCaseJ4.commit()); return; } for (SolrClient client : provider.all()) { client.commit(); } } public void deleteByQuery(String query, ModifiableSolrParams params) throws IOException, SolrServerException { if (local()) { assertU(delQ(query)); // todo - handle extra params return; } for (SolrClient client : provider.all()) { client.deleteByQuery(query); // todo - handle extra params } } } public static class ClientProvider { public static String idField = "id"; List<SolrClient> clients; Random r; int hashSeed; // thisIsIgnored needed because we need a diff signature public ClientProvider(List<SolrClient> clients, int seed) { this.hashSeed = Client.hash(seed); this.clients = clients; r = new Random(seed); } public SolrClient client(SolrInputDocument sdoc, SolrParams params) { String idStr = null; if (sdoc != null) { idStr = sdoc.getFieldValue(idField).toString(); } else if (params != null) { idStr = params.get(idField); } int hash; if (idStr != null) { // make the client chosen the same for a duplicate ID hash = idStr.hashCode() ^ hashSeed; } else { hash = r.nextInt(); } return clients.get((hash & Integer.MAX_VALUE) % clients.size()); } public List<SolrClient> all() { return clients; } public int getSeed() { return hashSeed; } } // // Helper to run an internal Jetty instance. // Example: // SolrInstance s1 = new SolrInstance(createTempDir("s1"), "solrconfig-tlog.xml", "schema_latest.xml"); // s1.start(); // SolrClient c1 = s1.getSolrJ(); // assertJQ(c1, params("q", "*:*"), "/response/numFound==3"); // String json = getJSON(c1, params("q","id:1")); // s1.stop(); // // To manage multiple servers, see SolrInstances // public static class SolrInstance { private static Logger log = SolrTestCaseHS.log; private String collection = "collection1"; private int port = 0; private String solrconfigFile; private String schemaFile; private File baseDir; private JettySolrRunner jetty; private SolrClient solrj; private boolean homeCreated = false; public SolrInstance(File homeDir, String solrconfigFile, String schemaFile) { this.baseDir = homeDir; this.solrconfigFile = solrconfigFile; this.schemaFile = schemaFile; } public String getBaseDir() { return baseDir.toString(); } public String getBaseURL() { return (SolrTestCaseJ4.isSSLMode() ? "https" : "http") + "://127.0.0.1:" + port + "/solr"; } public String getCollectionURL() { return getBaseURL() + "/" + collection; } /** string appropriate for passing in shards param (i.e. missing http://) */ public String getShardURL() { return "127.0.0.1:" + port + "/solr" + "/" + collection; } public SolrClient getSolrJ() { if (solrj == null) { solrj = getHttpSolrClient(getCollectionURL()); } return solrj; } /** If it needs to change */ public void setPort(int port) { this.port = port; } public void createHome() throws Exception { homeCreated = true; SolrTestCaseJ4.copySolrHomeToTemp(baseDir, collection); copyConfFile(baseDir, collection, solrconfigFile); copyConfFile(baseDir, collection, schemaFile); File collDir = new File(baseDir, collection); try (Writer w = new OutputStreamWriter( Files.newOutputStream(collDir.toPath().resolve("core.properties")), StandardCharsets.UTF_8)) { Properties coreProps = new Properties(); coreProps.put("name", "collection1"); coreProps.put("config", solrconfigFile); coreProps.put("schema", schemaFile); coreProps.store(w, ""); } } public void start() throws Exception { if (!homeCreated) { createHome(); } if (jetty == null) { JettyConfig jettyConfig = JettyConfig.builder().stopAtShutdown(true).setContext("/solr") .setPort(port).withSSLConfig(sslConfig).build(); Properties nodeProperties = new Properties(); nodeProperties.setProperty("solrconfig", solrconfigFile); nodeProperties.setProperty(CoreDescriptor.CORE_SCHEMA, schemaFile); jetty = new JettySolrRunner(baseDir.getAbsolutePath(), nodeProperties, jettyConfig); } // silly stuff included from solrconfig.snippet.randomindexconfig.xml System.setProperty("solr.tests.maxBufferedDocs", String.valueOf(100000)); jetty.start(); port = jetty.getLocalPort(); log.info("===> Started solr server port=" + port + " home=" + getBaseDir()); } public void stop() throws Exception { jetty.stop(); if (solrj != null) solrj.close(); } public void tearDown() throws Exception { IOUtils.deleteFilesIfExist(baseDir.toPath()); } private static void copyConfFile(File dstRoot, String destCollection, String file) throws Exception { File subHome = new File(dstRoot, destCollection + File.separator + "conf"); String top = SolrTestCaseJ4.TEST_HOME() + "/collection1/conf"; FileUtils.copyFile(new File(top, file), new File(subHome, file)); } public void copyConfigFile(File dstRoot, String destCollection, String file) throws Exception { if (!homeCreated) { createHome(); } File subHome = new File(dstRoot, destCollection + File.separator + "conf"); String top = SolrTestCaseJ4.TEST_HOME() + "/collection1/conf"; FileUtils.copyFile(new File(top, file), new File(subHome, file)); } } // Manages a number of Solr servers and provides a Client to partition documents and randomly assign query requests. // Example: // SolrInstances servers = new SolrInstances(3, "solrconfig-tlog.xml","schema_latest.xml"); // Client = servers.getClient(0); // client.add(sdoc("id", "3"), null); // client.commit(); // client.testJQ(params("q", "*:*"), "/response/numFound==3") // servers.stop(); // public static class SolrInstances { public List<SolrInstance> slist; public Client client; public SolrInstances(int numServers, String solrconfig, String schema) throws Exception { slist = new ArrayList<>(numServers); for (int i = 0; i < numServers; i++) { SolrInstance instance = new SolrInstance(createTempDir("s" + i).toFile(), solrconfig, schema); slist.add(instance); instance.start(); } } public void stop() throws Exception { for (SolrInstance instance : slist) { instance.stop(); } } // For params.set("shards", getShards()) public String getShards() { return getShardsParam(slist); } public List<SolrClient> getSolrJs() { List<SolrClient> solrjs = new ArrayList<>(slist.size()); for (SolrInstance instance : slist) { solrjs.add(instance.getSolrJ()); } return solrjs; } public Client getClient(int seed) { if (client == null) { client = new Client(getSolrJs(), seed); } return client; } public static String getShardsParam(List<SolrInstance> instances) { StringBuilder sb = new StringBuilder(); boolean first = true; for (SolrInstance instance : instances) { if (first) { first = false; } else { sb.append(','); } sb.append(instance.getShardURL()); } return sb.toString(); } } }