Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.cloud; import java.lang.invoke.MethodHandles; import java.util.Arrays; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Random; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.math3.primes.Primes; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.zookeeper.KeeperException; import org.junit.After; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Slow public class TestStressInPlaceUpdates extends AbstractFullDistribZkTestBase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @BeforeClass public static void beforeSuperClass() throws Exception { System.setProperty("solr.tests.intClassName", random().nextBoolean() ? "TrieIntField" : "IntPointField"); System.setProperty("solr.tests.longClassName", random().nextBoolean() ? "TrieLongField" : "LongPointField"); System.setProperty("solr.tests.floatClassName", random().nextBoolean() ? "TrieFloatField" : "FloatPointField"); System.setProperty("solr.tests.doubleClassName", random().nextBoolean() ? "TrieDoubleField" : "DoublePointField"); schemaString = "schema-inplace-updates.xml"; configString = "solrconfig-tlog.xml"; // sanity check that autocommits are disabled initCore(configString, schemaString); assertEquals(-1, h.getCore().getSolrConfig().getUpdateHandlerInfo().autoCommmitMaxTime); assertEquals(-1, h.getCore().getSolrConfig().getUpdateHandlerInfo().autoSoftCommmitMaxTime); assertEquals(-1, h.getCore().getSolrConfig().getUpdateHandlerInfo().autoCommmitMaxDocs); assertEquals(-1, h.getCore().getSolrConfig().getUpdateHandlerInfo().autoSoftCommmitMaxDocs); } @After public void after() { System.clearProperty("solr.tests.intClassName"); System.clearProperty("solr.tests.longClassName"); System.clearProperty("solr.tests.floatClassName"); System.clearProperty("solr.tests.doubleClassName"); } public TestStressInPlaceUpdates() { super(); sliceCount = 1; fixShardCount(3); } protected final ConcurrentHashMap<Integer, DocInfo> model = new ConcurrentHashMap<>(); protected Map<Integer, DocInfo> committedModel = new HashMap<>(); protected long snapshotCount; protected long committedModelClock; protected int clientIndexUsedForCommit; protected volatile int lastId; protected final String field = "val_l"; private void initModel(int ndocs) { for (int i = 0; i < ndocs; i++) { // seed versions w/-1 so "from scratch" adds/updates will fail optimistic concurrency checks // if some other thread beats us to adding the id model.put(i, new DocInfo(-1L, 0, 0)); } committedModel.putAll(model); } SolrClient leaderClient = null; @Test @ShardsFixed(num = 3) public void stressTest() throws Exception { waitForRecoveriesToFinish(true); this.leaderClient = getClientForLeader(); assertNotNull("Couldn't obtain client for the leader of the shard", this.leaderClient); final int commitPercent = 5 + random().nextInt(20); final int softCommitPercent = 30 + random().nextInt(75); // what percent of the commits are soft final int deletePercent = 4 + random().nextInt(25); final int deleteByQueryPercent = random().nextInt(8); final int ndocs = atLeast(5); int nWriteThreads = 5 + random().nextInt(25); int fullUpdatePercent = 5 + random().nextInt(50); // query variables final int percentRealtimeQuery = 75; // number of cumulative read/write operations by all threads final AtomicLong operations = new AtomicLong(25000); int nReadThreads = 5 + random().nextInt(25); /** // testing final int commitPercent = 5; final int softCommitPercent = 100; // what percent of the commits are soft final int deletePercent = 0; final int deleteByQueryPercent = 50; final int ndocs = 10; int nWriteThreads = 10; final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers // query variables final int percentRealtimeQuery = 101; final AtomicLong operations = new AtomicLong(50000); // number of query operations to perform in total int nReadThreads = 10; int fullUpdatePercent = 20; **/ log.info("{}", Arrays.asList("commitPercent", commitPercent, "softCommitPercent", softCommitPercent, "deletePercent", deletePercent, "deleteByQueryPercent", deleteByQueryPercent, "ndocs", ndocs, "nWriteThreads", nWriteThreads, "percentRealtimeQuery", percentRealtimeQuery, "operations", operations, "nReadThreads", nReadThreads)); initModel(ndocs); List<Thread> threads = new ArrayList<>(); for (int i = 0; i < nWriteThreads; i++) { Thread thread = new Thread("WRITER" + i) { Random rand = new Random(random().nextInt()); @Override public void run() { try { while (operations.decrementAndGet() > 0) { int oper = rand.nextInt(100); if (oper < commitPercent) { Map<Integer, DocInfo> newCommittedModel; long version; synchronized (TestStressInPlaceUpdates.this) { // take a snapshot of the model // this is safe to do w/o synchronizing on the model because it's a ConcurrentHashMap newCommittedModel = new HashMap<>(model); version = snapshotCount++; int chosenClientIndex = rand.nextInt(clients.size()); if (rand.nextInt(100) < softCommitPercent) { log.info("softCommit start"); clients.get(chosenClientIndex).commit(true, true, true); log.info("softCommit end"); } else { log.info("hardCommit start"); clients.get(chosenClientIndex).commit(); log.info("hardCommit end"); } // install this model snapshot only if it's newer than the current one if (version >= committedModelClock) { if (VERBOSE) { log.info("installing new committedModel version={}", committedModelClock); } clientIndexUsedForCommit = chosenClientIndex; committedModel = newCommittedModel; committedModelClock = version; } } continue; } int id; if (rand.nextBoolean()) { id = rand.nextInt(ndocs); } else { id = lastId; // reuse the last ID half of the time to force more race conditions } // set the lastId before we actually change it sometimes to try and // uncover more race conditions between writing and reading boolean before = rand.nextBoolean(); if (before) { lastId = id; } DocInfo info = model.get(id); // yield after getting the next version to increase the odds of updates happening out of order if (rand.nextBoolean()) Thread.yield(); if (oper < commitPercent + deletePercent + deleteByQueryPercent) { final boolean dbq = (oper >= commitPercent + deletePercent); final String delType = dbq ? "DBI" : "DBQ"; log.info("{} id {}: {}", delType, id, info); Long returnedVersion = null; try { returnedVersion = deleteDocAndGetVersion(Integer.toString(id), params("_version_", Long.toString(info.version)), dbq); log.info(delType + ": Deleting id=" + id + ", version=" + info.version + ". Returned version=" + returnedVersion); } catch (RuntimeException e) { if (e.getMessage() != null && e.getMessage().contains("version conflict") || e.getMessage() != null && e.getMessage().contains("Conflict")) { // Its okay for a leader to reject a concurrent request log.warn("Conflict during {}, rejected id={}, {}", delType, id, e); returnedVersion = null; } else { throw e; } } // only update model if update had no conflict & the version is newer synchronized (model) { DocInfo currInfo = model.get(id); if (null != returnedVersion && (Math.abs(returnedVersion.longValue()) > Math .abs(currInfo.version))) { model.put(id, new DocInfo(returnedVersion.longValue(), 0, 0)); } } } else { int val1 = info.intFieldValue; long val2 = info.longFieldValue; int nextVal1 = val1; long nextVal2 = val2; int addOper = rand.nextInt(100); Long returnedVersion; if (addOper < fullUpdatePercent || info.version <= 0) { // if document was never indexed or was deleted // FULL UPDATE nextVal1 = Primes.nextPrime(val1 + 1); nextVal2 = nextVal1 * 1000000000l; try { returnedVersion = addDocAndGetVersion("id", id, "title_s", "title" + id, "val1_i_dvo", nextVal1, "val2_l_dvo", nextVal2, "_version_", info.version); log.info("FULL: Writing id=" + id + ", val=[" + nextVal1 + "," + nextVal2 + "], version=" + info.version + ", Prev was=[" + val1 + "," + val2 + "]. Returned version=" + returnedVersion); } catch (RuntimeException e) { if (e.getMessage() != null && e.getMessage().contains("version conflict") || e.getMessage() != null && e.getMessage().contains("Conflict")) { // Its okay for a leader to reject a concurrent request log.warn("Conflict during full update, rejected id={}, {}", id, e); returnedVersion = null; } else { throw e; } } } else { // PARTIAL nextVal2 = val2 + val1; try { returnedVersion = addDocAndGetVersion("id", id, "val2_l_dvo", map("inc", String.valueOf(val1)), "_version_", info.version); log.info("PARTIAL: Writing id=" + id + ", val=[" + nextVal1 + "," + nextVal2 + "], version=" + info.version + ", Prev was=[" + val1 + "," + val2 + "]. Returned version=" + returnedVersion); } catch (RuntimeException e) { if (e.getMessage() != null && e.getMessage().contains("version conflict") || e.getMessage() != null && e.getMessage().contains("Conflict")) { // Its okay for a leader to reject a concurrent request log.warn("Conflict during partial update, rejected id={}, {}", id, e); } else if (e.getMessage() != null && e.getMessage().contains("Document not found for update.") && e.getMessage().contains("id=" + id)) { log.warn( "Attempted a partial update for a recently deleted document, rejected id={}, {}", id, e); } else { throw e; } returnedVersion = null; } } // only update model if update had no conflict & the version is newer synchronized (model) { DocInfo currInfo = model.get(id); if (null != returnedVersion && (Math.abs(returnedVersion.longValue()) > Math .abs(currInfo.version))) { model.put(id, new DocInfo(returnedVersion.longValue(), nextVal1, nextVal2)); } } } if (!before) { lastId = id; } } } catch (Throwable e) { operations.set(-1L); log.error("", e); throw new RuntimeException(e); } } }; threads.add(thread); } // Read threads for (int i = 0; i < nReadThreads; i++) { Thread thread = new Thread("READER" + i) { Random rand = new Random(random().nextInt()); @SuppressWarnings("unchecked") @Override public void run() { try { while (operations.decrementAndGet() >= 0) { // bias toward a recently changed doc int id = rand.nextInt(100) < 25 ? lastId : rand.nextInt(ndocs); // when indexing, we update the index, then the model // so when querying, we should first check the model, and then the index boolean realTime = rand.nextInt(100) < percentRealtimeQuery; DocInfo expected; if (realTime) { expected = model.get(id); } else { synchronized (TestStressInPlaceUpdates.this) { expected = committedModel.get(id); } } if (VERBOSE) { log.info("querying id {}", id); } ModifiableSolrParams params = new ModifiableSolrParams(); if (realTime) { params.set("wt", "json"); params.set("qt", "/get"); params.set("ids", Integer.toString(id)); } else { params.set("wt", "json"); params.set("q", "id:" + Integer.toString(id)); params.set("omitHeader", "true"); } int clientId = rand.nextInt(clients.size()); if (!realTime) clientId = clientIndexUsedForCommit; QueryResponse response = clients.get(clientId).query(params); if (response.getResults().size() == 0) { // there's no info we can get back with a delete, so not much we can check without further synchronization } else if (response.getResults().size() == 1) { final SolrDocument actual = response.getResults().get(0); final String msg = "Realtime=" + realTime + ", expected=" + expected + ", actual=" + actual; assertNotNull(msg, actual); final Long foundVersion = (Long) actual.getFieldValue("_version_"); assertNotNull(msg, foundVersion); assertTrue(msg + "... solr doc has non-positive version???", 0 < foundVersion.longValue()); final Integer intVal = (Integer) actual.getFieldValue("val1_i_dvo"); assertNotNull(msg, intVal); final Long longVal = (Long) actual.getFieldValue("val2_l_dvo"); assertNotNull(msg, longVal); assertTrue(msg + " ...solr returned older version then model. " + "should not be possible given the order of operations in writer threads", Math.abs(expected.version) <= foundVersion.longValue()); if (foundVersion.longValue() == expected.version) { assertEquals(msg, expected.intFieldValue, intVal.intValue()); assertEquals(msg, expected.longFieldValue, longVal.longValue()); } // Some things we can assert about any Doc returned from solr, // even if it's newer then our (expected) model information... assertTrue(msg + " ...how did a doc in solr get a non positive intVal?", 0 < intVal); assertTrue(msg + " ...how did a doc in solr get a non positive longVal?", 0 < longVal); assertEquals(msg + " ...intVal and longVal in solr doc are internally (modulo) inconsistent w/eachother", 0, (longVal % intVal)); // NOTE: when foundVersion is greater then the version read from the model, // it's not possible to make any assertions about the field values in solr relative to the // field values in the model -- ie: we can *NOT* assert expected.longFieldVal <= doc.longVal // // it's tempting to think that this would be possible if we changed our model to preserve the // "old" valuess when doing a delete, but that's still no garuntee because of how oportunistic // concurrency works with negative versions: When adding a doc, we can assert that it must not // exist with version<0, but we can't assert that the *reason* it doesn't exist was because of // a delete with the specific version of "-42". // So a wrtier thread might (1) prep to add a doc for the first time with "intValue=1,_version_=-1", // and that add may succeed and (2) return some version X which is put in the model. but // inbetween #1 and #2 other threads may have added & deleted the doc repeatedly, updating // the model with intValue=7,_version_=-42, and a reader thread might meanwhile read from the // model before #2 and expect intValue=5, but get intValue=1 from solr (with a greater version) } else { fail(String.format(Locale.ENGLISH, "There were more than one result: {}", response)); } } } catch (Throwable e) { operations.set(-1L); log.error("", e); throw new RuntimeException(e); } } }; threads.add(thread); } // Start all threads for (Thread thread : threads) { thread.start(); } for (Thread thread : threads) { thread.join(); } { // final pass over uncommitted model with RTG for (SolrClient client : clients) { for (Map.Entry<Integer, DocInfo> entry : model.entrySet()) { final Integer id = entry.getKey(); final DocInfo expected = entry.getValue(); final SolrDocument actual = client.getById(id.toString()); String msg = "RTG: " + id + "=" + expected; if (null == actual) { // a deleted or non-existent document // sanity check of the model agrees... assertTrue(msg + " is deleted/non-existent in Solr, but model has non-neg version", expected.version < 0); assertEquals(msg + " is deleted/non-existent in Solr", expected.intFieldValue, 0); assertEquals(msg + " is deleted/non-existent in Solr", expected.longFieldValue, 0); } else { msg = msg + " <==VS==> " + actual; assertEquals(msg, expected.intFieldValue, actual.getFieldValue("val1_i_dvo")); assertEquals(msg, expected.longFieldValue, actual.getFieldValue("val2_l_dvo")); assertEquals(msg, expected.version, actual.getFieldValue("_version_")); assertTrue(msg + " doc exists in solr, but version is negative???", 0 < expected.version); } } } } { // do a final search and compare every result with the model // because commits don't provide any sort of concrete versioning (or optimistic concurrency constraints) // there's no way to garuntee that our committedModel matches what was in Solr at the time of the last commit. // It's possible other threads made additional writes to solr before the commit was processed, but after // the committedModel variable was assigned it's new value. // // what we can do however, is commit all completed updates, and *then* compare solr search results // against the (new) committed model.... waitForThingsToLevelOut(30); // NOTE: this does an automatic commit for us & ensures replicas are up to date committedModel = new HashMap<>(model); // first, prune the model of any docs that have negative versions // ie: were never actually added, or were ultimately deleted. for (int i = 0; i < ndocs; i++) { DocInfo info = committedModel.get(i); if (info.version < 0) { // first, a quick sanity check of the model itself... assertEquals("Inconsistent int value in model for deleted doc" + i + "=" + info, 0, info.intFieldValue); assertEquals("Inconsistent long value in model for deleted doc" + i + "=" + info, 0L, info.longFieldValue); committedModel.remove(i); } } for (SolrClient client : clients) { QueryResponse rsp = client.query(params("q", "*:*", "sort", "id asc", "rows", ndocs + "")); for (SolrDocument actual : rsp.getResults()) { final Integer id = Integer.parseInt(actual.getFieldValue("id").toString()); final DocInfo expected = committedModel.get(id); assertNotNull("Doc found but missing/deleted from model: " + actual, expected); final String msg = "Search: " + id + "=" + expected + " <==VS==> " + actual; assertEquals(msg, expected.intFieldValue, actual.getFieldValue("val1_i_dvo")); assertEquals(msg, expected.longFieldValue, actual.getFieldValue("val2_l_dvo")); assertEquals(msg, expected.version, actual.getFieldValue("_version_")); assertTrue(msg + " doc exists in solr, but version is negative???", 0 < expected.version); // also sanity check the model (which we already know matches the doc) assertEquals("Inconsistent (modulo) values in model for id " + id + "=" + expected, 0, (expected.longFieldValue % expected.intFieldValue)); } assertEquals(committedModel.size(), rsp.getResults().getNumFound()); } } } /** * Used for storing the info for a document in an in-memory model. */ private static class DocInfo { long version; int intFieldValue; long longFieldValue; public DocInfo(long version, int val1, long val2) { assert version != 0; // must either be real positive version, or negative deleted version/indicator this.version = version; this.intFieldValue = val1; this.longFieldValue = val2; } @Override public String toString() { return "[version=" + version + ", intValue=" + intFieldValue + ",longValue=" + longFieldValue + "]"; } } @SuppressWarnings("rawtypes") protected long addDocAndGetVersion(Object... fields) throws Exception { SolrInputDocument doc = new SolrInputDocument(); addFields(doc, fields); ModifiableSolrParams params = new ModifiableSolrParams(); params.add("versions", "true"); UpdateRequest ureq = new UpdateRequest(); ureq.setParams(params); ureq.add(doc); UpdateResponse resp; // send updates to leader, to avoid SOLR-8733 resp = ureq.process(leaderClient); long returnedVersion = Long.parseLong(((NamedList) resp.getResponse().get("adds")).getVal(0).toString()); assertTrue("Due to SOLR-8733, sometimes returned version is 0. Let us assert that we have successfully" + " worked around that problem here.", returnedVersion > 0); return returnedVersion; } @SuppressWarnings("rawtypes") protected long deleteDocAndGetVersion(String id, ModifiableSolrParams params, boolean deleteByQuery) throws Exception { params.add("versions", "true"); UpdateRequest ureq = new UpdateRequest(); ureq.setParams(params); if (deleteByQuery) { ureq.deleteByQuery("id:" + id); } else { ureq.deleteById(id); } UpdateResponse resp; // send updates to leader, to avoid SOLR-8733 resp = ureq.process(leaderClient); String key = deleteByQuery ? "deleteByQuery" : "deletes"; long returnedVersion = Long.parseLong(((NamedList) resp.getResponse().get(key)).getVal(0).toString()); assertTrue("Due to SOLR-8733, sometimes returned version is 0. Let us assert that we have successfully" + " worked around that problem here.", returnedVersion < 0); return returnedVersion; } /** * Method gets the SolrClient for the leader replica. This is needed for a workaround for SOLR-8733. */ public SolrClient getClientForLeader() throws KeeperException, InterruptedException { ZkStateReader zkStateReader = cloudClient.getZkStateReader(); cloudClient.getZkStateReader().forceUpdateCollection(DEFAULT_COLLECTION); ClusterState clusterState = cloudClient.getZkStateReader().getClusterState(); Replica leader = null; Slice shard1 = clusterState.getCollection(DEFAULT_COLLECTION).getSlice(SHARD1); leader = shard1.getLeader(); for (int i = 0; i < clients.size(); i++) { String leaderBaseUrl = zkStateReader.getBaseUrlForNodeName(leader.getNodeName()); if (((HttpSolrClient) clients.get(i)).getBaseURL().startsWith(leaderBaseUrl)) return clients.get(i); } return null; } }