Java tutorial
/** * Copyright (c) 2016, All Contributors (see CONTRIBUTORS file) * * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ package com.eventsourcing.index; import com.eventsourcing.Entity; import com.eventsourcing.EntityHandle; import com.eventsourcing.ResolvedEntityHandle; import com.eventsourcing.StandardEntity; import com.eventsourcing.hlc.HybridTimestamp; import com.eventsourcing.hlc.NTPServerTimeProvider; import com.eventsourcing.models.Car; import com.eventsourcing.models.CarFactory; import com.eventsourcing.queries.Max; import com.eventsourcing.queries.Min; import com.google.common.collect.Lists; import com.googlecode.cqengine.ConcurrentIndexedCollection; import com.googlecode.cqengine.IndexedCollection; import com.googlecode.cqengine.index.AttributeIndex; import com.googlecode.cqengine.index.Index; import com.googlecode.cqengine.index.support.KeyStatistics; import com.googlecode.cqengine.index.support.KeyStatisticsIndex; import com.googlecode.cqengine.index.support.SortedKeyStatisticsIndex; import com.googlecode.cqengine.quantizer.IntegerQuantizer; import com.googlecode.cqengine.quantizer.Quantizer; import com.googlecode.cqengine.query.Query; import com.googlecode.cqengine.query.option.QueryOptions; import com.googlecode.cqengine.resultset.ResultSet; import lombok.SneakyThrows; import org.apache.commons.net.ntp.TimeStamp; import org.testng.annotations.Test; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.function.Function; import static com.eventsourcing.queries.QueryFactory.max; import static com.eventsourcing.queries.QueryFactory.min; import static com.eventsourcing.queries.QueryFactory.scoped; import static com.googlecode.cqengine.query.QueryFactory.*; import static java.util.Arrays.asList; import static org.testng.Assert.*; // This test was originally copied from CQEngine in order to test // indices the same way CQEngine does. public abstract class NavigableIndexTest<NavigableIndex extends AttributeIndex & SortedKeyStatisticsIndex> { public abstract <A extends Comparable<A>, O extends Entity> NavigableIndex onAttribute( Attribute<O, A> attribute); public abstract <A extends Comparable<A>, O extends Entity> Index<EntityHandle<O>> withQuantizerOnAttribute( Quantizer<A> quantizer, Attribute<O, A> attribute); public static <A> Set<A> setOf(A... values) { return new LinkedHashSet<>(Arrays.asList(values)); } public static <A> Set<A> setOf(Iterable<A> values) { Set<A> result = new LinkedHashSet<>(); for (A value : values) { result.add(value); } return result; } @Test public void getDistinctKeysAndCounts() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); SortedKeyStatisticsIndex<String, EntityHandle<Car>> MODEL_INDEX = onAttribute(Car.MODEL); MODEL_INDEX.clear(noQueryOptions()); collection.addIndex(MODEL_INDEX); collection.addAll(CarFactory.createCollectionOfCars(20)); Set<String> distinctModels = setOf(MODEL_INDEX.getDistinctKeys(noQueryOptions())); assertEquals(new ArrayList<>(distinctModels), asList("Accord", "Avensis", "Civic", "Focus", "Fusion", "Hilux", "Insight", "M6", "Prius", "Taurus")); for (String model : distinctModels) { assertEquals(MODEL_INDEX.getCountForKey(model, noQueryOptions()), Integer.valueOf(2)); } Set<String> distinctModelsDescending = setOf(MODEL_INDEX.getDistinctKeysDescending(noQueryOptions())); assertEquals(new ArrayList<>(distinctModelsDescending), asList("Taurus", "Prius", "M6", "Insight", "Hilux", "Fusion", "Focus", "Civic", "Avensis", "Accord")); } @Test public void getCountOfDistinctKeys() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); KeyStatisticsIndex<String, EntityHandle<Car>> MANUFACTURER_INDEX = onAttribute(Car.MANUFACTURER); MANUFACTURER_INDEX.clear(noQueryOptions()); collection.addIndex(MANUFACTURER_INDEX); collection.addAll(CarFactory.createCollectionOfCars(20)); assertEquals(MANUFACTURER_INDEX.getCountOfDistinctKeys(noQueryOptions()), Integer.valueOf(4)); } @Test public void getStatisticsForDistinctKeys() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); KeyStatisticsIndex<String, EntityHandle<Car>> MANUFACTURER_INDEX = onAttribute(Car.MANUFACTURER); MANUFACTURER_INDEX.clear(noQueryOptions()); collection.addIndex(MANUFACTURER_INDEX); collection.addAll(CarFactory.createCollectionOfCars(20)); Set<KeyStatistics<String>> keyStatistics = setOf( MANUFACTURER_INDEX.getStatisticsForDistinctKeys(noQueryOptions())); assertEquals(keyStatistics, setOf(new KeyStatistics<>("Ford", 6), new KeyStatistics<>("Honda", 6), new KeyStatistics<>("Toyota", 6), new KeyStatistics<>("BMW", 2) )); } @Test public void getStatisticsForDistinctKeysDescending() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); SortedKeyStatisticsIndex<String, EntityHandle<Car>> MANUFACTURER_INDEX = onAttribute(Car.MANUFACTURER); MANUFACTURER_INDEX.clear(noQueryOptions()); collection.addIndex(MANUFACTURER_INDEX); collection.addAll(CarFactory.createCollectionOfCars(20)); Set<KeyStatistics<String>> keyStatistics = setOf( MANUFACTURER_INDEX.getStatisticsForDistinctKeysDescending(noQueryOptions())); assertEquals(keyStatistics, setOf(new KeyStatistics<>("Toyota", 6), new KeyStatistics<>("Honda", 6), new KeyStatistics<>("Ford", 6), new KeyStatistics<>("BMW", 2) )); } @Test public void retrieveLess() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); SortedKeyStatisticsIndex<String, EntityHandle<Car>> PRICE_INDEX = onAttribute(Car.PRICE); PRICE_INDEX.clear(noQueryOptions()); collection.addIndex(PRICE_INDEX); collection.addAll(CarFactory.createCollectionOfCars(10)); try (ResultSet<EntityHandle<Car>> resultSet = collection.retrieve(lessThan(Car.PRICE, 3999.99))) { assertEquals(resultSet.size(), 1); assertEquals(resultSet.uniqueResult().get().getModel(), "Accord"); } try (ResultSet<EntityHandle<Car>> resultSet = collection.retrieve(lessThanOrEqualTo(Car.PRICE, 3999.99))) { assertEquals(resultSet.size(), 2); ArrayList<EntityHandle<Car>> values = Lists.newArrayList(resultSet.iterator()); assertTrue(values.stream().anyMatch(h -> h.get().getModel().contentEquals("Fusion"))); assertTrue(values.stream().anyMatch(h -> h.get().getModel().contentEquals("Accord"))); } } @Test public void retrieveGreater() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); SortedKeyStatisticsIndex<String, EntityHandle<Car>> PRICE_INDEX = onAttribute(Car.PRICE); PRICE_INDEX.clear(noQueryOptions()); collection.addIndex(PRICE_INDEX); collection.addAll(CarFactory.createCollectionOfCars(10)); try (ResultSet<EntityHandle<Car>> resultSet = collection.retrieve(greaterThan(Car.PRICE, 8500.00))) { assertEquals(resultSet.size(), 1); assertEquals(resultSet.uniqueResult().get().getModel(), "M6"); } try (ResultSet<EntityHandle<Car>> resultSet = collection .retrieve(greaterThanOrEqualTo(Car.PRICE, 8500.00))) { assertEquals(resultSet.size(), 2); ArrayList<EntityHandle<Car>> values = Lists.newArrayList(resultSet.iterator()); assertTrue(values.stream().anyMatch(h -> h.get().getModel().contentEquals("M6"))); assertTrue(values.stream().anyMatch(h -> h.get().getModel().contentEquals("Prius"))); } } @Test public void retrieveBetween() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); SortedKeyStatisticsIndex<String, EntityHandle<Car>> PRICE_INDEX = onAttribute(Car.PRICE); PRICE_INDEX.clear(noQueryOptions()); collection.addIndex(PRICE_INDEX); collection.addAll(CarFactory.createCollectionOfCars(10)); try (ResultSet<EntityHandle<Car>> resultSet = collection.retrieve(between(Car.PRICE, 8000.00, 9001.00))) { assertEquals(resultSet.size(), 2); ArrayList<EntityHandle<Car>> values = Lists.newArrayList(resultSet.iterator()); assertTrue(values.stream().anyMatch(h -> h.get().getModel().contentEquals("M6"))); assertTrue(values.stream().anyMatch(h -> h.get().getModel().contentEquals("Prius"))); } } @Test public void indexQuantization_SpanningTwoBucketsMidRange() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); Index<EntityHandle<Car>> index = withQuantizerOnAttribute(IntegerQuantizer.withCompressionFactor(10), Car.CAR_ID); index.clear(noQueryOptions()); collection.addIndex(index); collection.addAll(CarFactory.createCollectionOfCars(100)); // Merge cost should be 20 because this query spans 2 buckets (each containing 10 objects)... ResultSet<EntityHandle<Car>> resultSet = collection.retrieve(between(Car.CAR_ID, 47, 53)); assertEquals(resultSet.getMergeCost(), 20); resultSet.close(); // 7 objects match the query (between is inclusive)... resultSet = collection.retrieve(between(Car.CAR_ID, 47, 53)); assertEquals(resultSet.size(), 7); resultSet.close(); // The matching objects are... List<Integer> carIdsFound = retrieveCarIds(collection, between(Car.CAR_ID, 47, 53)); assertEquals(carIdsFound, asList(47, 48, 49, 50, 51, 52, 53)); } @Test public void indexQuantization_FirstBucket() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); Index<EntityHandle<Car>> index = withQuantizerOnAttribute(IntegerQuantizer.withCompressionFactor(10), Car.CAR_ID); index.clear(noQueryOptions()); collection.addIndex(index); collection.addAll(CarFactory.createCollectionOfCars(100)); // Merge cost should be 10, because objects matching this query are in a single bucket... ResultSet<EntityHandle<Car>> resultSet = collection.retrieve(between(Car.CAR_ID, 2, 4)); assertEquals(resultSet.getMergeCost(), 10); resultSet.close(); // 3 objects match the query... resultSet = collection.retrieve(between(Car.CAR_ID, 2, 4)); assertEquals(resultSet.size(), 3); resultSet.close(); List<Integer> carIdsFound = retrieveCarIds(collection, between(Car.CAR_ID, 2, 4)); assertEquals(carIdsFound, asList(2, 3, 4)); } @Test public void indexQuantization_LastBucket() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); Index<EntityHandle<Car>> index = withQuantizerOnAttribute(IntegerQuantizer.withCompressionFactor(10), Car.CAR_ID); index.clear(noQueryOptions()); collection.addIndex(index); collection.addAll(CarFactory.createCollectionOfCars(100)); // Merge cost should be 10, because objects matching this query are in a single bucket... ResultSet<EntityHandle<Car>> resultSet = collection.retrieve(between(Car.CAR_ID, 96, 98)); assertEquals(resultSet.getMergeCost(), 10); resultSet.close(); // 3 objects match the query... resultSet = collection.retrieve(between(Car.CAR_ID, 96, 98)); assertEquals(resultSet.size(), 3); resultSet.close(); List<Integer> carIdsFound = retrieveCarIds(collection, between(Car.CAR_ID, 96, 98)); assertEquals(carIdsFound, asList(96, 97, 98)); } @Test public void indexQuantization_LessThan() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); Index<EntityHandle<Car>> index = withQuantizerOnAttribute(IntegerQuantizer.withCompressionFactor(10), Car.CAR_ID); index.clear(noQueryOptions()); collection.addIndex(index); collection.addAll(CarFactory.createCollectionOfCars(100)); assertEquals(collection.retrieve(lessThan(Car.CAR_ID, 5)).size(), 5); assertEquals(collection.retrieve(lessThan(Car.CAR_ID, 15)).size(), 15); assertEquals(collection.retrieve(lessThanOrEqualTo(Car.CAR_ID, 5)).size(), 6); assertEquals(collection.retrieve(lessThanOrEqualTo(Car.CAR_ID, 15)).size(), 16); } @Test public void indexQuantization_GreaterThan() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); Index<EntityHandle<Car>> index = withQuantizerOnAttribute(IntegerQuantizer.withCompressionFactor(10), Car.CAR_ID); index.clear(noQueryOptions()); collection.addIndex(index); collection.addAll(CarFactory.createCollectionOfCars(100)); ResultSet<EntityHandle<Car>> resultSet; resultSet = collection.retrieve(greaterThan(Car.CAR_ID, 95)); assertEquals(resultSet.size(), 4); resultSet.close(); resultSet = collection.retrieve(greaterThan(Car.CAR_ID, 85)); assertEquals(resultSet.size(), 14); resultSet.close(); resultSet = collection.retrieve(greaterThanOrEqualTo(Car.CAR_ID, 95)); assertEquals(resultSet.size(), 5); resultSet.close(); resultSet = collection.retrieve(greaterThanOrEqualTo(Car.CAR_ID, 85)); assertEquals(resultSet.size(), 15); resultSet.close(); } @Test public void indexQuantization_Between() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); Index<EntityHandle<Car>> index = withQuantizerOnAttribute(IntegerQuantizer.withCompressionFactor(10), Car.CAR_ID); index.clear(noQueryOptions()); collection.addIndex(index); collection.addAll(CarFactory.createCollectionOfCars(100)); Query<EntityHandle<Car>> query = between(Car.CAR_ID, 88, 92); assertEquals(collection.retrieve(query).size(), 5); assertEquals(retrieveCarIds(collection, query), asList(88, 89, 90, 91, 92)); query = between(Car.CAR_ID, 88, true, 92, true); ResultSet<EntityHandle<Car>> resultSet = collection.retrieve(query); assertEquals(resultSet.size(), 5); resultSet.close(); assertEquals(retrieveCarIds(collection, query), asList(88, 89, 90, 91, 92)); query = between(Car.CAR_ID, 88, false, 92, true); resultSet = collection.retrieve(query); assertEquals(resultSet.size(), 4); resultSet.close(); assertEquals(retrieveCarIds(collection, query), asList(89, 90, 91, 92)); query = between(Car.CAR_ID, 88, true, 92, false); resultSet = collection.retrieve(query); assertEquals(resultSet.size(), 4); resultSet.close(); assertEquals(retrieveCarIds(collection, query), asList(88, 89, 90, 91)); query = between(Car.CAR_ID, 88, false, 92, false); resultSet = collection.retrieve(query); assertEquals(resultSet.size(), 3); resultSet.close(); assertEquals(retrieveCarIds(collection, query), asList(89, 90, 91)); } @Test public void indexQuantization_ComplexQuery() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); Index<EntityHandle<Car>> index = withQuantizerOnAttribute(IntegerQuantizer.withCompressionFactor(10), Car.CAR_ID); index.clear(noQueryOptions()); collection.addIndex(index); collection.addAll(CarFactory.createCollectionOfCars(100)); Query<EntityHandle<Car>> query = and(between(Car.CAR_ID, 96, 98), greaterThan(Car.CAR_ID, 95)); // 3 objects match the query... ResultSet<EntityHandle<Car>> resultSet = collection.retrieve(query); assertEquals(resultSet.size(), 3); resultSet.close(); List<Integer> carIdsFound = retrieveCarIds(collection, query); assertEquals(carIdsFound, asList(96, 97, 98)); // Merge cost should be 10, because objects matching this query are in a single bucket... resultSet = collection.retrieve(query); assertEquals(resultSet.getMergeCost(), 10); resultSet.close(); } static List<Integer> retrieveCarIds(IndexedCollection<EntityHandle<Car>> collection, Query<EntityHandle<Car>> query) { ResultSet<EntityHandle<Car>> cars = collection.retrieve(query, queryOptions(orderBy(ascending(Car.CAR_ID)))); List<Integer> carIds = new ArrayList<>(); for (EntityHandle<Car> car : cars) { carIds.add(car.get().getCarId()); } cars.close(); return carIds; } @Test @SneakyThrows public void serializableComparable() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); NavigableIndex index = onAttribute(Car.TIMESTAMP); index.clear(noQueryOptions()); collection.addIndex(index); Car car1 = CarFactory.createCar(1); Car car2 = CarFactory.createCar(2); NTPServerTimeProvider ntpServerTimeProvider = new NTPServerTimeProvider(); ntpServerTimeProvider.startAsync().awaitRunning(); HybridTimestamp ts1 = new HybridTimestamp(ntpServerTimeProvider); HybridTimestamp ts2 = ts1.clone(); Thread.sleep(1000); ts2.update(); assertTrue(ts2.compareTo(ts1) > 0); assertTrue(ts2.getSerializableComparable().compareTo(ts1.getSerializableComparable()) > 0); car1.timestamp(ts1); car2.timestamp(ts2); collection.add(new ResolvedEntityHandle<>(car1)); collection.add(new ResolvedEntityHandle<>(car2)); try (ResultSet<EntityHandle<Car>> resultSet = collection.retrieve(greaterThan(Car.TIMESTAMP, ts1))) { assertEquals(resultSet.size(), 1); assertEquals(resultSet.uniqueResult().get().getModel(), "Taurus"); } index.clear(noQueryOptions()); ntpServerTimeProvider.stopAsync().awaitTerminated(); } @Test public void reindexData() { IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); KeyStatisticsIndex<Double, EntityHandle<Car>> PRICE_INDEX = (KeyStatisticsIndex<Double, EntityHandle<Car>>) onAttribute( Car.PRICE); PRICE_INDEX.clear(noQueryOptions()); Set<EntityHandle<Car>> cars = CarFactory.createCollectionOfCars(10); collection.addAll(cars); collection.addIndex(PRICE_INDEX); IndexedCollection<EntityHandle<Car>> collection1 = new ConcurrentIndexedCollection<>(); KeyStatisticsIndex<Double, EntityHandle<Car>> PRICE_INDEX_1 = (KeyStatisticsIndex<Double, EntityHandle<Car>>) onAttribute( Car.PRICE); collection1.addAll(cars); collection1.addIndex(PRICE_INDEX_1); assertEquals((int) PRICE_INDEX.getCountForKey(9000.23, noQueryOptions()), 1); assertEquals((int) PRICE_INDEX_1.getCountForKey(9000.23, noQueryOptions()), 1); PRICE_INDEX.clear(noQueryOptions()); PRICE_INDEX_1.clear(noQueryOptions()); } @Test public void minMax() { Random random = new Random(); IndexedCollection<EntityHandle<Car>> collection = new ConcurrentIndexedCollection<>(); SortedKeyStatisticsIndex<HybridTimestamp, EntityHandle<Car>> MODEL_INDEX = onAttribute(Car.TIMESTAMP); WrappedSimpleIndex<Car, HybridTimestamp> i = new WrappedSimpleIndex<>( (Function<Car, HybridTimestamp>) StandardEntity::timestamp); i.setAttribute(Car.TIMESTAMP); if (MODEL_INDEX.supportsQuery(new Min<>(i), noQueryOptions()) && MODEL_INDEX.supportsQuery(new Max<>(i), noQueryOptions())) { MODEL_INDEX.clear(noQueryOptions()); collection.addIndex(MODEL_INDEX); QueryOptions queryOptions = queryOptions(); queryOptions.put(Iterable.class, collection); assertTrue(collection.retrieve(max(i), queryOptions).isEmpty()); assertTrue(collection.retrieve(min(i), queryOptions).isEmpty()); Set<EntityHandle<Car>> cars1 = CarFactory.createCollectionOfCars(100_000); cars1.forEach(car -> car.get().timestamp(new HybridTimestamp(random.nextInt(), 0))); collection.addAll(cars1); long t1 = System.nanoTime(); HybridTimestamp max1 = collection.retrieve(max(i), queryOptions).uniqueResult().get().timestamp(); HybridTimestamp min1 = collection.retrieve(min(i), queryOptions).uniqueResult().get().timestamp(); long t2 = System.nanoTime(); // make sure query scoping is respected assertNotEquals(collection.retrieve(scoped(equal(Car.MODEL, "Focus"), max(i)), queryOptions) .uniqueResult().get().uuid(), collection.retrieve(max(i), queryOptions).uniqueResult().get().uuid()); assertNotEquals(collection.retrieve(scoped(equal(Car.MODEL, "Focus"), min(i)), queryOptions) .uniqueResult().get().uuid(), collection.retrieve(min(i), queryOptions).uniqueResult().get().uuid()); assertFalse(cars1.stream().anyMatch(c -> c.get().timestamp().getSerializableComparable() .compareTo(max1.getSerializableComparable()) > 0)); assertFalse(cars1.stream().anyMatch(c -> c.get().timestamp().getSerializableComparable() .compareTo(min1.getSerializableComparable()) < 0)); Set<EntityHandle<Car>> cars2 = CarFactory.createCollectionOfCars(100_000); cars2.forEach(car -> car.get().timestamp(new HybridTimestamp(random.nextInt(), random.nextInt(2)))); collection.addAll(cars2); long t1_ = System.nanoTime(); HybridTimestamp max2 = collection.retrieve(max(i), queryOptions).uniqueResult().get().timestamp(); HybridTimestamp min2 = collection.retrieve(min(i), queryOptions).uniqueResult().get().timestamp(); long t2_ = System.nanoTime(); assertFalse(cars2.stream().anyMatch(c -> c.get().timestamp().getSerializableComparable() .compareTo(max2.getSerializableComparable()) > 0)); assertFalse(cars2.stream().anyMatch(c -> c.get().timestamp().getSerializableComparable() .compareTo(min2.getSerializableComparable()) < 0)); MODEL_INDEX.clear(noQueryOptions()); } } }