Java tutorial
/* Copyright (c) 2012-2014 Boundless and others. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Distribution License v1.0 * which accompanies this distribution, and is available at * https://www.eclipse.org/org/documents/edl-v10.html * * Contributors: * Gabriel Roldan (Boundless) - initial implementation */ package org.locationtech.geogig.api; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; import static org.locationtech.geogig.api.RevTree.NORMALIZED_SIZE_LIMIT; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; import org.eclipse.jdt.annotation.Nullable; import org.locationtech.geogig.api.RevObject.TYPE; import org.locationtech.geogig.api.plumbing.HashObject; import org.locationtech.geogig.repository.DepthSearch; import org.locationtech.geogig.repository.SpatialOps; import org.locationtech.geogig.storage.NodePathStorageOrder; import org.locationtech.geogig.storage.ObjectDatabase; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import com.vividsolutions.jts.geom.Envelope; public class RevTreeBuilder { private static final Logger LOGGER = LoggerFactory.getLogger(RevTreeBuilder.class); /** * How many children nodes to hold before forcing normalization of the internal data structures * into tree buckets on the database * * @todo make this configurable */ public static final int DEFAULT_NORMALIZATION_THRESHOLD = 1000 * 1000; private final ObjectDatabase db; private final Set<String> deletes; private final Map<String, Node> treeChanges; private final Map<String, Node> featureChanges; protected final TreeMap<Integer, Bucket> bucketTreesByBucket; private int depth; private long initialSize; private int initialNumTrees; protected NodePathStorageOrder storageOrder = new NodePathStorageOrder(); private Map<ObjectId, RevTree> pendingWritesCache; /** * Empty tree constructor, used to create trees from scratch * * @param db * @param serialFactory */ public RevTreeBuilder(ObjectDatabase db) { this(db, null); } /** * Only useful to {@link #build() build} the named {@link #empty() empty} tree */ private RevTreeBuilder() { db = null; treeChanges = Maps.newTreeMap(); featureChanges = Maps.newTreeMap(); deletes = Sets.newTreeSet(); bucketTreesByBucket = Maps.newTreeMap(); pendingWritesCache = Maps.newTreeMap(); } /** * Copy constructor with tree depth */ public RevTreeBuilder(ObjectDatabase db, @Nullable final RevTree copy) { this(db, copy, 0, new TreeMap<ObjectId, RevTree>()); } /** * Copy constructor */ private RevTreeBuilder(final ObjectDatabase db, @Nullable final RevTree copy, final int depth, final Map<ObjectId, RevTree> pendingWritesCache) { checkNotNull(db); checkNotNull(pendingWritesCache); this.db = db; this.depth = depth; this.pendingWritesCache = pendingWritesCache; this.deletes = Sets.newHashSet(); this.treeChanges = Maps.newHashMap(); this.featureChanges = Maps.newHashMap(); this.bucketTreesByBucket = Maps.newTreeMap(); if (copy != null) { this.initialSize = copy.size(); this.initialNumTrees = copy.numTrees(); if (copy.trees().isPresent()) { checkArgument(!copy.buckets().isPresent()); for (Node node : copy.trees().get()) { putInternal(node); } } if (copy.features().isPresent()) { checkArgument(!copy.buckets().isPresent()); for (Node node : copy.features().get()) { putInternal(node); } } if (copy.buckets().isPresent()) { checkArgument(!copy.features().isPresent()); bucketTreesByBucket.putAll(copy.buckets().get()); } } } /** */ private void putInternal(Node node) { deletes.remove(node.getName()); switch (node.getType()) { case FEATURE: featureChanges.put(node.getName(), node); break; case TREE: treeChanges.put(node.getName(), node); break; default: throw new IllegalArgumentException( "Only tree or feature nodes can be added to a tree: " + node + " " + node.getType()); } } private RevTree loadTree(final ObjectId subtreeId) { RevTree subtree = this.pendingWritesCache.get(subtreeId); if (subtree == null) { subtree = db.getTree(subtreeId); } return subtree; } private Optional<Node> getInternal(final String key, final boolean deep) { Node found = featureChanges.get(key); if (found == null) { found = treeChanges.get(key); } if (found != null) { return Optional.of(found); } if (!deep) { return Optional.absent(); } if (deletes.contains(key)) { return Optional.absent(); } final Integer bucketIndex = computeBucket(key); final Bucket bucket = bucketTreesByBucket.get(bucketIndex); if (bucket == null) { return Optional.absent(); } RevTree subtree = loadTree(bucket.id()); DepthSearch depthSearch = new DepthSearch(db); Optional<Node> node = depthSearch.getDirectChild(subtree, key, depth + 1); if (node.isPresent()) { return Optional.of(node.get()); } else { return Optional.absent(); } } private long sizeOfTree(ObjectId treeId) { if (treeId.isNull()) { return 0L; } RevTree tree = loadTree(treeId); return tree.size(); } private int numPendingChanges() { int totalChanges = featureChanges.size() + treeChanges.size() + deletes.size(); return totalChanges; } /** * Splits the cached entries into subtrees and saves them, making sure the tree contains either * only entries or subtrees */ private RevTree normalize() { Stopwatch sw = Stopwatch.createStarted(); RevTree unnamedTree; final int numPendingChanges = numPendingChanges(); if (bucketTreesByBucket.isEmpty() && numPendingChanges <= NORMALIZED_SIZE_LIMIT) { unnamedTree = normalizeToChildren(); } else { unnamedTree = normalizeToBuckets(); checkState(featureChanges.isEmpty()); checkState(treeChanges.isEmpty()); if (unnamedTree.size() <= NORMALIZED_SIZE_LIMIT) { this.bucketTreesByBucket.clear(); if (unnamedTree.buckets().isPresent()) { unnamedTree = moveBucketsToChildren(unnamedTree); } if (this.depth == 0) { pendingWritesCache.clear(); } } } final int pendingWritesThreshold = 10 * 1000; final boolean topLevelTree = this.depth == 0;// am I an actual (addressable) tree or bucket // tree of a higher level one? final boolean forceWrite = pendingWritesCache.size() >= pendingWritesThreshold; if (!pendingWritesCache.isEmpty() && (topLevelTree || forceWrite)) { LOGGER.debug("calling db.putAll for {} buckets because {}...", pendingWritesCache.size(), (topLevelTree ? "writing top level tree" : "there are " + pendingWritesCache.size() + " pending bucket writes")); Stopwatch sw2 = Stopwatch.createStarted(); db.putAll(pendingWritesCache.values().iterator()); pendingWritesCache.clear(); LOGGER.debug("done in {}", sw2.stop()); } this.initialSize = unnamedTree.size(); this.initialNumTrees = unnamedTree.numTrees(); if (this.depth == 0) { LOGGER.debug("Normalization took {}. Changes: {}", sw.stop(), numPendingChanges); } return unnamedTree; } /** * @param tree * @return */ private RevTree moveBucketsToChildren(RevTree tree) { checkState(tree.buckets().isPresent()); checkState(this.bucketTreesByBucket.isEmpty()); for (Bucket bucket : tree.buckets().get().values()) { ObjectId id = bucket.id(); RevTree bucketTree = this.loadTree(id); if (bucketTree.buckets().isPresent()) { moveBucketsToChildren(bucketTree); } else { Iterator<Node> children = bucketTree.children(); while (children.hasNext()) { Node next = children.next(); putInternal(next); } } } return normalizeToChildren(); } /** * */ private RevTree normalizeToChildren() { Preconditions.checkState(this.bucketTreesByBucket.isEmpty()); // remove delete requests, we're building a leaf tree out of our nodes deletes.clear(); long size = featureChanges.size(); if (!treeChanges.isEmpty()) { for (Node node : treeChanges.values()) { size += sizeOf(node); } } Collection<Node> features = featureChanges.values(); Collection<Node> trees = treeChanges.values(); RevTreeImpl unnamedTree = RevTreeImpl.createLeafTree(ObjectId.NULL, size, features, trees); return unnamedTree; } private long sizeOf(Node node) { return node.getType().equals(TYPE.TREE) ? sizeOfTree(node.getObjectId()) : 1L; } /** * @return * */ private RevTree normalizeToBuckets() { // update all inner trees final ImmutableSet<Integer> changedBucketIndexes; // aggregate size delta for all changed buckets long sizeDelta = 0L; // aggregate number of trees delta for all changed buckets int treesDelta = 0; try { Multimap<Integer, Node> changesByBucket = getChangesByBucket(); Preconditions.checkState(featureChanges.isEmpty()); Preconditions.checkState(treeChanges.isEmpty()); Preconditions.checkState(deletes.isEmpty()); changedBucketIndexes = ImmutableSet.copyOf(changesByBucket.keySet()); final Map<Integer, RevTree> bucketTrees = getBucketTrees(changedBucketIndexes); List<RevTree> newLeafTreesToSave = Lists.newArrayList(); for (Integer bucketIndex : changedBucketIndexes) { final RevTree currentBucketTree = bucketTrees.get(bucketIndex); final int bucketDepth = this.depth + 1; final RevTreeBuilder bucketTreeBuilder = new RevTreeBuilder(this.db, currentBucketTree, bucketDepth, this.pendingWritesCache); { final Collection<Node> bucketEntries = changesByBucket.removeAll(bucketIndex); for (Node node : bucketEntries) { if (node.getObjectId().isNull()) { bucketTreeBuilder.remove(node.getName()); } else { bucketTreeBuilder.put(node); } } } final RevTree modifiedBucketTree = bucketTreeBuilder.build(); final long bucketSizeDelta = modifiedBucketTree.size() - currentBucketTree.size(); final int bucketTreesDelta = modifiedBucketTree.numTrees() - currentBucketTree.numTrees(); sizeDelta += bucketSizeDelta; treesDelta += bucketTreesDelta; if (modifiedBucketTree.isEmpty()) { bucketTreesByBucket.remove(bucketIndex); } else { final Bucket currBucket = this.bucketTreesByBucket.get(bucketIndex); if (currBucket == null || !currBucket.id().equals(modifiedBucketTree.getId())) { // if (currBucket != null) { // db.delete(currBucket.id()); // } // have it on the pending writes set only if its not a leaf tree. Non bucket // trees may be too large and cause OOM if (null != pendingWritesCache.remove(currentBucketTree.getId())) { // System.err.printf(" ---> removed bucket %s from list\n", // currentBucketTree.getId()); } if (modifiedBucketTree.buckets().isPresent()) { pendingWritesCache.put(modifiedBucketTree.getId(), modifiedBucketTree); } else { // db.put(modifiedBucketTree); newLeafTreesToSave.add(modifiedBucketTree); } Envelope bucketBounds = SpatialOps.boundsOf(modifiedBucketTree); Bucket bucket = Bucket.create(modifiedBucketTree.getId(), bucketBounds); bucketTreesByBucket.put(bucketIndex, bucket); } } } if (!newLeafTreesToSave.isEmpty()) { db.putAll(newLeafTreesToSave.iterator()); newLeafTreesToSave.clear(); newLeafTreesToSave = null; } } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new RuntimeException(e); } // compute final size and number of trees out of the aggregate deltas long accSize = sizeDelta; if (initialSize > RevTree.NORMALIZED_SIZE_LIMIT) { accSize += initialSize; } int accChildTreeCount = this.initialNumTrees + treesDelta; RevTreeImpl unnamedTree; unnamedTree = RevTreeImpl.createNodeTree(ObjectId.NULL, accSize, accChildTreeCount, this.bucketTreesByBucket); return unnamedTree; } private Map<Integer, RevTree> getBucketTrees(ImmutableSet<Integer> changedBucketIndexes) { Map<Integer, RevTree> bucketTrees = new HashMap<>(); List<Integer> missing = new ArrayList<>(changedBucketIndexes.size()); for (Integer bucketIndex : changedBucketIndexes) { Bucket bucket = bucketTreesByBucket.get(bucketIndex); RevTree cached = bucket == null ? RevTree.EMPTY : pendingWritesCache.get(bucket.id()); if (cached == null) { missing.add(bucketIndex); } else { bucketTrees.put(bucketIndex, cached); } } if (!missing.isEmpty()) { Map<ObjectId, Integer> ids = Maps.uniqueIndex(missing, new Function<Integer, ObjectId>() { @Override public ObjectId apply(Integer index) { return bucketTreesByBucket.get(index).id(); } }); Iterator<RevObject> all = db.getAll(ids.keySet()); while (all.hasNext()) { RevObject next = all.next(); bucketTrees.put(ids.get(next.getId()), (RevTree) next); } } return bucketTrees; } /** * @return the bucket tree or {@link RevTree#EMPTY} if this tree does not have a bucket for the * given bucket index */ private RevTree getBucketTree(Integer bucketIndex) { final Bucket bucket = bucketTreesByBucket.get(bucketIndex); if (bucket == null) { return RevTree.EMPTY; } else { return loadTree(bucket.id()); } } private Multimap<Integer, Node> getChangesByBucket() { Multimap<Integer, Node> changesByBucket = ArrayListMultimap.create(); if (!featureChanges.isEmpty()) { for (Node change : featureChanges.values()) { Integer bucketIndex = computeBucket(change.getName()); changesByBucket.put(bucketIndex, change); } featureChanges.clear(); } if (!treeChanges.isEmpty()) { for (Node change : treeChanges.values()) { Integer bucketIndex = computeBucket(change.getName()); changesByBucket.put(bucketIndex, change); } treeChanges.clear(); } if (!deletes.isEmpty()) { for (String delete : deletes) { Integer bucketIndex = computeBucket(delete); Node node = Node.create(delete, ObjectId.NULL, ObjectId.NULL, TYPE.FEATURE, null); changesByBucket.put(bucketIndex, node); } deletes.clear(); } return changesByBucket; } protected final Integer computeBucket(final String path) { return this.storageOrder.bucket(path, this.depth); } /** * Gets an entry by key, this is potentially slow. * * @param key * @return */ public Optional<Node> get(final String key) { return getInternal(key, true); } /** * Adds or replaces an element in the tree with the given key. * <p> * <!-- Implementation detail: If the number of cached entries (entries held directly by this * tree) reaches {@link #DEFAULT_NORMALIZATION_THRESHOLD}, this tree will {@link #normalize()} * itself. * * --> * * @param key non null * @param value non null */ public RevTreeBuilder put(final Node node) { Preconditions.checkNotNull(node, "node can't be null"); putInternal(node); if (numPendingChanges() >= DEFAULT_NORMALIZATION_THRESHOLD) { // hit the split factor modification tolerance, lets normalize normalize(); } return this; } /** * Removes an element from the tree * * @param childName the name of the child to remove * @return {@code this} */ public RevTreeBuilder remove(final String childName) { Preconditions.checkNotNull(childName, "key can't be null"); if (null == featureChanges.remove(childName)) { treeChanges.remove(childName); } deletes.add(childName); return this; } /** * @return the new tree, not saved to the object database. Any bucket tree though is saved when * this method returns. */ public RevTree build() { RevTree unnamedTree = normalize(); checkState(bucketTreesByBucket.isEmpty() || (featureChanges.isEmpty() && treeChanges.isEmpty())); ObjectId treeId = new HashObject().setObject(unnamedTree).call(); RevTreeImpl namedTree = RevTreeImpl.create(treeId, unnamedTree.size(), unnamedTree); return namedTree; } /** * Deletes all nodes that represent subtrees * * @return {@code this} */ public RevTreeBuilder clearSubtrees() { this.treeChanges.clear(); return this; } /** * @return a new instance of a properly "named" empty tree (as in with a proper object id after * applying {@link HashObject}) */ public static RevTree empty() { RevTree theEmptyTree = new RevTreeBuilder().build(); return theEmptyTree; } }