Java tutorial
/* * Copyright (C) 2008-2010, Google Inc. * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com> * and other copyright owners as documented in the project's IP log. * * This program and the accompanying materials are made available * under the terms of the Eclipse Distribution License v1.0 which * accompanies this distribution, is reproduced below, and is * available at http://www.eclipse.org/org/documents/edl-v10.php * * All rights reserved. * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials provided * with the distribution. * * - Neither the name of the Eclipse Foundation, Inc. nor the * names of its contributors may be used to endorse or promote * products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package org.eclipse.jgit.internal.storage.pack; import static java.util.Objects.requireNonNull; import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_DELTA; import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_WHOLE; import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; import static org.eclipse.jgit.lib.Constants.OBJ_TAG; import static org.eclipse.jgit.lib.Constants.OBJ_TREE; import java.io.IOException; import java.io.OutputStream; import java.lang.ref.WeakReference; import java.security.MessageDigest; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.zip.CRC32; import java.util.zip.CheckedOutputStream; import java.util.zip.Deflater; import java.util.zip.DeflaterOutputStream; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; import org.eclipse.jgit.internal.storage.file.PackBitmapIndexWriterV1; import org.eclipse.jgit.internal.storage.file.PackIndexWriter; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.AsyncObjectSizeQueue; import org.eclipse.jgit.lib.BatchingProgressMonitor; import org.eclipse.jgit.lib.BitmapIndex; import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; import org.eclipse.jgit.lib.BitmapObject; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdOwnerMap; import org.eclipse.jgit.lib.ObjectIdSet; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.ProgressMonitor; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.ThreadSafeProgressMonitor; import org.eclipse.jgit.revwalk.AsyncRevObjectQueue; import org.eclipse.jgit.revwalk.BitmapWalker; import org.eclipse.jgit.revwalk.DepthWalk; import org.eclipse.jgit.revwalk.ObjectWalk; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevFlag; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevSort; import org.eclipse.jgit.revwalk.RevTag; import org.eclipse.jgit.revwalk.RevTree; import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.storage.pack.PackStatistics; import org.eclipse.jgit.transport.FilterSpec; import org.eclipse.jgit.transport.ObjectCountCallback; import org.eclipse.jgit.transport.PacketLineOut; import org.eclipse.jgit.transport.WriteAbortedException; import org.eclipse.jgit.util.BlockList; import org.eclipse.jgit.util.TemporaryBuffer; /** * <p> * PackWriter class is responsible for generating pack files from specified set * of objects from repository. This implementation produce pack files in format * version 2. * </p> * <p> * Source of objects may be specified in two ways: * <ul> * <li>(usually) by providing sets of interesting and uninteresting objects in * repository - all interesting objects and their ancestors except uninteresting * objects and their ancestors will be included in pack, or</li> * <li>by providing iterator of {@link org.eclipse.jgit.revwalk.RevObject} * specifying exact list and order of objects in pack</li> * </ul> * <p> * Typical usage consists of creating an instance, configuring options, * preparing the list of objects by calling {@link #preparePack(Iterator)} or * {@link #preparePack(ProgressMonitor, Set, Set)}, and streaming with * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. If the * pack is being stored as a file the matching index can be written out after * writing the pack by {@link #writeIndex(OutputStream)}. An optional bitmap * index can be made by calling {@link #prepareBitmapIndex(ProgressMonitor)} * followed by {@link #writeBitmapIndex(OutputStream)}. * </p> * <p> * Class provide set of configurable options and * {@link org.eclipse.jgit.lib.ProgressMonitor} support, as operations may take * a long time for big repositories. Deltas searching algorithm is <b>NOT * IMPLEMENTED</b> yet - this implementation relies only on deltas and objects * reuse. * </p> * <p> * This class is not thread safe. It is intended to be used in one thread as a * single pass to produce one pack. Invoking methods multiple times or out of * order is not supported as internal data structures are destroyed during * certain phases to save memory when packing large repositories. * </p> */ public class PackWriter implements AutoCloseable { private static final int PACK_VERSION_GENERATED = 2; /** Empty set of objects for {@code preparePack()}. */ public static final Set<ObjectId> NONE = Collections.emptySet(); private static final Map<WeakReference<PackWriter>, Boolean> instances = new ConcurrentHashMap<>(); private static final Iterable<PackWriter> instancesIterable = new Iterable<PackWriter>() { @Override public Iterator<PackWriter> iterator() { return new Iterator<PackWriter>() { private final Iterator<WeakReference<PackWriter>> it = instances.keySet().iterator(); private PackWriter next; @Override public boolean hasNext() { if (next != null) return true; while (it.hasNext()) { WeakReference<PackWriter> ref = it.next(); next = ref.get(); if (next != null) return true; it.remove(); } return false; } @Override public PackWriter next() { if (hasNext()) { PackWriter result = next; next = null; return result; } throw new NoSuchElementException(); } @Override public void remove() { throw new UnsupportedOperationException(); } }; } }; /** * Get all allocated, non-released PackWriters instances. * * @return all allocated, non-released PackWriters instances. */ public static Iterable<PackWriter> getInstances() { return instancesIterable; } @SuppressWarnings("unchecked") BlockList<ObjectToPack> objectsLists[] = new BlockList[OBJ_TAG + 1]; { objectsLists[OBJ_COMMIT] = new BlockList<>(); objectsLists[OBJ_TREE] = new BlockList<>(); objectsLists[OBJ_BLOB] = new BlockList<>(); objectsLists[OBJ_TAG] = new BlockList<>(); } private ObjectIdOwnerMap<ObjectToPack> objectsMap = new ObjectIdOwnerMap<>(); // edge objects for thin packs private List<ObjectToPack> edgeObjects = new BlockList<>(); // Objects the client is known to have already. private BitmapBuilder haveObjects; private List<CachedPack> cachedPacks = new ArrayList<>(2); private Set<ObjectId> tagTargets = NONE; private Set<? extends ObjectId> excludeFromBitmapSelection = NONE; private ObjectIdSet[] excludeInPacks; private ObjectIdSet excludeInPackLast; private Deflater myDeflater; private final ObjectReader reader; /** {@link #reader} recast to the reuse interface, if it supports it. */ private final ObjectReuseAsIs reuseSupport; final PackConfig config; private final PackStatistics.Accumulator stats; private final MutableState state; private final WeakReference<PackWriter> selfRef; private PackStatistics.ObjectType.Accumulator typeStats; private List<ObjectToPack> sortedByName; private byte packcsum[]; private boolean deltaBaseAsOffset; private boolean reuseDeltas; private boolean reuseDeltaCommits; private boolean reuseValidate; private boolean thin; private boolean useCachedPacks; private boolean useBitmaps; private boolean ignoreMissingUninteresting = true; private boolean pruneCurrentObjectList; private boolean shallowPack; private boolean canBuildBitmaps; private boolean indexDisabled; private int depth; private Collection<? extends ObjectId> unshallowObjects; private PackBitmapIndexBuilder writeBitmaps; private CRC32 crc32; private ObjectCountCallback callback; private FilterSpec filterSpec = FilterSpec.NO_FILTER; private PackfileUriConfig packfileUriConfig; /** * Create writer for specified repository. * <p> * Objects for packing are specified in {@link #preparePack(Iterator)} or * {@link #preparePack(ProgressMonitor, Set, Set)}. * * @param repo * repository where objects are stored. */ public PackWriter(Repository repo) { this(repo, repo.newObjectReader()); } /** * Create a writer to load objects from the specified reader. * <p> * Objects for packing are specified in {@link #preparePack(Iterator)} or * {@link #preparePack(ProgressMonitor, Set, Set)}. * * @param reader * reader to read from the repository with. */ public PackWriter(ObjectReader reader) { this(new PackConfig(), reader); } /** * Create writer for specified repository. * <p> * Objects for packing are specified in {@link #preparePack(Iterator)} or * {@link #preparePack(ProgressMonitor, Set, Set)}. * * @param repo * repository where objects are stored. * @param reader * reader to read from the repository with. */ public PackWriter(Repository repo, ObjectReader reader) { this(new PackConfig(repo), reader); } /** * Create writer with a specified configuration. * <p> * Objects for packing are specified in {@link #preparePack(Iterator)} or * {@link #preparePack(ProgressMonitor, Set, Set)}. * * @param config * configuration for the pack writer. * @param reader * reader to read from the repository with. */ public PackWriter(PackConfig config, ObjectReader reader) { this(config, reader, null); } /** * Create writer with a specified configuration. * <p> * Objects for packing are specified in {@link #preparePack(Iterator)} or * {@link #preparePack(ProgressMonitor, Set, Set)}. * * @param config * configuration for the pack writer. * @param reader * reader to read from the repository with. * @param statsAccumulator * accumulator for statics */ public PackWriter(PackConfig config, final ObjectReader reader, @Nullable PackStatistics.Accumulator statsAccumulator) { this.config = config; this.reader = reader; if (reader instanceof ObjectReuseAsIs) reuseSupport = ((ObjectReuseAsIs) reader); else reuseSupport = null; deltaBaseAsOffset = config.isDeltaBaseAsOffset(); reuseDeltas = config.isReuseDeltas(); reuseValidate = true; // be paranoid by default stats = statsAccumulator != null ? statsAccumulator : new PackStatistics.Accumulator(); state = new MutableState(); selfRef = new WeakReference<>(this); instances.put(selfRef, Boolean.TRUE); } /** * Set the {@code ObjectCountCallback}. * <p> * It should be set before calling * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. * * @param callback * the callback to set * @return this object for chaining. */ public PackWriter setObjectCountCallback(ObjectCountCallback callback) { this.callback = callback; return this; } /** * Records the set of shallow commits in the client. * * @param clientShallowCommits * the shallow commits in the client */ public void setClientShallowCommits(Set<ObjectId> clientShallowCommits) { stats.clientShallowCommits = Collections.unmodifiableSet(new HashSet<>(clientShallowCommits)); } /** * Check whether writer can store delta base as an offset (new style * reducing pack size) or should store it as an object id (legacy style, * compatible with old readers). * * Default setting: {@value PackConfig#DEFAULT_DELTA_BASE_AS_OFFSET} * * @return true if delta base is stored as an offset; false if it is stored * as an object id. */ public boolean isDeltaBaseAsOffset() { return deltaBaseAsOffset; } /** * Set writer delta base format. Delta base can be written as an offset in a * pack file (new approach reducing file size) or as an object id (legacy * approach, compatible with old readers). * * Default setting: {@value PackConfig#DEFAULT_DELTA_BASE_AS_OFFSET} * * @param deltaBaseAsOffset * boolean indicating whether delta base can be stored as an * offset. */ public void setDeltaBaseAsOffset(boolean deltaBaseAsOffset) { this.deltaBaseAsOffset = deltaBaseAsOffset; } /** * Check if the writer will reuse commits that are already stored as deltas. * * @return true if the writer would reuse commits stored as deltas, assuming * delta reuse is already enabled. */ public boolean isReuseDeltaCommits() { return reuseDeltaCommits; } /** * Set the writer to reuse existing delta versions of commits. * * @param reuse * if true, the writer will reuse any commits stored as deltas. * By default the writer does not reuse delta commits. */ public void setReuseDeltaCommits(boolean reuse) { reuseDeltaCommits = reuse; } /** * Check if the writer validates objects before copying them. * * @return true if validation is enabled; false if the reader will handle * object validation as a side-effect of it consuming the output. */ public boolean isReuseValidatingObjects() { return reuseValidate; } /** * Enable (or disable) object validation during packing. * * @param validate * if true the pack writer will validate an object before it is * put into the output. This additional validation work may be * necessary to avoid propagating corruption from one local pack * file to another local pack file. */ public void setReuseValidatingObjects(boolean validate) { reuseValidate = validate; } /** * Whether this writer is producing a thin pack. * * @return true if this writer is producing a thin pack. */ public boolean isThin() { return thin; } /** * Whether writer may pack objects with delta base object not within set of * objects to pack * * @param packthin * a boolean indicating whether writer may pack objects with * delta base object not within set of objects to pack, but * belonging to party repository (uninteresting/boundary) as * determined by set; this kind of pack is used only for * transport; true - to produce thin pack, false - otherwise. */ public void setThin(boolean packthin) { thin = packthin; } /** * Whether to reuse cached packs. * * @return {@code true} to reuse cached packs. If true index creation isn't * available. */ public boolean isUseCachedPacks() { return useCachedPacks; } /** * Whether to use cached packs * * @param useCached * if set to {@code true} and a cached pack is present, it will * be appended onto the end of a thin-pack, reducing the amount * of working set space and CPU used by PackWriter. Enabling this * feature prevents PackWriter from creating an index for the * newly created pack, so its only suitable for writing to a * network client, where the client will make the index. */ public void setUseCachedPacks(boolean useCached) { useCachedPacks = useCached; } /** * Whether to use bitmaps * * @return {@code true} to use bitmaps for ObjectWalks, if available. */ public boolean isUseBitmaps() { return useBitmaps; } /** * Whether to use bitmaps * * @param useBitmaps * if set to true, bitmaps will be used when preparing a pack. */ public void setUseBitmaps(boolean useBitmaps) { this.useBitmaps = useBitmaps; } /** * Whether the index file cannot be created by this PackWriter. * * @return {@code true} if the index file cannot be created by this * PackWriter. */ public boolean isIndexDisabled() { return indexDisabled || !cachedPacks.isEmpty(); } /** * Whether to disable creation of the index file. * * @param noIndex * {@code true} to disable creation of the index file. */ public void setIndexDisabled(boolean noIndex) { this.indexDisabled = noIndex; } /** * Whether to ignore missing uninteresting objects * * @return {@code true} to ignore objects that are uninteresting and also * not found on local disk; false to throw a * {@link org.eclipse.jgit.errors.MissingObjectException} out of * {@link #preparePack(ProgressMonitor, Set, Set)} if an * uninteresting object is not in the source repository. By default, * true, permitting gracefully ignoring of uninteresting objects. */ public boolean isIgnoreMissingUninteresting() { return ignoreMissingUninteresting; } /** * Whether writer should ignore non existing uninteresting objects * * @param ignore * {@code true} if writer should ignore non existing * uninteresting objects during construction set of objects to * pack; false otherwise - non existing uninteresting objects may * cause {@link org.eclipse.jgit.errors.MissingObjectException} */ public void setIgnoreMissingUninteresting(boolean ignore) { ignoreMissingUninteresting = ignore; } /** * Set the tag targets that should be hoisted earlier during packing. * <p> * Callers may put objects into this set before invoking any of the * preparePack methods to influence where an annotated tag's target is * stored within the resulting pack. Typically these will be clustered * together, and hoisted earlier in the file even if they are ancient * revisions, allowing readers to find tag targets with better locality. * * @param objects * objects that annotated tags point at. */ public void setTagTargets(Set<ObjectId> objects) { tagTargets = objects; } /** * Configure this pack for a shallow clone. * * @param depth * maximum depth of history to return. 1 means return only the * "wants". * @param unshallow * objects which used to be shallow on the client, but are being * extended as part of this fetch */ public void setShallowPack(int depth, Collection<? extends ObjectId> unshallow) { this.shallowPack = true; this.depth = depth; this.unshallowObjects = unshallow; } /** * @param filter the filter which indicates what and what not this writer * should include */ public void setFilterSpec(@NonNull FilterSpec filter) { filterSpec = requireNonNull(filter); } /** * @param config configuration related to packfile URIs * @since 5.5 */ public void setPackfileUriConfig(PackfileUriConfig config) { packfileUriConfig = config; } /** * Returns objects number in a pack file that was created by this writer. * * @return number of objects in pack. * @throws java.io.IOException * a cached pack cannot supply its object count. */ public long getObjectCount() throws IOException { if (stats.totalObjects == 0) { long objCnt = 0; objCnt += objectsLists[OBJ_COMMIT].size(); objCnt += objectsLists[OBJ_TREE].size(); objCnt += objectsLists[OBJ_BLOB].size(); objCnt += objectsLists[OBJ_TAG].size(); for (CachedPack pack : cachedPacks) objCnt += pack.getObjectCount(); return objCnt; } return stats.totalObjects; } private long getUnoffloadedObjectCount() throws IOException { long objCnt = 0; objCnt += objectsLists[OBJ_COMMIT].size(); objCnt += objectsLists[OBJ_TREE].size(); objCnt += objectsLists[OBJ_BLOB].size(); objCnt += objectsLists[OBJ_TAG].size(); for (CachedPack pack : cachedPacks) { CachedPackUriProvider.PackInfo packInfo = packfileUriConfig.cachedPackUriProvider.getInfo(pack, packfileUriConfig.protocolsSupported); if (packInfo == null) { objCnt += pack.getObjectCount(); } } return objCnt; } /** * Returns the object ids in the pack file that was created by this writer. * <p> * This method can only be invoked after * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)} has * been invoked and completed successfully. * * @return set of objects in pack. * @throws java.io.IOException * a cached pack cannot supply its object ids. */ public ObjectIdOwnerMap<ObjectIdOwnerMap.Entry> getObjectSet() throws IOException { if (!cachedPacks.isEmpty()) throw new IOException(JGitText.get().cachedPacksPreventsListingObjects); if (writeBitmaps != null) { return writeBitmaps.getObjectSet(); } ObjectIdOwnerMap<ObjectIdOwnerMap.Entry> r = new ObjectIdOwnerMap<>(); for (BlockList<ObjectToPack> objList : objectsLists) { if (objList != null) { for (ObjectToPack otp : objList) r.add(new ObjectIdOwnerMap.Entry(otp) { // A new entry that copies the ObjectId }); } } return r; } /** * Add a pack index whose contents should be excluded from the result. * * @param idx * objects in this index will not be in the output pack. */ public void excludeObjects(ObjectIdSet idx) { if (excludeInPacks == null) { excludeInPacks = new ObjectIdSet[] { idx }; excludeInPackLast = idx; } else { int cnt = excludeInPacks.length; ObjectIdSet[] newList = new ObjectIdSet[cnt + 1]; System.arraycopy(excludeInPacks, 0, newList, 0, cnt); newList[cnt] = idx; excludeInPacks = newList; } } /** * Prepare the list of objects to be written to the pack stream. * <p> * Iterator <b>exactly</b> determines which objects are included in a pack * and order they appear in pack (except that objects order by type is not * needed at input). This order should conform general rules of ordering * objects in git - by recency and path (type and delta-base first is * internally secured) and responsibility for guaranteeing this order is on * a caller side. Iterator must return each id of object to write exactly * once. * </p> * * @param objectsSource * iterator of object to store in a pack; order of objects within * each type is important, ordering by type is not needed; * allowed types for objects are * {@link org.eclipse.jgit.lib.Constants#OBJ_COMMIT}, * {@link org.eclipse.jgit.lib.Constants#OBJ_TREE}, * {@link org.eclipse.jgit.lib.Constants#OBJ_BLOB} and * {@link org.eclipse.jgit.lib.Constants#OBJ_TAG}; objects * returned by iterator may be later reused by caller as object * id and type are internally copied in each iteration. * @throws java.io.IOException * when some I/O problem occur during reading objects. */ public void preparePack(@NonNull Iterator<RevObject> objectsSource) throws IOException { while (objectsSource.hasNext()) { addObject(objectsSource.next()); } } /** * Prepare the list of objects to be written to the pack stream. * <p> * Basing on these 2 sets, another set of objects to put in a pack file is * created: this set consists of all objects reachable (ancestors) from * interesting objects, except uninteresting objects and their ancestors. * This method uses class {@link org.eclipse.jgit.revwalk.ObjectWalk} * extensively to find out that appropriate set of output objects and their * optimal order in output pack. Order is consistent with general git * in-pack rules: sort by object type, recency, path and delta-base first. * </p> * * @param countingMonitor * progress during object enumeration. * @param want * collection of objects to be marked as interesting (start * points of graph traversal). Must not be {@code null}. * @param have * collection of objects to be marked as uninteresting (end * points of graph traversal). Pass {@link #NONE} if all objects * reachable from {@code want} are desired, such as when serving * a clone. * @throws java.io.IOException * when some I/O problem occur during reading objects. */ public void preparePack(ProgressMonitor countingMonitor, @NonNull Set<? extends ObjectId> want, @NonNull Set<? extends ObjectId> have) throws IOException { preparePack(countingMonitor, want, have, NONE, NONE); } /** * Prepare the list of objects to be written to the pack stream. * <p> * Like {@link #preparePack(ProgressMonitor, Set, Set)} but also allows * specifying commits that should not be walked past ("shallow" commits). * The caller is responsible for filtering out commits that should not be * shallow any more ("unshallow" commits as in {@link #setShallowPack}) from * the shallow set. * * @param countingMonitor * progress during object enumeration. * @param want * objects of interest, ancestors of which will be included in * the pack. Must not be {@code null}. * @param have * objects whose ancestors (up to and including {@code shallow} * commits) do not need to be included in the pack because they * are already available from elsewhere. Must not be * {@code null}. * @param shallow * commits indicating the boundary of the history marked with * {@code have}. Shallow commits have parents but those parents * are considered not to be already available. Parents of * {@code shallow} commits and earlier generations will be * included in the pack if requested by {@code want}. Must not be * {@code null}. * @throws java.io.IOException * an I/O problem occurred while reading objects. */ public void preparePack(ProgressMonitor countingMonitor, @NonNull Set<? extends ObjectId> want, @NonNull Set<? extends ObjectId> have, @NonNull Set<? extends ObjectId> shallow) throws IOException { preparePack(countingMonitor, want, have, shallow, NONE); } /** * Prepare the list of objects to be written to the pack stream. * <p> * Like {@link #preparePack(ProgressMonitor, Set, Set)} but also allows * specifying commits that should not be walked past ("shallow" commits). * The caller is responsible for filtering out commits that should not be * shallow any more ("unshallow" commits as in {@link #setShallowPack}) from * the shallow set. * * @param countingMonitor * progress during object enumeration. * @param want * objects of interest, ancestors of which will be included in * the pack. Must not be {@code null}. * @param have * objects whose ancestors (up to and including {@code shallow} * commits) do not need to be included in the pack because they * are already available from elsewhere. Must not be * {@code null}. * @param shallow * commits indicating the boundary of the history marked with * {@code have}. Shallow commits have parents but those parents * are considered not to be already available. Parents of * {@code shallow} commits and earlier generations will be * included in the pack if requested by {@code want}. Must not be * {@code null}. * @param noBitmaps * collection of objects to be excluded from bitmap commit * selection. * @throws java.io.IOException * an I/O problem occurred while reading objects. */ public void preparePack(ProgressMonitor countingMonitor, @NonNull Set<? extends ObjectId> want, @NonNull Set<? extends ObjectId> have, @NonNull Set<? extends ObjectId> shallow, @NonNull Set<? extends ObjectId> noBitmaps) throws IOException { try (ObjectWalk ow = getObjectWalk()) { ow.assumeShallow(shallow); preparePack(countingMonitor, ow, want, have, noBitmaps); } } private ObjectWalk getObjectWalk() { return shallowPack ? new DepthWalk.ObjectWalk(reader, depth - 1) : new ObjectWalk(reader); } /** * A visitation policy which uses the depth at which the object is seen to * decide if re-traversal is necessary. In particular, if the object has * already been visited at this depth or shallower, it is not necessary to * re-visit at this depth. */ private static class DepthAwareVisitationPolicy implements ObjectWalk.VisitationPolicy { private final Map<ObjectId, Integer> lowestDepthVisited = new HashMap<>(); private final ObjectWalk walk; DepthAwareVisitationPolicy(ObjectWalk walk) { this.walk = requireNonNull(walk); } @Override public boolean shouldVisit(RevObject o) { Integer lastDepth = lowestDepthVisited.get(o); if (lastDepth == null) { return true; } return walk.getTreeDepth() < lastDepth.intValue(); } @Override public void visited(RevObject o) { lowestDepthVisited.put(o, Integer.valueOf(walk.getTreeDepth())); } } /** * Prepare the list of objects to be written to the pack stream. * <p> * Basing on these 2 sets, another set of objects to put in a pack file is * created: this set consists of all objects reachable (ancestors) from * interesting objects, except uninteresting objects and their ancestors. * This method uses class {@link org.eclipse.jgit.revwalk.ObjectWalk} * extensively to find out that appropriate set of output objects and their * optimal order in output pack. Order is consistent with general git * in-pack rules: sort by object type, recency, path and delta-base first. * </p> * * @param countingMonitor * progress during object enumeration. * @param walk * ObjectWalk to perform enumeration. * @param interestingObjects * collection of objects to be marked as interesting (start * points of graph traversal). Must not be {@code null}. * @param uninterestingObjects * collection of objects to be marked as uninteresting (end * points of graph traversal). Pass {@link #NONE} if all objects * reachable from {@code want} are desired, such as when serving * a clone. * @param noBitmaps * collection of objects to be excluded from bitmap commit * selection. * @throws java.io.IOException * when some I/O problem occur during reading objects. */ public void preparePack(ProgressMonitor countingMonitor, @NonNull ObjectWalk walk, @NonNull Set<? extends ObjectId> interestingObjects, @NonNull Set<? extends ObjectId> uninterestingObjects, @NonNull Set<? extends ObjectId> noBitmaps) throws IOException { if (countingMonitor == null) countingMonitor = NullProgressMonitor.INSTANCE; if (shallowPack && !(walk instanceof DepthWalk.ObjectWalk)) throw new IllegalArgumentException(JGitText.get().shallowPacksRequireDepthWalk); if (filterSpec.getTreeDepthLimit() >= 0) { walk.setVisitationPolicy(new DepthAwareVisitationPolicy(walk)); } findObjectsToPack(countingMonitor, walk, interestingObjects, uninterestingObjects, noBitmaps); } /** * Determine if the pack file will contain the requested object. * * @param id * the object to test the existence of. * @return true if the object will appear in the output pack file. * @throws java.io.IOException * a cached pack cannot be examined. */ public boolean willInclude(AnyObjectId id) throws IOException { ObjectToPack obj = objectsMap.get(id); return obj != null && !obj.isEdge(); } /** * Lookup the ObjectToPack object for a given ObjectId. * * @param id * the object to find in the pack. * @return the object we are packing, or null. */ public ObjectToPack get(AnyObjectId id) { ObjectToPack obj = objectsMap.get(id); return obj != null && !obj.isEdge() ? obj : null; } /** * Computes SHA-1 of lexicographically sorted objects ids written in this * pack, as used to name a pack file in repository. * * @return ObjectId representing SHA-1 name of a pack that was created. */ public ObjectId computeName() { final byte[] buf = new byte[OBJECT_ID_LENGTH]; final MessageDigest md = Constants.newMessageDigest(); for (ObjectToPack otp : sortByName()) { otp.copyRawTo(buf, 0); md.update(buf, 0, OBJECT_ID_LENGTH); } return ObjectId.fromRaw(md.digest()); } /** * Returns the index format version that will be written. * <p> * This method can only be invoked after * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)} has * been invoked and completed successfully. * * @return the index format version. */ public int getIndexVersion() { int indexVersion = config.getIndexVersion(); if (indexVersion <= 0) { for (BlockList<ObjectToPack> objs : objectsLists) indexVersion = Math.max(indexVersion, PackIndexWriter.oldestPossibleFormat(objs)); } return indexVersion; } /** * Create an index file to match the pack file just written. * <p> * Called after * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. * <p> * Writing an index is only required for local pack storage. Packs sent on * the network do not need to create an index. * * @param indexStream * output for the index data. Caller is responsible for closing * this stream. * @throws java.io.IOException * the index data could not be written to the supplied stream. */ public void writeIndex(OutputStream indexStream) throws IOException { if (isIndexDisabled()) throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation); long writeStart = System.currentTimeMillis(); final PackIndexWriter iw = PackIndexWriter.createVersion(indexStream, getIndexVersion()); iw.write(sortByName(), packcsum); stats.timeWriting += System.currentTimeMillis() - writeStart; } /** * Create a bitmap index file to match the pack file just written. * <p> * Called after {@link #prepareBitmapIndex(ProgressMonitor)}. * * @param bitmapIndexStream * output for the bitmap index data. Caller is responsible for * closing this stream. * @throws java.io.IOException * the index data could not be written to the supplied stream. */ public void writeBitmapIndex(OutputStream bitmapIndexStream) throws IOException { if (writeBitmaps == null) throw new IOException(JGitText.get().bitmapsMustBePrepared); long writeStart = System.currentTimeMillis(); final PackBitmapIndexWriterV1 iw = new PackBitmapIndexWriterV1(bitmapIndexStream); iw.write(writeBitmaps, packcsum); stats.timeWriting += System.currentTimeMillis() - writeStart; } private List<ObjectToPack> sortByName() { if (sortedByName == null) { int cnt = 0; cnt += objectsLists[OBJ_COMMIT].size(); cnt += objectsLists[OBJ_TREE].size(); cnt += objectsLists[OBJ_BLOB].size(); cnt += objectsLists[OBJ_TAG].size(); sortedByName = new BlockList<>(cnt); sortedByName.addAll(objectsLists[OBJ_COMMIT]); sortedByName.addAll(objectsLists[OBJ_TREE]); sortedByName.addAll(objectsLists[OBJ_BLOB]); sortedByName.addAll(objectsLists[OBJ_TAG]); Collections.sort(sortedByName); } return sortedByName; } private void beginPhase(PackingPhase phase, ProgressMonitor monitor, long cnt) { state.phase = phase; String task; switch (phase) { case COUNTING: task = JGitText.get().countingObjects; break; case GETTING_SIZES: task = JGitText.get().searchForSizes; break; case FINDING_SOURCES: task = JGitText.get().searchForReuse; break; case COMPRESSING: task = JGitText.get().compressingObjects; break; case WRITING: task = JGitText.get().writingObjects; break; case BUILDING_BITMAPS: task = JGitText.get().buildingBitmaps; break; default: throw new IllegalArgumentException(MessageFormat.format(JGitText.get().illegalPackingPhase, phase)); } monitor.beginTask(task, (int) cnt); } private void endPhase(ProgressMonitor monitor) { monitor.endTask(); } /** * Write the prepared pack to the supplied stream. * <p> * Called after * {@link #preparePack(ProgressMonitor, ObjectWalk, Set, Set, Set)} or * {@link #preparePack(ProgressMonitor, Set, Set)}. * <p> * Performs delta search if enabled and writes the pack stream. * <p> * All reused objects data checksum (Adler32/CRC32) is computed and * validated against existing checksum. * * @param compressMonitor * progress monitor to report object compression work. * @param writeMonitor * progress monitor to report the number of objects written. * @param packStream * output stream of pack data. The stream should be buffered by * the caller. The caller is responsible for closing the stream. * @throws java.io.IOException * an error occurred reading a local object's data to include in * the pack, or writing compressed object data to the output * stream. * @throws WriteAbortedException * the write operation is aborted by * {@link org.eclipse.jgit.transport.ObjectCountCallback} . */ public void writePack(ProgressMonitor compressMonitor, ProgressMonitor writeMonitor, OutputStream packStream) throws IOException { if (compressMonitor == null) compressMonitor = NullProgressMonitor.INSTANCE; if (writeMonitor == null) writeMonitor = NullProgressMonitor.INSTANCE; excludeInPacks = null; excludeInPackLast = null; boolean needSearchForReuse = reuseSupport != null && (reuseDeltas || config.isReuseObjects() || !cachedPacks.isEmpty()); if (compressMonitor instanceof BatchingProgressMonitor) { long delay = 1000; if (needSearchForReuse && config.isDeltaCompress()) delay = 500; ((BatchingProgressMonitor) compressMonitor).setDelayStart(delay, TimeUnit.MILLISECONDS); } if (needSearchForReuse) searchForReuse(compressMonitor); if (config.isDeltaCompress()) searchForDeltas(compressMonitor); crc32 = new CRC32(); final PackOutputStream out = new PackOutputStream(writeMonitor, isIndexDisabled() ? packStream : new CheckedOutputStream(packStream, crc32), this); long objCnt = packfileUriConfig == null ? getObjectCount() : getUnoffloadedObjectCount(); stats.totalObjects = objCnt; if (callback != null) callback.setObjectCount(objCnt); beginPhase(PackingPhase.WRITING, writeMonitor, objCnt); long writeStart = System.currentTimeMillis(); try { List<CachedPack> unwrittenCachedPacks; if (packfileUriConfig != null) { unwrittenCachedPacks = new ArrayList<>(); CachedPackUriProvider p = packfileUriConfig.cachedPackUriProvider; PacketLineOut o = packfileUriConfig.pckOut; o.writeString("packfile-uris\n"); //$NON-NLS-1$ for (CachedPack pack : cachedPacks) { CachedPackUriProvider.PackInfo packInfo = p.getInfo(pack, packfileUriConfig.protocolsSupported); if (packInfo != null) { o.writeString(packInfo.getHash() + ' ' + packInfo.getUri() + '\n'); stats.offloadedPackfiles += 1; stats.offloadedPackfileSize += packInfo.getSize(); } else { unwrittenCachedPacks.add(pack); } } packfileUriConfig.pckOut.writeDelim(); packfileUriConfig.pckOut.writeString("packfile\n"); //$NON-NLS-1$ } else { unwrittenCachedPacks = cachedPacks; } out.writeFileHeader(PACK_VERSION_GENERATED, objCnt); out.flush(); writeObjects(out); if (!edgeObjects.isEmpty() || !cachedPacks.isEmpty()) { for (PackStatistics.ObjectType.Accumulator typeStat : stats.objectTypes) { if (typeStat == null) continue; stats.thinPackBytes += typeStat.bytes; } } stats.reusedPacks = Collections.unmodifiableList(cachedPacks); for (CachedPack pack : unwrittenCachedPacks) { long deltaCnt = pack.getDeltaCount(); stats.reusedObjects += pack.getObjectCount(); stats.reusedDeltas += deltaCnt; stats.totalDeltas += deltaCnt; reuseSupport.copyPackAsIs(out, pack); } writeChecksum(out); out.flush(); } finally { stats.timeWriting = System.currentTimeMillis() - writeStart; stats.depth = depth; for (PackStatistics.ObjectType.Accumulator typeStat : stats.objectTypes) { if (typeStat == null) continue; typeStat.cntDeltas += typeStat.reusedDeltas; stats.reusedObjects += typeStat.reusedObjects; stats.reusedDeltas += typeStat.reusedDeltas; stats.totalDeltas += typeStat.cntDeltas; } } stats.totalBytes = out.length(); reader.close(); endPhase(writeMonitor); } /** * Get statistics of what this PackWriter did in order to create the final * pack stream. * * @return description of what this PackWriter did in order to create the * final pack stream. This should only be invoked after the calls to * create the pack/index/bitmap have completed. */ public PackStatistics getStatistics() { return new PackStatistics(stats); } /** * Get snapshot of the current state of this PackWriter. * * @return snapshot of the current state of this PackWriter. */ public State getState() { return state.snapshot(); } /** * {@inheritDoc} * <p> * Release all resources used by this writer. */ @Override public void close() { reader.close(); if (myDeflater != null) { myDeflater.end(); myDeflater = null; } instances.remove(selfRef); } private void searchForReuse(ProgressMonitor monitor) throws IOException { long cnt = 0; cnt += objectsLists[OBJ_COMMIT].size(); cnt += objectsLists[OBJ_TREE].size(); cnt += objectsLists[OBJ_BLOB].size(); cnt += objectsLists[OBJ_TAG].size(); long start = System.currentTimeMillis(); beginPhase(PackingPhase.FINDING_SOURCES, monitor, cnt); if (cnt <= 4096) { // For small object counts, do everything as one list. BlockList<ObjectToPack> tmp = new BlockList<>((int) cnt); tmp.addAll(objectsLists[OBJ_TAG]); tmp.addAll(objectsLists[OBJ_COMMIT]); tmp.addAll(objectsLists[OBJ_TREE]); tmp.addAll(objectsLists[OBJ_BLOB]); searchForReuse(monitor, tmp); if (pruneCurrentObjectList) { // If the list was pruned, we need to re-prune the main lists. pruneEdgesFromObjectList(objectsLists[OBJ_COMMIT]); pruneEdgesFromObjectList(objectsLists[OBJ_TREE]); pruneEdgesFromObjectList(objectsLists[OBJ_BLOB]); pruneEdgesFromObjectList(objectsLists[OBJ_TAG]); } } else { searchForReuse(monitor, objectsLists[OBJ_TAG]); searchForReuse(monitor, objectsLists[OBJ_COMMIT]); searchForReuse(monitor, objectsLists[OBJ_TREE]); searchForReuse(monitor, objectsLists[OBJ_BLOB]); } endPhase(monitor); stats.timeSearchingForReuse = System.currentTimeMillis() - start; if (config.isReuseDeltas() && config.getCutDeltaChains()) { cutDeltaChains(objectsLists[OBJ_TREE]); cutDeltaChains(objectsLists[OBJ_BLOB]); } } private void searchForReuse(ProgressMonitor monitor, List<ObjectToPack> list) throws IOException, MissingObjectException { pruneCurrentObjectList = false; reuseSupport.selectObjectRepresentation(this, monitor, list); if (pruneCurrentObjectList) pruneEdgesFromObjectList(list); } private void cutDeltaChains(BlockList<ObjectToPack> list) throws IOException { int max = config.getMaxDeltaDepth(); for (int idx = list.size() - 1; idx >= 0; idx--) { int d = 0; ObjectToPack b = list.get(idx).getDeltaBase(); while (b != null) { if (d < b.getChainLength()) break; b.setChainLength(++d); if (d >= max && b.isDeltaRepresentation()) { reselectNonDelta(b); break; } b = b.getDeltaBase(); } } if (config.isDeltaCompress()) { for (ObjectToPack otp : list) otp.clearChainLength(); } } private void searchForDeltas(ProgressMonitor monitor) throws MissingObjectException, IncorrectObjectTypeException, IOException { // Commits and annotated tags tend to have too many differences to // really benefit from delta compression. Consequently just don't // bother examining those types here. // ObjectToPack[] list = new ObjectToPack[objectsLists[OBJ_TREE].size() + objectsLists[OBJ_BLOB].size() + edgeObjects.size()]; int cnt = 0; cnt = findObjectsNeedingDelta(list, cnt, OBJ_TREE); cnt = findObjectsNeedingDelta(list, cnt, OBJ_BLOB); if (cnt == 0) return; int nonEdgeCnt = cnt; // Queue up any edge objects that we might delta against. We won't // be sending these as we assume the other side has them, but we need // them in the search phase below. // for (ObjectToPack eo : edgeObjects) { eo.setWeight(0); list[cnt++] = eo; } // Compute the sizes of the objects so we can do a proper sort. // We let the reader skip missing objects if it chooses. For // some readers this can be a huge win. We detect missing objects // by having set the weights above to 0 and allowing the delta // search code to discover the missing object and skip over it, or // abort with an exception if we actually had to have it. // final long sizingStart = System.currentTimeMillis(); beginPhase(PackingPhase.GETTING_SIZES, monitor, cnt); AsyncObjectSizeQueue<ObjectToPack> sizeQueue = reader .getObjectSize(Arrays.<ObjectToPack>asList(list).subList(0, cnt), false); try { final long limit = Math.min(config.getBigFileThreshold(), Integer.MAX_VALUE); for (;;) { try { if (!sizeQueue.next()) break; } catch (MissingObjectException notFound) { monitor.update(1); if (ignoreMissingUninteresting) { ObjectToPack otp = sizeQueue.getCurrent(); if (otp != null && otp.isEdge()) { otp.setDoNotDelta(); continue; } otp = objectsMap.get(notFound.getObjectId()); if (otp != null && otp.isEdge()) { otp.setDoNotDelta(); continue; } } throw notFound; } ObjectToPack otp = sizeQueue.getCurrent(); if (otp == null) otp = objectsMap.get(sizeQueue.getObjectId()); long sz = sizeQueue.getSize(); if (DeltaIndex.BLKSZ < sz && sz < limit) otp.setWeight((int) sz); else otp.setDoNotDelta(); // too small, or too big monitor.update(1); } } finally { sizeQueue.release(); } endPhase(monitor); stats.timeSearchingForSizes = System.currentTimeMillis() - sizingStart; // Sort the objects by path hash so like files are near each other, // and then by size descending so that bigger files are first. This // applies "Linus' Law" which states that newer files tend to be the // bigger ones, because source files grow and hardly ever shrink. // Arrays.sort(list, 0, cnt, (ObjectToPack a, ObjectToPack b) -> { int cmp = (a.isDoNotDelta() ? 1 : 0) - (b.isDoNotDelta() ? 1 : 0); if (cmp != 0) { return cmp; } cmp = a.getType() - b.getType(); if (cmp != 0) { return cmp; } cmp = (a.getPathHash() >>> 1) - (b.getPathHash() >>> 1); if (cmp != 0) { return cmp; } cmp = (a.getPathHash() & 1) - (b.getPathHash() & 1); if (cmp != 0) { return cmp; } cmp = (a.isEdge() ? 0 : 1) - (b.isEdge() ? 0 : 1); if (cmp != 0) { return cmp; } return b.getWeight() - a.getWeight(); }); // Above we stored the objects we cannot delta onto the end. // Remove them from the list so we don't waste time on them. while (0 < cnt && list[cnt - 1].isDoNotDelta()) { if (!list[cnt - 1].isEdge()) nonEdgeCnt--; cnt--; } if (cnt == 0) return; final long searchStart = System.currentTimeMillis(); searchForDeltas(monitor, list, cnt); stats.deltaSearchNonEdgeObjects = nonEdgeCnt; stats.timeCompressing = System.currentTimeMillis() - searchStart; for (int i = 0; i < cnt; i++) if (!list[i].isEdge() && list[i].isDeltaRepresentation()) stats.deltasFound++; } private int findObjectsNeedingDelta(ObjectToPack[] list, int cnt, int type) { for (ObjectToPack otp : objectsLists[type]) { if (otp.isDoNotDelta()) // delta is disabled for this path continue; if (otp.isDeltaRepresentation()) // already reusing a delta continue; otp.setWeight(0); list[cnt++] = otp; } return cnt; } private void reselectNonDelta(ObjectToPack otp) throws IOException { otp.clearDeltaBase(); otp.clearReuseAsIs(); boolean old = reuseDeltas; reuseDeltas = false; reuseSupport.selectObjectRepresentation(this, NullProgressMonitor.INSTANCE, Collections.singleton(otp)); reuseDeltas = old; } private void searchForDeltas(final ProgressMonitor monitor, final ObjectToPack[] list, final int cnt) throws MissingObjectException, IncorrectObjectTypeException, LargeObjectException, IOException { int threads = config.getThreads(); if (threads == 0) threads = Runtime.getRuntime().availableProcessors(); if (threads <= 1 || cnt <= config.getDeltaSearchWindowSize()) singleThreadDeltaSearch(monitor, list, cnt); else parallelDeltaSearch(monitor, list, cnt, threads); } private void singleThreadDeltaSearch(ProgressMonitor monitor, ObjectToPack[] list, int cnt) throws IOException { long totalWeight = 0; for (int i = 0; i < cnt; i++) { ObjectToPack o = list[i]; totalWeight += DeltaTask.getAdjustedWeight(o); } long bytesPerUnit = 1; while (DeltaTask.MAX_METER <= (totalWeight / bytesPerUnit)) bytesPerUnit <<= 10; int cost = (int) (totalWeight / bytesPerUnit); if (totalWeight % bytesPerUnit != 0) cost++; beginPhase(PackingPhase.COMPRESSING, monitor, cost); new DeltaWindow(config, new DeltaCache(config), reader, monitor, bytesPerUnit, list, 0, cnt).search(); endPhase(monitor); } private void parallelDeltaSearch(ProgressMonitor monitor, ObjectToPack[] list, int cnt, int threads) throws IOException { DeltaCache dc = new ThreadSafeDeltaCache(config); ThreadSafeProgressMonitor pm = new ThreadSafeProgressMonitor(monitor); DeltaTask.Block taskBlock = new DeltaTask.Block(threads, config, reader, dc, pm, list, 0, cnt); taskBlock.partitionTasks(); beginPhase(PackingPhase.COMPRESSING, monitor, taskBlock.cost()); pm.startWorkers(taskBlock.tasks.size()); Executor executor = config.getExecutor(); final List<Throwable> errors = Collections.synchronizedList(new ArrayList<>(threads)); if (executor instanceof ExecutorService) { // Caller supplied us a service, use it directly. runTasks((ExecutorService) executor, pm, taskBlock, errors); } else if (executor == null) { // Caller didn't give us a way to run the tasks, spawn up a // temporary thread pool and make sure it tears down cleanly. ExecutorService pool = Executors.newFixedThreadPool(threads); try { runTasks(pool, pm, taskBlock, errors); } finally { pool.shutdown(); for (;;) { try { if (pool.awaitTermination(60, TimeUnit.SECONDS)) break; } catch (InterruptedException e) { throw new IOException(JGitText.get().packingCancelledDuringObjectsWriting); } } } } else { // The caller gave us an executor, but it might not do // asynchronous execution. Wrap everything and hope it // can schedule these for us. for (DeltaTask task : taskBlock.tasks) { executor.execute(() -> { try { task.call(); } catch (Throwable failure) { errors.add(failure); } }); } try { pm.waitForCompletion(); } catch (InterruptedException ie) { // We can't abort the other tasks as we have no handle. // Cross our fingers and just break out anyway. // throw new IOException(JGitText.get().packingCancelledDuringObjectsWriting); } } // If any task threw an error, try to report it back as // though we weren't using a threaded search algorithm. // if (!errors.isEmpty()) { Throwable err = errors.get(0); if (err instanceof Error) throw (Error) err; if (err instanceof RuntimeException) throw (RuntimeException) err; if (err instanceof IOException) throw (IOException) err; throw new IOException(err.getMessage(), err); } endPhase(monitor); } private static void runTasks(ExecutorService pool, ThreadSafeProgressMonitor pm, DeltaTask.Block tb, List<Throwable> errors) throws IOException { List<Future<?>> futures = new ArrayList<>(tb.tasks.size()); for (DeltaTask task : tb.tasks) futures.add(pool.submit(task)); try { pm.waitForCompletion(); for (Future<?> f : futures) { try { f.get(); } catch (ExecutionException failed) { errors.add(failed.getCause()); } } } catch (InterruptedException ie) { for (Future<?> f : futures) f.cancel(true); throw new IOException(JGitText.get().packingCancelledDuringObjectsWriting); } } private void writeObjects(PackOutputStream out) throws IOException { writeObjects(out, objectsLists[OBJ_COMMIT]); writeObjects(out, objectsLists[OBJ_TAG]); writeObjects(out, objectsLists[OBJ_TREE]); writeObjects(out, objectsLists[OBJ_BLOB]); } private void writeObjects(PackOutputStream out, List<ObjectToPack> list) throws IOException { if (list.isEmpty()) return; typeStats = stats.objectTypes[list.get(0).getType()]; long beginOffset = out.length(); if (reuseSupport != null) { reuseSupport.writeObjects(out, list); } else { for (ObjectToPack otp : list) out.writeObject(otp); } typeStats.bytes += out.length() - beginOffset; typeStats.cntObjects = list.size(); } void writeObject(PackOutputStream out, ObjectToPack otp) throws IOException { if (!otp.isWritten()) writeObjectImpl(out, otp); } private void writeObjectImpl(PackOutputStream out, ObjectToPack otp) throws IOException { if (otp.wantWrite()) { // A cycle exists in this delta chain. This should only occur if a // selected object representation disappeared during writing // (for example due to a concurrent repack) and a different base // was chosen, forcing a cycle. Select something other than a // delta, and write this object. reselectNonDelta(otp); } otp.markWantWrite(); while (otp.isReuseAsIs()) { writeBase(out, otp.getDeltaBase()); if (otp.isWritten()) return; // Delta chain cycle caused this to write already. crc32.reset(); otp.setOffset(out.length()); try { reuseSupport.copyObjectAsIs(out, otp, reuseValidate); out.endObject(); otp.setCRC((int) crc32.getValue()); typeStats.reusedObjects++; if (otp.isDeltaRepresentation()) { typeStats.reusedDeltas++; typeStats.deltaBytes += out.length() - otp.getOffset(); } return; } catch (StoredObjectRepresentationNotAvailableException gone) { if (otp.getOffset() == out.length()) { otp.setOffset(0); otp.clearDeltaBase(); otp.clearReuseAsIs(); reuseSupport.selectObjectRepresentation(this, NullProgressMonitor.INSTANCE, Collections.singleton(otp)); continue; } // Object writing already started, we cannot recover. // CorruptObjectException coe; coe = new CorruptObjectException(otp, ""); //$NON-NLS-1$ coe.initCause(gone); throw coe; } } // If we reached here, reuse wasn't possible. // if (otp.isDeltaRepresentation()) { writeDeltaObjectDeflate(out, otp); } else { writeWholeObjectDeflate(out, otp); } out.endObject(); otp.setCRC((int) crc32.getValue()); } private void writeBase(PackOutputStream out, ObjectToPack base) throws IOException { if (base != null && !base.isWritten() && !base.isEdge()) writeObjectImpl(out, base); } private void writeWholeObjectDeflate(PackOutputStream out, final ObjectToPack otp) throws IOException { final Deflater deflater = deflater(); final ObjectLoader ldr = reader.open(otp, otp.getType()); crc32.reset(); otp.setOffset(out.length()); out.writeHeader(otp, ldr.getSize()); deflater.reset(); DeflaterOutputStream dst = new DeflaterOutputStream(out, deflater); ldr.copyTo(dst); dst.finish(); } private void writeDeltaObjectDeflate(PackOutputStream out, final ObjectToPack otp) throws IOException { writeBase(out, otp.getDeltaBase()); crc32.reset(); otp.setOffset(out.length()); DeltaCache.Ref ref = otp.popCachedDelta(); if (ref != null) { byte[] zbuf = ref.get(); if (zbuf != null) { out.writeHeader(otp, otp.getCachedSize()); out.write(zbuf); typeStats.cntDeltas++; typeStats.deltaBytes += out.length() - otp.getOffset(); return; } } try (TemporaryBuffer.Heap delta = delta(otp)) { out.writeHeader(otp, delta.length()); Deflater deflater = deflater(); deflater.reset(); DeflaterOutputStream dst = new DeflaterOutputStream(out, deflater); delta.writeTo(dst, null); dst.finish(); } typeStats.cntDeltas++; typeStats.deltaBytes += out.length() - otp.getOffset(); } private TemporaryBuffer.Heap delta(ObjectToPack otp) throws IOException { DeltaIndex index = new DeltaIndex(buffer(otp.getDeltaBaseId())); byte[] res = buffer(otp); // We never would have proposed this pair if the delta would be // larger than the unpacked version of the object. So using it // as our buffer limit is valid: we will never reach it. // TemporaryBuffer.Heap delta = new TemporaryBuffer.Heap(res.length); index.encode(delta, res); return delta; } private byte[] buffer(AnyObjectId objId) throws IOException { return buffer(config, reader, objId); } static byte[] buffer(PackConfig config, ObjectReader or, AnyObjectId objId) throws IOException { // PackWriter should have already pruned objects that // are above the big file threshold, so our chances of // the object being below it are very good. We really // shouldn't be here, unless the implementation is odd. return or.open(objId).getCachedBytes(config.getBigFileThreshold()); } private Deflater deflater() { if (myDeflater == null) myDeflater = new Deflater(config.getCompressionLevel()); return myDeflater; } private void writeChecksum(PackOutputStream out) throws IOException { packcsum = out.getDigest(); out.write(packcsum); } private void findObjectsToPack(@NonNull ProgressMonitor countingMonitor, @NonNull ObjectWalk walker, @NonNull Set<? extends ObjectId> want, @NonNull Set<? extends ObjectId> have, @NonNull Set<? extends ObjectId> noBitmaps) throws IOException { final long countingStart = System.currentTimeMillis(); beginPhase(PackingPhase.COUNTING, countingMonitor, ProgressMonitor.UNKNOWN); stats.interestingObjects = Collections.unmodifiableSet(new HashSet<ObjectId>(want)); stats.uninterestingObjects = Collections.unmodifiableSet(new HashSet<ObjectId>(have)); excludeFromBitmapSelection = noBitmaps; canBuildBitmaps = config.isBuildBitmaps() && !shallowPack && have.isEmpty() && (excludeInPacks == null || excludeInPacks.length == 0); if (!shallowPack && useBitmaps) { BitmapIndex bitmapIndex = reader.getBitmapIndex(); if (bitmapIndex != null) { BitmapWalker bitmapWalker = new BitmapWalker(walker, bitmapIndex, countingMonitor); findObjectsToPackUsingBitmaps(bitmapWalker, want, have); endPhase(countingMonitor); stats.timeCounting = System.currentTimeMillis() - countingStart; stats.bitmapIndexMisses = bitmapWalker.getCountOfBitmapIndexMisses(); return; } } List<ObjectId> all = new ArrayList<>(want.size() + have.size()); all.addAll(want); all.addAll(have); final RevFlag include = walker.newFlag("include"); //$NON-NLS-1$ final RevFlag added = walker.newFlag("added"); //$NON-NLS-1$ walker.carry(include); int haveEst = have.size(); if (have.isEmpty()) { walker.sort(RevSort.COMMIT_TIME_DESC); } else { walker.sort(RevSort.TOPO); if (thin) walker.sort(RevSort.BOUNDARY, true); } List<RevObject> wantObjs = new ArrayList<>(want.size()); List<RevObject> haveObjs = new ArrayList<>(haveEst); List<RevTag> wantTags = new ArrayList<>(want.size()); // Retrieve the RevWalk's versions of "want" and "have" objects to // maintain any state previously set in the RevWalk. AsyncRevObjectQueue q = walker.parseAny(all, true); try { for (;;) { try { RevObject o = q.next(); if (o == null) break; if (have.contains(o)) haveObjs.add(o); if (want.contains(o)) { o.add(include); wantObjs.add(o); if (o instanceof RevTag) wantTags.add((RevTag) o); } } catch (MissingObjectException e) { if (ignoreMissingUninteresting && have.contains(e.getObjectId())) continue; throw e; } } } finally { q.release(); } if (!wantTags.isEmpty()) { all = new ArrayList<>(wantTags.size()); for (RevTag tag : wantTags) all.add(tag.getObject()); q = walker.parseAny(all, true); try { while (q.next() != null) { // Just need to pop the queue item to parse the object. } } finally { q.release(); } } if (walker instanceof DepthWalk.ObjectWalk) { DepthWalk.ObjectWalk depthWalk = (DepthWalk.ObjectWalk) walker; for (RevObject obj : wantObjs) { depthWalk.markRoot(obj); } // Mark the tree objects associated with "have" commits as // uninteresting to avoid writing redundant blobs. A normal RevWalk // lazily propagates the "uninteresting" state from a commit to its // tree during the walk, but DepthWalks can terminate early so // preemptively propagate that state here. for (RevObject obj : haveObjs) { if (obj instanceof RevCommit) { RevTree t = ((RevCommit) obj).getTree(); depthWalk.markUninteresting(t); } } if (unshallowObjects != null) { for (ObjectId id : unshallowObjects) { depthWalk.markUnshallow(walker.parseAny(id)); } } } else { for (RevObject obj : wantObjs) walker.markStart(obj); } for (RevObject obj : haveObjs) walker.markUninteresting(obj); final int maxBases = config.getDeltaSearchWindowSize(); Set<RevTree> baseTrees = new HashSet<>(); BlockList<RevCommit> commits = new BlockList<>(); Set<ObjectId> roots = new HashSet<>(); RevCommit c; while ((c = walker.next()) != null) { if (exclude(c)) continue; if (c.has(RevFlag.UNINTERESTING)) { if (baseTrees.size() <= maxBases) baseTrees.add(c.getTree()); continue; } commits.add(c); if (c.getParentCount() == 0) { roots.add(c.copy()); } countingMonitor.update(1); } stats.rootCommits = Collections.unmodifiableSet(roots); if (shallowPack) { for (RevCommit cmit : commits) { addObject(cmit, 0); } } else { int commitCnt = 0; boolean putTagTargets = false; for (RevCommit cmit : commits) { if (!cmit.has(added)) { cmit.add(added); addObject(cmit, 0); commitCnt++; } for (int i = 0; i < cmit.getParentCount(); i++) { RevCommit p = cmit.getParent(i); if (!p.has(added) && !p.has(RevFlag.UNINTERESTING) && !exclude(p)) { p.add(added); addObject(p, 0); commitCnt++; } } if (!putTagTargets && 4096 < commitCnt) { for (ObjectId id : tagTargets) { RevObject obj = walker.lookupOrNull(id); if (obj instanceof RevCommit && obj.has(include) && !obj.has(RevFlag.UNINTERESTING) && !obj.has(added)) { obj.add(added); addObject(obj, 0); } } putTagTargets = true; } } } commits = null; if (thin && !baseTrees.isEmpty()) { BaseSearch bases = new BaseSearch(countingMonitor, baseTrees, // objectsMap, edgeObjects, reader); RevObject o; while ((o = walker.nextObject()) != null) { if (o.has(RevFlag.UNINTERESTING)) continue; if (exclude(o)) continue; int pathHash = walker.getPathHashCode(); byte[] pathBuf = walker.getPathBuffer(); int pathLen = walker.getPathLength(); bases.addBase(o.getType(), pathBuf, pathLen, pathHash); if (!depthSkip(o, walker)) { filterAndAddObject(o, o.getType(), pathHash, want); } countingMonitor.update(1); } } else { RevObject o; while ((o = walker.nextObject()) != null) { if (o.has(RevFlag.UNINTERESTING)) continue; if (exclude(o)) continue; if (!depthSkip(o, walker)) { filterAndAddObject(o, o.getType(), walker.getPathHashCode(), want); } countingMonitor.update(1); } } for (CachedPack pack : cachedPacks) countingMonitor.update((int) pack.getObjectCount()); endPhase(countingMonitor); stats.timeCounting = System.currentTimeMillis() - countingStart; stats.bitmapIndexMisses = -1; } private void findObjectsToPackUsingBitmaps(BitmapWalker bitmapWalker, Set<? extends ObjectId> want, Set<? extends ObjectId> have) throws MissingObjectException, IncorrectObjectTypeException, IOException { BitmapBuilder haveBitmap = bitmapWalker.findObjects(have, null, true); BitmapBuilder wantBitmap = bitmapWalker.findObjects(want, haveBitmap, false); BitmapBuilder needBitmap = wantBitmap.andNot(haveBitmap); if (useCachedPacks && reuseSupport != null && !reuseValidate && (excludeInPacks == null || excludeInPacks.length == 0)) cachedPacks.addAll(reuseSupport.getCachedPacksAndUpdate(needBitmap)); for (BitmapObject obj : needBitmap) { ObjectId objectId = obj.getObjectId(); if (exclude(objectId)) { needBitmap.remove(objectId); continue; } filterAndAddObject(objectId, obj.getType(), 0, want); } if (thin) haveObjects = haveBitmap; } private static void pruneEdgesFromObjectList(List<ObjectToPack> list) { final int size = list.size(); int src = 0; int dst = 0; for (; src < size; src++) { ObjectToPack obj = list.get(src); if (obj.isEdge()) continue; if (dst != src) list.set(dst, obj); dst++; } while (dst < list.size()) list.remove(list.size() - 1); } /** * Include one object to the output file. * <p> * Objects are written in the order they are added. If the same object is * added twice, it may be written twice, creating a larger than necessary * file. * * @param object * the object to add. * @throws org.eclipse.jgit.errors.IncorrectObjectTypeException * the object is an unsupported type. */ public void addObject(RevObject object) throws IncorrectObjectTypeException { if (!exclude(object)) addObject(object, 0); } private void addObject(RevObject object, int pathHashCode) { addObject(object, object.getType(), pathHashCode); } private void addObject(final AnyObjectId src, final int type, final int pathHashCode) { final ObjectToPack otp; if (reuseSupport != null) otp = reuseSupport.newObjectToPack(src, type); else otp = new ObjectToPack(src, type); otp.setPathHash(pathHashCode); objectsLists[type].add(otp); objectsMap.add(otp); } /** * Determines if the object should be omitted from the pack as a result of * its depth (probably because of the tree:<depth> filter). * <p> * Causes {@code walker} to skip traversing the current tree, which ought to * have just started traversal, assuming this method is called as soon as a * new depth is reached. * <p> * This method increments the {@code treesTraversed} statistic. * * @param obj * the object to check whether it should be omitted. * @param walker * the walker being used for traveresal. * @return whether the given object should be skipped. */ private boolean depthSkip(@NonNull RevObject obj, ObjectWalk walker) { long treeDepth = walker.getTreeDepth(); // Check if this object needs to be rejected because it is a tree or // blob that is too deep from the root tree. // A blob is considered one level deeper than the tree that contains it. if (obj.getType() == OBJ_BLOB) { treeDepth++; } else { stats.treesTraversed++; } if (filterSpec.getTreeDepthLimit() < 0 || treeDepth <= filterSpec.getTreeDepthLimit()) { return false; } walker.skipTree(); return true; } // Adds the given object as an object to be packed, first performing // filtering on blobs at or exceeding a given size. private void filterAndAddObject(@NonNull AnyObjectId src, int type, int pathHashCode, @NonNull Set<? extends AnyObjectId> want) throws IOException { // Check if this object needs to be rejected, doing the cheaper // checks first. boolean reject = filterSpec.getBlobLimit() >= 0 && type == OBJ_BLOB && !want.contains(src) && reader.getObjectSize(src, OBJ_BLOB) > filterSpec.getBlobLimit(); if (!reject) { addObject(src, type, pathHashCode); } } private boolean exclude(AnyObjectId objectId) { if (excludeInPacks == null) return false; if (excludeInPackLast.contains(objectId)) return true; for (ObjectIdSet idx : excludeInPacks) { if (idx.contains(objectId)) { excludeInPackLast = idx; return true; } } return false; } /** * Select an object representation for this writer. * <p> * An {@link org.eclipse.jgit.lib.ObjectReader} implementation should invoke * this method once for each representation available for an object, to * allow the writer to find the most suitable one for the output. * * @param otp * the object being packed. * @param next * the next available representation from the repository. */ public void select(ObjectToPack otp, StoredObjectRepresentation next) { int nFmt = next.getFormat(); if (!cachedPacks.isEmpty()) { if (otp.isEdge()) return; if (nFmt == PACK_WHOLE || nFmt == PACK_DELTA) { for (CachedPack pack : cachedPacks) { if (pack.hasObject(otp, next)) { otp.setEdge(); otp.clearDeltaBase(); otp.clearReuseAsIs(); pruneCurrentObjectList = true; return; } } } } if (nFmt == PACK_DELTA && reuseDeltas && reuseDeltaFor(otp)) { ObjectId baseId = next.getDeltaBase(); ObjectToPack ptr = objectsMap.get(baseId); if (ptr != null && !ptr.isEdge()) { otp.setDeltaBase(ptr); otp.setReuseAsIs(); } else if (thin && have(ptr, baseId)) { otp.setDeltaBase(baseId); otp.setReuseAsIs(); } else { otp.clearDeltaBase(); otp.clearReuseAsIs(); } } else if (nFmt == PACK_WHOLE && config.isReuseObjects()) { int nWeight = next.getWeight(); if (otp.isReuseAsIs() && !otp.isDeltaRepresentation()) { // We've chosen another PACK_WHOLE format for this object, // choose the one that has the smaller compressed size. // if (otp.getWeight() <= nWeight) return; } otp.clearDeltaBase(); otp.setReuseAsIs(); otp.setWeight(nWeight); } else { otp.clearDeltaBase(); otp.clearReuseAsIs(); } otp.setDeltaAttempted(reuseDeltas && next.wasDeltaAttempted()); otp.select(next); } private final boolean have(ObjectToPack ptr, AnyObjectId objectId) { return (ptr != null && ptr.isEdge()) || (haveObjects != null && haveObjects.contains(objectId)); } /** * Prepares the bitmaps to be written to the bitmap index file. * <p> * Bitmaps can be used to speed up fetches and clones by storing the entire * object graph at selected commits. Writing a bitmap index is an optional * feature that not all pack users may require. * <p> * Called after {@link #writeIndex(OutputStream)}. * <p> * To reduce memory internal state is cleared during this method, rendering * the PackWriter instance useless for anything further than a call to write * out the new bitmaps with {@link #writeBitmapIndex(OutputStream)}. * * @param pm * progress monitor to report bitmap building work. * @return whether a bitmap index may be written. * @throws java.io.IOException * when some I/O problem occur during reading objects. */ public boolean prepareBitmapIndex(ProgressMonitor pm) throws IOException { if (!canBuildBitmaps || getObjectCount() > Integer.MAX_VALUE || !cachedPacks.isEmpty()) return false; if (pm == null) pm = NullProgressMonitor.INSTANCE; int numCommits = objectsLists[OBJ_COMMIT].size(); List<ObjectToPack> byName = sortByName(); sortedByName = null; objectsLists = null; objectsMap = null; writeBitmaps = new PackBitmapIndexBuilder(byName); byName = null; PackWriterBitmapPreparer bitmapPreparer = new PackWriterBitmapPreparer(reader, writeBitmaps, pm, stats.interestingObjects, config); Collection<PackWriterBitmapPreparer.BitmapCommit> selectedCommits = bitmapPreparer.selectCommits(numCommits, excludeFromBitmapSelection); beginPhase(PackingPhase.BUILDING_BITMAPS, pm, selectedCommits.size()); BitmapWalker walker = bitmapPreparer.newBitmapWalker(); AnyObjectId last = null; for (PackWriterBitmapPreparer.BitmapCommit cmit : selectedCommits) { if (!cmit.isReuseWalker()) { walker = bitmapPreparer.newBitmapWalker(); } BitmapBuilder bitmap = walker.findObjects(Collections.singleton(cmit), null, false); if (last != null && cmit.isReuseWalker() && !bitmap.contains(last)) throw new IllegalStateException( MessageFormat.format(JGitText.get().bitmapMissingObject, cmit.name(), last.name())); last = cmit; writeBitmaps.addBitmap(cmit, bitmap.build(), cmit.getFlags()); pm.update(1); } endPhase(pm); return true; } private boolean reuseDeltaFor(ObjectToPack otp) { int type = otp.getType(); if ((type & 2) != 0) // OBJ_TREE(2) or OBJ_BLOB(3) return true; if (type == OBJ_COMMIT) return reuseDeltaCommits; if (type == OBJ_TAG) return false; return true; } private class MutableState { /** Estimated size of a single ObjectToPack instance. */ // Assume 64-bit pointers, since this is just an estimate. private static final long OBJECT_TO_PACK_SIZE = (2 * 8) // Object header + (2 * 8) + (2 * 8) // ObjectToPack fields + (8 + 8) // PackedObjectInfo fields + 8 // ObjectIdOwnerMap fields + 40 // AnyObjectId fields + 8; // Reference in BlockList private final long totalDeltaSearchBytes; private volatile PackingPhase phase; MutableState() { phase = PackingPhase.COUNTING; if (config.isDeltaCompress()) { int threads = config.getThreads(); if (threads <= 0) threads = Runtime.getRuntime().availableProcessors(); totalDeltaSearchBytes = (threads * config.getDeltaSearchMemoryLimit()) + config.getBigFileThreshold(); } else totalDeltaSearchBytes = 0; } State snapshot() { long objCnt = 0; BlockList<ObjectToPack>[] lists = objectsLists; if (lists != null) { objCnt += lists[OBJ_COMMIT].size(); objCnt += lists[OBJ_TREE].size(); objCnt += lists[OBJ_BLOB].size(); objCnt += lists[OBJ_TAG].size(); // Exclude CachedPacks. } long bytesUsed = OBJECT_TO_PACK_SIZE * objCnt; PackingPhase curr = phase; if (curr == PackingPhase.COMPRESSING) bytesUsed += totalDeltaSearchBytes; return new State(curr, bytesUsed); } } /** Possible states that a PackWriter can be in. */ public static enum PackingPhase { /** Counting objects phase. */ COUNTING, /** Getting sizes phase. */ GETTING_SIZES, /** Finding sources phase. */ FINDING_SOURCES, /** Compressing objects phase. */ COMPRESSING, /** Writing objects phase. */ WRITING, /** Building bitmaps phase. */ BUILDING_BITMAPS; } /** Summary of the current state of a PackWriter. */ public class State { private final PackingPhase phase; private final long bytesUsed; State(PackingPhase phase, long bytesUsed) { this.phase = phase; this.bytesUsed = bytesUsed; } /** @return the PackConfig used to build the writer. */ public PackConfig getConfig() { return config; } /** @return the current phase of the writer. */ public PackingPhase getPhase() { return phase; } /** @return an estimate of the total memory used by the writer. */ public long estimateBytesUsed() { return bytesUsed; } @SuppressWarnings("nls") @Override public String toString() { return "PackWriter.State[" + phase + ", memory=" + bytesUsed + "]"; } } /** * Configuration related to the packfile URI feature. * * @since 5.5 */ public static class PackfileUriConfig { @NonNull private final PacketLineOut pckOut; @NonNull private final Collection<String> protocolsSupported; @NonNull private final CachedPackUriProvider cachedPackUriProvider; /** * @param pckOut where to write "packfile-uri" lines to (should * output to the same stream as the one passed to * PackWriter#writePack) * @param protocolsSupported list of protocols supported (e.g. "https") * @param cachedPackUriProvider provider of URIs corresponding * to cached packs * @since 5.5 */ public PackfileUriConfig(@NonNull PacketLineOut pckOut, @NonNull Collection<String> protocolsSupported, @NonNull CachedPackUriProvider cachedPackUriProvider) { this.pckOut = pckOut; this.protocolsSupported = protocolsSupported; this.cachedPackUriProvider = cachedPackUriProvider; } } }