From c8aa19f39142c31d98492696329bcc13d50c04c7 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 17 May 2008 00:00:48 -0400 Subject: [PATCH] Implement fetch support for TagOpt AUTO_FOLLOW, FETCH_TAGS If tagopt == TagOpt.FETCH_TAGS we want all of the tags the remote has, even if that means getting objects we do not yet have and would not have requested otherwise in our want list. This case is quite trivial as we just need to add any tag object which is not already in our local repository and which is not in our ask list to the list of things we want. The transport will get everything in a single shot. If tagopt == TagOpt.AUTO_FOLLOW things are a little bit more complex. Here we want to get an annotated tag only if we do not already have it and it peels to something we will have when the fetch is complete. For cases where a branch we want is pointing at the peeled object for a tag the branch is in our ask list, so we can easily see that the referred object will be available when the fetch completes. Adding in a request for the tag is quite cheap, as only the tag needs to be transferred and we want that anyway. This may allow us to avoid a second network connection to acquire the tags after the branch objects are obtained. If we aren't asking for the tag's peeled object, but we already have it locally in our repository, chances are its reachable from our existing local refs. Asking for the tag is not likely to cause additional network transfer time. This case can happen if a maintainer has recently tagged a historical revision, one that we have previously fetched. If the FetchConnection is able to automatically follow tags (such as due to the native protocol "include-tags" extension) we may be able to complete the fetch in only one connection by checking to see if the objects are complete after the fetch. If they are then we don't need any further network transport. However there is no point in testing for completeness of tags we do not have (but want) if the transport did not magically include them in our last fetch call. If we have to open a second connection for tag fetch we may not be talking to the same repository. This can easily happen on busy public mirror sites that utilize round-robin DNS. To avoid asking for objects that the remote does not advertise (as perhaps the one mirror is ahead of or behind the other) we correct our state to reflect the new connection's advertised refs. Signed-off-by: Shawn O. Pearce --- .../org/spearce/jgit/transport/FetchProcess.java | 171 +++++++++++++++++++-- .../src/org/spearce/jgit/transport/Transport.java | 2 +- 2 files changed, 160 insertions(+), 13 deletions(-) diff --git a/org.spearce.jgit/src/org/spearce/jgit/transport/FetchProcess.java b/org.spearce.jgit/src/org/spearce/jgit/transport/FetchProcess.java index 6475e285..f501bf53 100644 --- a/org.spearce.jgit/src/org/spearce/jgit/transport/FetchProcess.java +++ b/org.spearce.jgit/src/org/spearce/jgit/transport/FetchProcess.java @@ -22,13 +22,17 @@ import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; import java.util.Set; import org.spearce.jgit.errors.MissingObjectException; import org.spearce.jgit.errors.NotSupportedException; import org.spearce.jgit.errors.TransportException; +import org.spearce.jgit.lib.Constants; import org.spearce.jgit.lib.LockFile; import org.spearce.jgit.lib.ObjectId; import org.spearce.jgit.lib.ProgressMonitor; @@ -43,6 +47,9 @@ class FetchProcess { /** List of things we want to fetch from the remote repository. */ private final Collection toFetch; + /** How to handle annotated tags, if any are advertised. */ + private final TagOpt tagopt; + /** Set of refs we will actually wind up asking to obtain. */ private final HashMap askFor = new HashMap(); @@ -52,15 +59,16 @@ class FetchProcess { /** Records to be recorded into FETCH_HEAD. */ private final ArrayList fetchHeadUpdates = new ArrayList(); - FetchProcess(final Transport t, final Collection f) { + private FetchConnection conn; + + FetchProcess(final Transport t, final Collection f, final TagOpt o) { transport = t; toFetch = f; + tagopt = o; } void execute(final ProgressMonitor monitor, final FetchResult result) throws NotSupportedException, TransportException { - FetchConnection conn; - askFor.clear(); localUpdates.clear(); fetchHeadUpdates.clear(); @@ -71,15 +79,49 @@ class FetchProcess { final Set matched = new HashSet(); for (final RefSpec spec : toFetch) { if (spec.isWildcard()) - expandWildcard(conn, spec, matched); + expandWildcard(spec, matched); else - expandSingle(conn, spec, matched); + expandSingle(spec, matched); } - if (!askFor.isEmpty() && !askForIsComplete()) + + Collection additionalTags = Collections. emptyList(); + if (tagopt == TagOpt.AUTO_FOLLOW) + additionalTags = expandAutoFollowTags(); + else if (tagopt == TagOpt.FETCH_TAGS) + expandFetchTags(); + + final boolean includedTags; + if (!askFor.isEmpty() && !askForIsComplete()) { conn.fetch(monitor, askFor.values()); + includedTags = conn.didFetchIncludeTags(); + + // Connection was used for object transfer. If we + // do another fetch we must open a new connection. + // + closeConnection(); + } else { + includedTags = false; + } + + if (tagopt == TagOpt.AUTO_FOLLOW && !additionalTags.isEmpty()) { + // There are more tags that we want to follow, but + // not all were asked for on the initial request. + // + askFor.clear(); + for (final Ref r : additionalTags) { + final ObjectId id = r.getPeeledObjectId(); + if (id == null || transport.local.hasObject(id)) + wantTag(r); + } + + if (!askFor.isEmpty() && (!includedTags || !askForIsComplete())) { + reopenConnection(); + if (!askFor.isEmpty()) + conn.doFetch(monitor, askFor.values()); + } + } } finally { - conn.close(); - conn = null; + closeConnection(); } final RevWalk walk = new RevWalk(transport.local); @@ -103,6 +145,63 @@ class FetchProcess { } } + private void closeConnection() { + if (conn != null) { + conn.close(); + conn = null; + } + } + + private void reopenConnection() throws NotSupportedException, + TransportException { + if (conn != null) + return; + + conn = transport.openFetch(); + + // Since we opened a new connection we cannot be certain + // that the system we connected to has the same exact set + // of objects available (think round-robin DNS and mirrors + // that aren't updated at the same time). + // + // We rebuild our askFor list using only the refs that the + // new connection has offered to us. + // + final HashMap avail = new HashMap(); + for (final Ref r : conn.getRefs()) + avail.put(r.getObjectId(), r); + + final Collection wants = new ArrayList(askFor.values()); + askFor.clear(); + for (final Ref want : wants) { + final Ref newRef = avail.get(want.getObjectId()); + if (newRef != null) { + askFor.put(newRef.getObjectId(), newRef); + } else { + removeFetchHeadRecord(want.getObjectId()); + removeTrackingRefUpdate(want.getObjectId()); + } + } + } + + private void removeTrackingRefUpdate(final ObjectId want) { + final Iterator i = localUpdates.iterator(); + while (i.hasNext()) { + final TrackingRefUpdate u = i.next(); + if (u.getNewObjectId().equals(want)) + i.remove(); + } + } + + private void removeFetchHeadRecord(final ObjectId want) { + final Iterator i = fetchHeadUpdates.iterator(); + while (i.hasNext()) { + final FetchHeadRecord fh = i.next(); + if (fh.newValue.equals(want)) + i.remove(); + } + } + private void updateFETCH_HEAD(final FetchResult result) throws IOException { final LockFile lock = new LockFile(new File(transport.local .getDirectory(), "FETCH_HEAD")); @@ -139,16 +238,16 @@ class FetchProcess { } } - private void expandWildcard(final FetchConnection conn, final RefSpec spec, - final Set matched) throws TransportException { + private void expandWildcard(final RefSpec spec, final Set matched) + throws TransportException { for (final Ref src : conn.getRefs()) { if (spec.matchSource(src) && matched.add(src)) want(src, spec.expandFromSource(src)); } } - private void expandSingle(final FetchConnection conn, final RefSpec spec, - final Set matched) throws TransportException { + private void expandSingle(final RefSpec spec, final Set matched) + throws TransportException { final Ref src = conn.getRef(spec.getSource()); if (src == null) { throw new TransportException("Remote does not have " @@ -158,6 +257,46 @@ class FetchProcess { want(src, spec); } + private Collection expandAutoFollowTags() throws TransportException { + final Collection additionalTags = new ArrayList(); + final Map have = transport.local.getAllRefs(); + for (final Ref r : conn.getRefs()) { + if (!isTag(r)) + continue; + if (r.getPeeledObjectId() == null) { + additionalTags.add(r); + continue; + } + + final Ref local = have.get(r.getName()); + if (local != null) { + if (!r.getObjectId().equals(local.getObjectId())) + wantTag(r); + } else if (askFor.containsKey(r.getPeeledObjectId()) + || transport.local.hasObject(r.getPeeledObjectId())) + wantTag(r); + else + additionalTags.add(r); + } + return additionalTags; + } + + private void expandFetchTags() throws TransportException { + final Map have = transport.local.getAllRefs(); + for (final Ref r : conn.getRefs()) { + if (!isTag(r)) + continue; + final Ref local = have.get(r.getName()); + if (local == null || !r.getObjectId().equals(local.getObjectId())) + wantTag(r); + } + } + + private void wantTag(final Ref r) throws TransportException { + want(r, new RefSpec().setSource(r.getName()) + .setDestination(r.getName())); + } + private void want(final Ref src, final RefSpec spec) throws TransportException { final ObjectId newId = src.getObjectId(); @@ -190,4 +329,12 @@ class FetchProcess { final ObjectId newId) throws IOException { return new TrackingRefUpdate(transport.local, spec, newId, "fetch"); } + + private static boolean isTag(final Ref r) { + return isTag(r.getName()); + } + + private static boolean isTag(final String name) { + return name.startsWith(Constants.TAGS_PREFIX + "/"); + } } diff --git a/org.spearce.jgit/src/org/spearce/jgit/transport/Transport.java b/org.spearce.jgit/src/org/spearce/jgit/transport/Transport.java index 565b2a9c..85d69af6 100644 --- a/org.spearce.jgit/src/org/spearce/jgit/transport/Transport.java +++ b/org.spearce.jgit/src/org/spearce/jgit/transport/Transport.java @@ -266,7 +266,7 @@ public abstract class Transport { } final FetchResult result = new FetchResult(); - new FetchProcess(this, toFetch).execute(monitor, result); + new FetchProcess(this, toFetch, tagopt).execute(monitor, result); return result; } -- 2.11.4.GIT