org.spearce.jgit/src/org/spearce/jgit/transport/FetchProcess.java

   1 /*
   2  * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com>
   3  * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
   4  *
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  * - Redistributions of source code must retain the above copyright
  12  *   notice, this list of conditions and the following disclaimer.
  13  *
  14  * - Redistributions in binary form must reproduce the above
  15  *   copyright notice, this list of conditions and the following
  16  *   disclaimer in the documentation and/or other materials provided
  17  *   with the distribution.
  18  *
  19  * - Neither the name of the Git Development Community nor the
  20  *   names of its contributors may be used to endorse or promote
  21  *   products derived from this software without specific prior
  22  *   written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  25  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  26  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  27  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  29  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  31  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  32  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  33  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  34  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  36  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37  */
  38
  39 package org.spearce.jgit.transport;
  40
  41 import java.io.File;
  42 import java.io.IOException;
  43 import java.io.OutputStreamWriter;
  44 import java.io.PrintWriter;
  45 import java.util.ArrayList;
  46 import java.util.Collection;
  47 import java.util.Collections;
  48 import java.util.HashMap;
  49 import java.util.HashSet;
  50 import java.util.Iterator;
  51 import java.util.Map;
  52 import java.util.Set;
  53
  54 import org.spearce.jgit.errors.MissingObjectException;
  55 import org.spearce.jgit.errors.NotSupportedException;
  56 import org.spearce.jgit.errors.TransportException;
  57 import org.spearce.jgit.lib.Constants;
  58 import org.spearce.jgit.lib.LockFile;
  59 import org.spearce.jgit.lib.ObjectId;
  60 import org.spearce.jgit.lib.ProgressMonitor;
  61 import org.spearce.jgit.lib.Ref;
  62 import org.spearce.jgit.lib.Repository;
  63 import org.spearce.jgit.revwalk.ObjectWalk;
  64 import org.spearce.jgit.revwalk.RevWalk;
  65
  66 class FetchProcess {
  67         /** Transport we will fetch over. */
  68         private final Transport transport;
  69
  70         /** List of things we want to fetch from the remote repository. */
  71         private final Collection<RefSpec> toFetch;
  72
  73         /** Set of refs we will actually wind up asking to obtain. */
  74         private final HashMap<ObjectId, Ref> askFor = new HashMap<ObjectId, Ref>();
  75
  76         /** Objects we know we have locally. */
  77         private final HashSet<ObjectId> have = new HashSet<ObjectId>();
  78
  79         /** Updates to local tracking branches (if any). */
  80         private final ArrayList<TrackingRefUpdate> localUpdates = new ArrayList<TrackingRefUpdate>();
  81
  82         /** Records to be recorded into FETCH_HEAD. */
  83         private final ArrayList<FetchHeadRecord> fetchHeadUpdates = new ArrayList<FetchHeadRecord>();
  84
  85         private FetchConnection conn;
  86
  87         FetchProcess(final Transport t, final Collection<RefSpec> f) {
  88                 transport = t;
  89                 toFetch = f;
  90         }
  91
  92         void execute(final ProgressMonitor monitor, final FetchResult result)
  93                         throws NotSupportedException, TransportException {
  94                 askFor.clear();
  95                 localUpdates.clear();
  96                 fetchHeadUpdates.clear();
  97
  98                 conn = transport.openFetch();
  99                 try {
 100                         result.setAdvertisedRefs(transport.getURI(), conn.getRefsMap());
 101                         final Set<Ref> matched = new HashSet<Ref>();
 102                         for (final RefSpec spec : toFetch) {
 103                                 if (spec.getSource() == null)
 104                                         throw new TransportException(
 105                                                         "Source ref not specified for refspec: " + spec);
 106
 107                                 if (spec.isWildcard())
 108                                         expandWildcard(spec, matched);
 109                                 else
 110                                         expandSingle(spec, matched);
 111                         }
 112
 113                         Collection<Ref> additionalTags = Collections.<Ref> emptyList();
 114                         final TagOpt tagopt = transport.getTagOpt();
 115                         if (tagopt == TagOpt.AUTO_FOLLOW)
 116                                 additionalTags = expandAutoFollowTags();
 117                         else if (tagopt == TagOpt.FETCH_TAGS)
 118                                 expandFetchTags();
 119
 120                         final boolean includedTags;
 121                         if (!askFor.isEmpty() && !askForIsComplete()) {
 122                                 fetchObjects(monitor);
 123                                 includedTags = conn.didFetchIncludeTags();
 124
 125                                 // Connection was used for object transfer. If we
 126                                 // do another fetch we must open a new connection.
 127                                 //
 128                                 closeConnection();
 129                         } else {
 130                                 includedTags = false;
 131                         }
 132
 133                         if (tagopt == TagOpt.AUTO_FOLLOW && !additionalTags.isEmpty()) {
 134                                 // There are more tags that we want to follow, but
 135                                 // not all were asked for on the initial request.
 136                                 //
 137                                 have.addAll(askFor.keySet());
 138                                 askFor.clear();
 139                                 for (final Ref r : additionalTags) {
 140                                         final ObjectId id = r.getPeeledObjectId();
 141                                         if (id == null || transport.local.hasObject(id))
 142                                                 wantTag(r);
 143                                 }
 144
 145                                 if (!askFor.isEmpty() && (!includedTags || !askForIsComplete())) {
 146                                         reopenConnection();
 147                                         if (!askFor.isEmpty())
 148                                                 fetchObjects(monitor);
 149                                 }
 150                         }
 151                 } finally {
 152                         closeConnection();
 153                 }
 154
 155                 final RevWalk walk = new RevWalk(transport.local);
 156                 if (transport.isRemoveDeletedRefs())
 157                         deleteStaleTrackingRefs(result, walk);
 158                 for (TrackingRefUpdate u : localUpdates) {
 159                         try {
 160                                 u.update(walk);
 161                                 result.add(u);
 162                         } catch (IOException err) {
 163                                 throw new TransportException("Failure updating tracking ref "
 164                                                 + u.getLocalName() + ": " + err.getMessage(), err);
 165                         }
 166                 }
 167
 168                 if (!fetchHeadUpdates.isEmpty()) {
 169                         try {
 170                                 updateFETCH_HEAD(result);
 171                         } catch (IOException err) {
 172                                 throw new TransportException("Failure updating FETCH_HEAD: "
 173                                                 + err.getMessage(), err);
 174                         }
 175                 }
 176         }
 177
 178         private void fetchObjects(final ProgressMonitor monitor)
 179                         throws TransportException {
 180                 conn.fetch(monitor, askFor.values(), have);
 181                 if (transport.isCheckFetchedObjects()
 182                                 && !conn.didFetchTestConnectivity() && !askForIsComplete())
 183                         throw new TransportException(transport.getURI(),
 184                                         "peer did not supply a complete object graph");
 185         }
 186
 187         private void closeConnection() {
 188                 if (conn != null) {
 189                         conn.close();
 190                         conn = null;
 191                 }
 192         }
 193
 194         private void reopenConnection() throws NotSupportedException,
 195                         TransportException {
 196                 if (conn != null)
 197                         return;
 198
 199                 conn = transport.openFetch();
 200
 201                 // Since we opened a new connection we cannot be certain
 202                 // that the system we connected to has the same exact set
 203                 // of objects available (think round-robin DNS and mirrors
 204                 // that aren't updated at the same time).
 205                 //
 206                 // We rebuild our askFor list using only the refs that the
 207                 // new connection has offered to us.
 208                 //
 209                 final HashMap<ObjectId, Ref> avail = new HashMap<ObjectId, Ref>();
 210                 for (final Ref r : conn.getRefs())
 211                         avail.put(r.getObjectId(), r);
 212
 213                 final Collection<Ref> wants = new ArrayList<Ref>(askFor.values());
 214                 askFor.clear();
 215                 for (final Ref want : wants) {
 216                         final Ref newRef = avail.get(want.getObjectId());
 217                         if (newRef != null) {
 218                                 askFor.put(newRef.getObjectId(), newRef);
 219                         } else {
 220                                 removeFetchHeadRecord(want.getObjectId());
 221                                 removeTrackingRefUpdate(want.getObjectId());
 222                         }
 223                 }
 224         }
 225
 226         private void removeTrackingRefUpdate(final ObjectId want) {
 227                 final Iterator<TrackingRefUpdate> i = localUpdates.iterator();
 228                 while (i.hasNext()) {
 229                         final TrackingRefUpdate u = i.next();
 230                         if (u.getNewObjectId().equals(want))
 231                                 i.remove();
 232                 }
 233         }
 234
 235         private void removeFetchHeadRecord(final ObjectId want) {
 236                 final Iterator<FetchHeadRecord> i = fetchHeadUpdates.iterator();
 237                 while (i.hasNext()) {
 238                         final FetchHeadRecord fh = i.next();
 239                         if (fh.newValue.equals(want))
 240                                 i.remove();
 241                 }
 242         }
 243
 244         private void updateFETCH_HEAD(final FetchResult result) throws IOException {
 245                 final LockFile lock = new LockFile(new File(transport.local
 246                                 .getDirectory(), "FETCH_HEAD"));
 247                 if (lock.lock()) {
 248                         final PrintWriter pw = new PrintWriter(new OutputStreamWriter(lock
 249                                         .getOutputStream())) {
 250                                 @Override
 251                                 public void println() {
 252                                         print('\n');
 253                                 }
 254                         };
 255                         for (final FetchHeadRecord h : fetchHeadUpdates) {
 256                                 h.write(pw);
 257                                 result.add(h);
 258                         }
 259                         pw.close();
 260                         lock.commit();
 261                 }
 262         }
 263
 264         private boolean askForIsComplete() throws TransportException {
 265                 try {
 266                         final ObjectWalk ow = new ObjectWalk(transport.local);
 267                         for (final ObjectId want : askFor.keySet())
 268                                 ow.markStart(ow.parseAny(want));
 269                         for (final Ref ref : transport.local.getAllRefs().values())
 270                                 ow.markUninteresting(ow.parseAny(ref.getObjectId()));
 271                         ow.checkConnectivity();
 272                         return true;
 273                 } catch (MissingObjectException e) {
 274                         return false;
 275                 } catch (IOException e) {
 276                         throw new TransportException("Unable to check connectivity.", e);
 277                 }
 278         }
 279
 280         private void expandWildcard(final RefSpec spec, final Set<Ref> matched)
 281                         throws TransportException {
 282                 for (final Ref src : conn.getRefs()) {
 283                         if (spec.matchSource(src) && matched.add(src))
 284                                 want(src, spec.expandFromSource(src));
 285                 }
 286         }
 287
 288         private void expandSingle(final RefSpec spec, final Set<Ref> matched)
 289                         throws TransportException {
 290                 final Ref src = conn.getRef(spec.getSource());
 291                 if (src == null) {
 292                         throw new TransportException("Remote does not have "
 293                                         + spec.getSource() + " available for fetch.");
 294                 }
 295                 if (matched.add(src))
 296                         want(src, spec);
 297         }
 298
 299         private Collection<Ref> expandAutoFollowTags() throws TransportException {
 300                 final Collection<Ref> additionalTags = new ArrayList<Ref>();
 301                 final Map<String, Ref> haveRefs = transport.local.getAllRefs();
 302                 for (final Ref r : conn.getRefs()) {
 303                         if (!isTag(r))
 304                                 continue;
 305                         if (r.getPeeledObjectId() == null) {
 306                                 additionalTags.add(r);
 307                                 continue;
 308                         }
 309
 310                         final Ref local = haveRefs.get(r.getName());
 311                         if (local != null) {
 312                                 if (!r.getObjectId().equals(local.getObjectId()))
 313                                         wantTag(r);
 314                         } else if (askFor.containsKey(r.getPeeledObjectId())
 315                                         || transport.local.hasObject(r.getPeeledObjectId()))
 316                                 wantTag(r);
 317                         else
 318                                 additionalTags.add(r);
 319                 }
 320                 return additionalTags;
 321         }
 322
 323         private void expandFetchTags() throws TransportException {
 324                 final Map<String, Ref> haveRefs = transport.local.getAllRefs();
 325                 for (final Ref r : conn.getRefs()) {
 326                         if (!isTag(r))
 327                                 continue;
 328                         final Ref local = haveRefs.get(r.getName());
 329                         if (local == null || !r.getObjectId().equals(local.getObjectId()))
 330                                 wantTag(r);
 331                 }
 332         }
 333
 334         private void wantTag(final Ref r) throws TransportException {
 335                 want(r, new RefSpec().setSource(r.getName())
 336                                 .setDestination(r.getName()));
 337         }
 338
 339         private void want(final Ref src, final RefSpec spec)
 340                         throws TransportException {
 341                 final ObjectId newId = src.getObjectId();
 342                 if (spec.getDestination() != null) {
 343                         try {
 344                                 final TrackingRefUpdate tru = createUpdate(spec, newId);
 345                                 if (newId.equals(tru.getOldObjectId()))
 346                                         return;
 347                                 localUpdates.add(tru);
 348                         } catch (IOException err) {
 349                                 // Bad symbolic ref? That is the most likely cause.
 350                                 //
 351                                 throw new TransportException("Cannot resolve"
 352                                                 + " local tracking ref " + spec.getDestination()
 353                                                 + " for updating.", err);
 354                         }
 355                 }
 356
 357                 askFor.put(newId, src);
 358
 359                 final FetchHeadRecord fhr = new FetchHeadRecord();
 360                 fhr.newValue = newId;
 361                 fhr.notForMerge = spec.getDestination() != null;
 362                 fhr.sourceName = src.getName();
 363                 fhr.sourceURI = transport.getURI();
 364                 fetchHeadUpdates.add(fhr);
 365         }
 366
 367         private TrackingRefUpdate createUpdate(final RefSpec spec,
 368                         final ObjectId newId) throws IOException {
 369                 return new TrackingRefUpdate(transport.local, spec, newId, "fetch");
 370         }
 371
 372         private void deleteStaleTrackingRefs(final FetchResult result,
 373                         final RevWalk walk) throws TransportException {
 374                 final Repository db = transport.local;
 375                 for (final Ref ref : db.getAllRefs().values()) {
 376                         final String refname = ref.getName();
 377                         for (final RefSpec spec : toFetch) {
 378                                 if (spec.matchDestination(refname)) {
 379                                         final RefSpec s = spec.expandFromDestination(refname);
 380                                         if (result.getAdvertisedRef(s.getSource()) == null) {
 381                                                 deleteTrackingRef(result, db, walk, s, ref);
 382                                         }
 383                                 }
 384                         }
 385                 }
 386         }
 387
 388         private void deleteTrackingRef(final FetchResult result,
 389                         final Repository db, final RevWalk walk, final RefSpec spec,
 390                         final Ref localRef) throws TransportException {
 391                 final String name = localRef.getName();
 392                 try {
 393                         final TrackingRefUpdate u = new TrackingRefUpdate(db, name, spec
 394                                         .getSource(), true, ObjectId.zeroId(), "deleted");
 395                         result.add(u);
 396                         if (transport.isDryRun()){
 397                                 return;
 398                         }
 399                         u.delete(walk);
 400                         switch (u.getResult()) {
 401                         case NEW:
 402                         case NO_CHANGE:
 403                         case FAST_FORWARD:
 404                         case FORCED:
 405                                 break;
 406                         default:
 407                                 throw new TransportException(transport.getURI(),
 408                                                 "Cannot delete stale tracking ref " + name + ": "
 409                                                                 + u.getResult().name());
 410                         }
 411                 } catch (IOException e) {
 412                         throw new TransportException(transport.getURI(),
 413                                         "Cannot delete stale tracking ref " + name, e);
 414                 }
 415         }
 416
 417         private static boolean isTag(final Ref r) {
 418                 return isTag(r.getName());
 419         }
 420
 421         private static boolean isTag(final String name) {
 422                 return name.startsWith(Constants.R_TAGS);
 423         }
 424 }