2 * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com>
3 * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
7 * Redistribution and use in source and binary forms, with or
8 * without modification, are permitted provided that the following
11 * - Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
14 * - Redistributions in binary form must reproduce the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer in the documentation and/or other materials provided
17 * with the distribution.
19 * - Neither the name of the Git Development Community nor the
20 * names of its contributors may be used to endorse or promote
21 * products derived from this software without specific prior
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
25 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
26 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
27 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
29 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
31 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
32 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
33 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
34 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
36 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 package org
.spearce
.jgit
.transport
;
41 import java
.io
.BufferedInputStream
;
42 import java
.io
.BufferedOutputStream
;
43 import java
.io
.EOFException
;
44 import java
.io
.IOException
;
45 import java
.io
.InputStream
;
46 import java
.io
.OutputStream
;
47 import java
.util
.Collection
;
48 import java
.util
.Date
;
49 import java
.util
.HashSet
;
50 import java
.util
.LinkedHashMap
;
53 import org
.spearce
.jgit
.errors
.PackProtocolException
;
54 import org
.spearce
.jgit
.errors
.TransportException
;
55 import org
.spearce
.jgit
.lib
.AnyObjectId
;
56 import org
.spearce
.jgit
.lib
.MutableObjectId
;
57 import org
.spearce
.jgit
.lib
.ObjectId
;
58 import org
.spearce
.jgit
.lib
.ProgressMonitor
;
59 import org
.spearce
.jgit
.lib
.Ref
;
60 import org
.spearce
.jgit
.lib
.Repository
;
61 import org
.spearce
.jgit
.revwalk
.RevCommit
;
62 import org
.spearce
.jgit
.revwalk
.RevCommitList
;
63 import org
.spearce
.jgit
.revwalk
.RevFlag
;
64 import org
.spearce
.jgit
.revwalk
.RevObject
;
65 import org
.spearce
.jgit
.revwalk
.RevSort
;
66 import org
.spearce
.jgit
.revwalk
.RevWalk
;
67 import org
.spearce
.jgit
.revwalk
.filter
.CommitTimeRevFilter
;
68 import org
.spearce
.jgit
.revwalk
.filter
.RevFilter
;
71 * Fetch implementation using the native Git pack transfer service.
73 * This is the canonical implementation for transferring objects from the remote
74 * repository to the local repository by talking to the 'git-upload-pack'
75 * service. Objects are packed on the remote side into a pack file and then sent
76 * down the pipe to us.
78 * This connection requires only a bi-directional pipe or socket, and thus is
79 * easily wrapped up into a local process pipe, anonymous TCP socket, or a
80 * command executed through an SSH tunnel.
82 abstract class PackFetchConnection
extends FetchConnection
{
84 * Maximum number of 'have' lines to send before giving up.
86 * During {@link #negotiate(ProgressMonitor)} we send at most this many
87 * commits to the remote peer as 'have' lines without an ACK response before
90 private static final int MAX_HAVES
= 256;
92 static final String OPTION_INCLUDE_TAG
= "include-tag";
94 static final String OPTION_MULTI_ACK
= "multi_ack";
96 static final String OPTION_THIN_PACK
= "thin-pack";
98 static final String OPTION_SIDE_BAND
= "side-band";
100 static final String OPTION_SIDE_BAND_64K
= "side-band-64k";
102 static final String OPTION_OFS_DELTA
= "ofs-delta";
104 static final String OPTION_SHALLOW
= "shallow";
106 /** The repository this transport fetches into, or pushes out of. */
107 protected final Repository local
;
109 /** Remote repository location. */
110 protected final URIish uri
;
112 /** Capability tokens advertised by the remote side. */
113 protected final Set
<String
> remoteCapablities
= new HashSet
<String
>();
115 /** Buffered input stream reading from the remote. */
116 protected InputStream in
;
118 /** Buffered output stream sending to the remote. */
119 protected OutputStream out
;
121 /** Packet line decoder around {@link #in}. */
122 protected PacketLineIn pckIn
;
124 /** Packet line encoder around {@link #out}. */
125 protected PacketLineOut pckOut
;
127 private final RevWalk walk
;
129 /** All commits that are immediately reachable by a local ref. */
130 private RevCommitList
<RevCommit
> reachableCommits
;
132 /** Marks an object as having all its dependencies. */
133 final RevFlag REACHABLE
;
135 /** Marks a commit known to both sides of the connection. */
136 final RevFlag COMMON
;
138 /** Marks a commit listed in the advertised refs. */
139 final RevFlag ADVERTISED
;
141 private boolean multiAck
;
143 private boolean thinPack
;
145 private boolean sideband
;
147 private boolean includeTags
;
149 PackFetchConnection(final PackTransport packTransport
) {
150 local
= packTransport
.local
;
151 uri
= packTransport
.uri
;
152 includeTags
= packTransport
.getTagOpt() != TagOpt
.NO_TAGS
;
154 walk
= new RevWalk(local
);
155 reachableCommits
= new RevCommitList
<RevCommit
>();
156 REACHABLE
= walk
.newFlag("REACHABLE");
157 COMMON
= walk
.newFlag("COMMON");
158 ADVERTISED
= walk
.newFlag("ADVERTISED");
161 walk
.carry(REACHABLE
);
162 walk
.carry(ADVERTISED
);
165 protected void init(final InputStream myIn
, final OutputStream myOut
) {
166 in
= myIn
instanceof BufferedInputStream ? myIn
167 : new BufferedInputStream(myIn
);
168 out
= myOut
instanceof BufferedOutputStream ? myOut
169 : new BufferedOutputStream(myOut
);
171 pckIn
= new PacketLineIn(in
);
172 pckOut
= new PacketLineOut(out
);
176 public boolean didFetchIncludeTags() {
180 protected void readAdvertisedRefs() throws TransportException
{
182 readAdvertisedRefsImpl();
183 } catch (TransportException err
) {
186 } catch (IOException err
) {
188 throw new TransportException(err
.getMessage(), err
);
189 } catch (RuntimeException err
) {
191 throw new TransportException(err
.getMessage(), err
);
195 private void readAdvertisedRefsImpl() throws IOException
{
196 final LinkedHashMap
<String
, Ref
> avail
= new LinkedHashMap
<String
, Ref
>();
201 line
= pckIn
.readString();
202 } catch (EOFException eof
) {
204 throw new TransportException(uri
+ " not found.");
208 if (avail
.isEmpty()) {
209 // The first line (if any) may contain "hidden"
210 // capability values after a NUL byte.
212 final int nul
= line
.indexOf('\0');
214 for (String c
: line
.substring(nul
+ 1).split(" "))
215 remoteCapablities
.add(c
);
216 line
= line
.substring(0, nul
);
220 if (line
.length() == 0)
223 String name
= line
.substring(41, line
.length());
224 final ObjectId id
= ObjectId
.fromString(line
.substring(0, 40));
225 if (name
.endsWith("^{}")) {
226 name
= name
.substring(0, name
.length() - 3);
227 final Ref prior
= avail
.get(name
);
229 throw new PackProtocolException("advertisement of " + name
230 + "^{} came before " + name
);
232 if (prior
.getPeeledObjectId() != null)
233 throw duplicateAdvertisement(name
+ "^{}");
235 avail
.put(name
, new Ref(name
, prior
.getObjectId(), id
));
237 final Ref prior
= avail
.put(name
, new Ref(name
, id
));
239 throw duplicateAdvertisement(name
);
245 private PackProtocolException
duplicateAdvertisement(final String name
) {
246 return new PackProtocolException("duplicate advertisements of " + name
);
250 protected void doFetch(final ProgressMonitor monitor
,
251 final Collection
<Ref
> want
) throws TransportException
{
253 markRefsAdvertised();
254 markReachable(maxTimeWanted(want
));
256 if (sendWants(want
)) {
260 reachableCommits
= null;
262 receivePack(monitor
);
264 } catch (CancelledException ce
) {
266 return; // Caller should test (or just know) this themselves.
267 } catch (IOException err
) {
269 throw new TransportException(err
.getMessage(), err
);
270 } catch (RuntimeException err
) {
272 throw new TransportException(err
.getMessage(), err
);
276 private int maxTimeWanted(final Collection
<Ref
> wants
) {
278 for (final Ref r
: wants
) {
280 final RevObject obj
= walk
.parseAny(r
.getObjectId());
281 if (obj
instanceof RevCommit
) {
282 final int cTime
= ((RevCommit
) obj
).getCommitTime();
286 } catch (IOException error
) {
287 // We don't have it, but we want to fetch (thus fixing error).
293 private void markReachable(final int maxTime
) throws IOException
{
294 for (final Ref r
: local
.getAllRefs().values()) {
296 final RevCommit o
= walk
.parseCommit(r
.getObjectId());
298 reachableCommits
.add(o
);
299 } catch (IOException readError
) {
300 // If we cannot read the value of the ref skip it.
301 } catch (ClassCastException cce
) {
302 // Not a commit type.
307 // Mark reachable commits until we reach maxTime. These may
308 // wind up later matching up against things we want and we
309 // can avoid asking for something we already happen to have.
311 final Date maxWhen
= new Date(maxTime
* 1000L);
312 walk
.sort(RevSort
.COMMIT_TIME_DESC
);
313 walk
.markStart(reachableCommits
);
314 walk
.setRevFilter(CommitTimeRevFilter
.after(maxWhen
));
316 final RevCommit c
= walk
.next();
319 if (c
.has(ADVERTISED
) && !c
.has(COMMON
)) {
320 // This is actually going to be a common commit, but
321 // our peer doesn't know that fact yet.
325 reachableCommits
.add(c
);
331 private boolean sendWants(final Collection
<Ref
> want
) throws IOException
{
332 boolean first
= true;
333 for (final Ref r
: want
) {
335 if (walk
.parseAny(r
.getObjectId()).has(REACHABLE
)) {
336 // We already have this object. Asking for it is
337 // not a very good idea.
341 } catch (IOException err
) {
342 // Its OK, we don't have it, but we want to fix that
343 // by fetching the object from the other side.
346 final StringBuilder line
= new StringBuilder(46);
347 line
.append("want ");
348 line
.append(r
.getObjectId());
350 line
.append(enableCapabilities());
354 pckOut
.writeString(line
.toString());
360 private String
enableCapabilities() {
361 final StringBuilder line
= new StringBuilder();
363 includeTags
= wantCapability(line
, OPTION_INCLUDE_TAG
);
364 wantCapability(line
, OPTION_OFS_DELTA
);
365 multiAck
= wantCapability(line
, OPTION_MULTI_ACK
);
366 thinPack
= wantCapability(line
, OPTION_THIN_PACK
);
367 if (wantCapability(line
, OPTION_SIDE_BAND_64K
))
369 else if (wantCapability(line
, OPTION_SIDE_BAND
))
371 return line
.toString();
374 private boolean wantCapability(final StringBuilder b
, final String option
) {
375 if (!remoteCapablities
.contains(option
))
383 private void negotiate(final ProgressMonitor monitor
) throws IOException
,
385 final MutableObjectId ackId
= new MutableObjectId();
386 int resultsPending
= 0;
388 int havesSinceLastContinue
= 0;
389 boolean receivedContinue
= false;
390 boolean receivedAck
= false;
391 boolean sendHaves
= true;
395 final RevCommit c
= walk
.next();
399 pckOut
.writeString("have " + c
.getId() + "\n");
401 havesSinceLastContinue
++;
403 if ((31 & havesSent
) != 0) {
404 // We group the have lines into blocks of 32, each marked
405 // with a flush (aka end). This one is within a block so
406 // continue with another have line.
411 if (monitor
.isCancelled())
412 throw new CancelledException();
415 resultsPending
++; // Each end will cause a result to come back.
417 if (havesSent
== 32) {
418 // On the first block we race ahead and try to send
419 // more of the second block while waiting for the
420 // remote to respond to our first block request.
421 // This keeps us one block ahead of the peer.
426 while (resultsPending
> 0) {
427 final PacketLineIn
.AckNackResult anr
;
429 anr
= pckIn
.readACK(ackId
);
431 if (anr
== PacketLineIn
.AckNackResult
.NAK
) {
432 // More have lines are necessary to compute the
433 // pack on the remote side. Keep doing that.
438 if (anr
== PacketLineIn
.AckNackResult
.ACK
) {
439 // The remote side is happy and knows exactly what
440 // to send us. There is no further negotiation and
441 // we can break out immediately.
450 if (anr
== PacketLineIn
.AckNackResult
.ACK_CONTINUE
) {
451 // The server knows this commit (ackId). We don't
452 // need to send any further along its ancestry, but
453 // we need to continue to talk about other parts of
454 // our local history.
456 markCommon(walk
.parseAny(ackId
));
458 receivedContinue
= true;
459 havesSinceLastContinue
= 0;
462 if (monitor
.isCancelled())
463 throw new CancelledException();
466 if (receivedContinue
&& havesSinceLastContinue
> MAX_HAVES
) {
467 // Our history must be really different from the remote's.
468 // We just sent a whole slew of have lines, and it did not
469 // recognize any of them. Avoid sending our entire history
470 // to them by giving up early.
476 // Tell the remote side we have run out of things to talk about.
478 if (monitor
.isCancelled())
479 throw new CancelledException();
480 pckOut
.writeString("done\n");
484 // Apparently if we have never received an ACK earlier
485 // there is one more result expected from the done we
486 // just sent to the remote.
492 while (resultsPending
> 0 || multiAck
) {
493 final PacketLineIn
.AckNackResult anr
;
495 anr
= pckIn
.readACK(ackId
);
498 if (anr
== PacketLineIn
.AckNackResult
.ACK
)
499 break; // commit negotiation is finished.
501 if (anr
== PacketLineIn
.AckNackResult
.ACK_CONTINUE
) {
502 // There must be a normal ACK following this.
507 if (monitor
.isCancelled())
508 throw new CancelledException();
512 private void negotiateBegin() throws IOException
{
513 walk
.resetRetain(REACHABLE
, ADVERTISED
);
514 walk
.markStart(reachableCommits
);
515 walk
.sort(RevSort
.COMMIT_TIME_DESC
);
516 walk
.setRevFilter(new RevFilter() {
518 public RevFilter
clone() {
523 public boolean include(final RevWalk walker
, final RevCommit c
) {
524 final boolean remoteKnowsIsCommon
= c
.has(COMMON
);
525 if (c
.has(ADVERTISED
)) {
526 // Remote advertised this, and we have it, hence common.
527 // Whether or not the remote knows that fact is tested
528 // before we added the flag. If the remote doesn't know
529 // we have to still send them this object.
533 return !remoteKnowsIsCommon
;
538 private void markRefsAdvertised() {
539 for (final Ref r
: getRefs()) {
540 markAdvertised(r
.getObjectId());
541 if (r
.getPeeledObjectId() != null)
542 markAdvertised(r
.getPeeledObjectId());
546 private void markAdvertised(final AnyObjectId id
) {
548 walk
.parseAny(id
).add(ADVERTISED
);
549 } catch (IOException readError
) {
550 // We probably just do not have this object locally.
554 private void markCommon(final RevObject obj
) {
556 if (obj
instanceof RevCommit
)
557 ((RevCommit
) obj
).carry(COMMON
);
560 private void receivePack(final ProgressMonitor monitor
) throws IOException
{
563 ip
= IndexPack
.create(local
, sideband ? pckIn
.sideband(monitor
) : in
);
564 ip
.setFixThin(thinPack
);
566 ip
.renameAndOpenPack();
570 public void close() {
575 } catch (IOException err
) {
576 // Ignore any close errors.
586 } catch (IOException err
) {
587 // Ignore any close errors.
595 private static class CancelledException
extends Exception
{
596 private static final long serialVersionUID
= 1L;