2 * Copyright (C) 2007,2008 Robin Rosenberg
3 * Copyright (C) 2008 Shawn Pearce <spearce@spearce.org>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public
7 * License, version 2, as published by the Free Software Foundation.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
14 * You should have received a copy of the GNU General Public
15 * License along with this library; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
18 package org
.spearce
.jgit
.transport
;
20 import java
.io
.BufferedInputStream
;
21 import java
.io
.BufferedOutputStream
;
22 import java
.io
.EOFException
;
23 import java
.io
.IOException
;
24 import java
.io
.InputStream
;
25 import java
.io
.OutputStream
;
26 import java
.util
.Collection
;
27 import java
.util
.Date
;
28 import java
.util
.HashSet
;
29 import java
.util
.LinkedHashMap
;
32 import org
.spearce
.jgit
.errors
.PackProtocolException
;
33 import org
.spearce
.jgit
.errors
.TransportException
;
34 import org
.spearce
.jgit
.lib
.AnyObjectId
;
35 import org
.spearce
.jgit
.lib
.MutableObjectId
;
36 import org
.spearce
.jgit
.lib
.ObjectId
;
37 import org
.spearce
.jgit
.lib
.ProgressMonitor
;
38 import org
.spearce
.jgit
.lib
.Ref
;
39 import org
.spearce
.jgit
.lib
.Repository
;
40 import org
.spearce
.jgit
.revwalk
.RevCommit
;
41 import org
.spearce
.jgit
.revwalk
.RevCommitList
;
42 import org
.spearce
.jgit
.revwalk
.RevFlag
;
43 import org
.spearce
.jgit
.revwalk
.RevObject
;
44 import org
.spearce
.jgit
.revwalk
.RevSort
;
45 import org
.spearce
.jgit
.revwalk
.RevWalk
;
46 import org
.spearce
.jgit
.revwalk
.filter
.CommitTimeRevFilter
;
47 import org
.spearce
.jgit
.revwalk
.filter
.RevFilter
;
50 * Fetch implementation using the native Git pack transfer service.
52 * This is the canonical implementation for transferring objects from the remote
53 * repository to the local repository by talking to the 'git-upload-pack'
54 * service. Objects are packed on the remote side into a pack file and then sent
55 * down the pipe to us.
57 * This connection requires only a bi-directional pipe or socket, and thus is
58 * easily wrapped up into a local process pipe, anonymous TCP socket, or a
59 * command executed through an SSH tunnel.
61 abstract class PackFetchConnection
extends FetchConnection
{
63 * Maximum number of 'have' lines to send before giving up.
65 * During {@link #negotiate(ProgressMonitor)} we send at most this many
66 * commits to the remote peer as 'have' lines without an ACK response before
69 private static final int MAX_HAVES
= 256;
71 static final String OPTION_INCLUDE_TAG
= "include-tag";
73 static final String OPTION_MULTI_ACK
= "multi_ack";
75 static final String OPTION_THIN_PACK
= "thin-pack";
77 static final String OPTION_SIDE_BAND
= "side-band";
79 static final String OPTION_SIDE_BAND_64K
= "side-band-64k";
81 static final String OPTION_OFS_DELTA
= "ofs-delta";
83 static final String OPTION_SHALLOW
= "shallow";
85 /** The repository this transport fetches into, or pushes out of. */
86 protected final Repository local
;
88 /** Remote repository location. */
89 protected final URIish uri
;
91 /** Capability tokens advertised by the remote side. */
92 protected final Set
<String
> remoteCapablities
= new HashSet
<String
>();
94 /** Buffered input stream reading from the remote. */
95 protected InputStream in
;
97 /** Buffered output stream sending to the remote. */
98 protected OutputStream out
;
100 /** Packet line decoder around {@link #in}. */
101 protected PacketLineIn pckIn
;
103 /** Packet line encoder around {@link #out}. */
104 protected PacketLineOut pckOut
;
106 private final RevWalk walk
;
108 /** All commits that are immediately reachable by a local ref. */
109 private RevCommitList
<RevCommit
> reachableCommits
;
111 /** Marks an object as having all its dependencies. */
112 final RevFlag REACHABLE
;
114 /** Marks a commit known to both sides of the connection. */
115 final RevFlag COMMON
;
117 /** Marks a commit listed in the advertised refs. */
118 final RevFlag ADVERTISED
;
120 private boolean multiAck
;
122 private boolean thinPack
;
124 private boolean sideband
;
126 private boolean includeTags
;
128 PackFetchConnection(final PackTransport packTransport
) {
129 local
= packTransport
.local
;
130 uri
= packTransport
.uri
;
131 includeTags
= packTransport
.getTagOpt() != TagOpt
.NO_TAGS
;
133 walk
= new RevWalk(local
);
134 reachableCommits
= new RevCommitList
<RevCommit
>();
135 REACHABLE
= walk
.newFlag("REACHABLE");
136 COMMON
= walk
.newFlag("COMMON");
137 ADVERTISED
= walk
.newFlag("ADVERTISED");
140 walk
.carry(REACHABLE
);
141 walk
.carry(ADVERTISED
);
144 protected void init(final InputStream myIn
, final OutputStream myOut
) {
145 in
= myIn
instanceof BufferedInputStream ? myIn
146 : new BufferedInputStream(myIn
);
147 out
= myOut
instanceof BufferedOutputStream ? myOut
148 : new BufferedOutputStream(myOut
);
150 pckIn
= new PacketLineIn(in
);
151 pckOut
= new PacketLineOut(out
);
154 protected void readAdvertisedRefs() throws TransportException
{
156 readAdvertisedRefsImpl();
157 } catch (TransportException err
) {
160 } catch (IOException err
) {
162 throw new TransportException(err
.getMessage(), err
);
163 } catch (RuntimeException err
) {
165 throw new TransportException(err
.getMessage(), err
);
169 private void readAdvertisedRefsImpl() throws IOException
{
170 final LinkedHashMap
<String
, Ref
> avail
= new LinkedHashMap
<String
, Ref
>();
175 line
= pckIn
.readString();
176 } catch (EOFException eof
) {
178 throw new TransportException(uri
+ " not found.");
182 if (avail
.isEmpty()) {
183 // The first line (if any) may contain "hidden"
184 // capability values after a NUL byte.
186 final int nul
= line
.indexOf('\0');
188 for (String c
: line
.substring(nul
+ 1).split(" "))
189 remoteCapablities
.add(c
);
190 line
= line
.substring(0, nul
);
194 if (line
.length() == 0)
197 String name
= line
.substring(41, line
.length());
198 final ObjectId id
= ObjectId
.fromString(line
.substring(0, 40));
199 if (name
.endsWith("^{}")) {
200 name
= name
.substring(0, name
.length() - 3);
201 final Ref prior
= avail
.get(name
);
203 throw new PackProtocolException("advertisement of " + name
204 + "^{} came before " + name
);
206 if (prior
.getPeeledObjectId() != null)
207 throw duplicateAdvertisement(name
+ "^{}");
209 avail
.put(name
, new Ref(name
, prior
.getObjectId(), id
));
211 final Ref prior
= avail
.put(name
, new Ref(name
, id
));
213 throw duplicateAdvertisement(name
);
219 private PackProtocolException
duplicateAdvertisement(final String name
) {
220 return new PackProtocolException("duplicate advertisements of " + name
);
224 protected void doFetch(final ProgressMonitor monitor
,
225 final Collection
<Ref
> want
) throws TransportException
{
227 markRefsAdvertised();
228 markReachable(maxTimeWanted(want
));
230 if (sendWants(want
)) {
234 reachableCommits
= null;
236 receivePack(monitor
);
238 } catch (CancelledException ce
) {
240 return; // Caller should test (or just know) this themselves.
241 } catch (IOException err
) {
243 throw new TransportException(err
.getMessage(), err
);
244 } catch (RuntimeException err
) {
246 throw new TransportException(err
.getMessage(), err
);
250 private int maxTimeWanted(final Collection
<Ref
> wants
) {
252 for (final Ref r
: wants
) {
254 final RevObject obj
= walk
.parseAny(r
.getObjectId());
255 if (obj
instanceof RevCommit
) {
256 final int cTime
= ((RevCommit
) obj
).getCommitTime();
260 } catch (IOException error
) {
261 // We don't have it, but we want to fetch (thus fixing error).
267 private void markReachable(final int maxTime
) throws IOException
{
268 for (final Ref r
: local
.getAllRefs().values()) {
270 final RevCommit o
= walk
.parseCommit(r
.getObjectId());
272 reachableCommits
.add(o
);
273 } catch (IOException readError
) {
274 // If we cannot read the value of the ref skip it.
275 } catch (ClassCastException cce
) {
276 // Not a commit type.
281 // Mark reachable commits until we reach maxTime. These may
282 // wind up later matching up against things we want and we
283 // can avoid asking for something we already happen to have.
285 final Date maxWhen
= new Date(maxTime
* 1000L);
286 walk
.sort(RevSort
.COMMIT_TIME_DESC
);
287 walk
.markStart(reachableCommits
);
288 walk
.setRevFilter(CommitTimeRevFilter
.after(maxWhen
));
290 final RevCommit c
= walk
.next();
293 if (c
.has(ADVERTISED
) && !c
.has(COMMON
)) {
294 // This is actually going to be a common commit, but
295 // our peer doesn't know that fact yet.
299 reachableCommits
.add(c
);
305 private boolean sendWants(final Collection
<Ref
> want
) throws IOException
{
306 boolean first
= true;
307 for (final Ref r
: want
) {
309 if (walk
.parseAny(r
.getObjectId()).has(REACHABLE
)) {
310 // We already have this object. Asking for it is
311 // not a very good idea.
315 } catch (IOException err
) {
316 // Its OK, we don't have it, but we want to fix that
317 // by fetching the object from the other side.
320 final StringBuilder line
= new StringBuilder(46);
321 line
.append("want ");
322 line
.append(r
.getObjectId());
324 line
.append(enableCapabilities());
328 pckOut
.writeString(line
.toString());
334 private String
enableCapabilities() {
335 final StringBuilder line
= new StringBuilder();
337 includeTags
= wantCapability(line
, OPTION_INCLUDE_TAG
);
338 wantCapability(line
, OPTION_OFS_DELTA
);
339 multiAck
= wantCapability(line
, OPTION_MULTI_ACK
);
340 thinPack
= wantCapability(line
, OPTION_THIN_PACK
);
341 if (wantCapability(line
, OPTION_SIDE_BAND_64K
))
343 else if (wantCapability(line
, OPTION_SIDE_BAND
))
345 return line
.toString();
348 private boolean wantCapability(final StringBuilder b
, final String option
) {
349 if (!remoteCapablities
.contains(option
))
357 private void negotiate(final ProgressMonitor monitor
) throws IOException
,
359 final MutableObjectId ackId
= new MutableObjectId();
360 int resultsPending
= 0;
362 int havesSinceLastContinue
= 0;
363 boolean receivedContinue
= false;
364 boolean receivedAck
= false;
365 boolean sendHaves
= true;
369 final RevCommit c
= walk
.next();
373 pckOut
.writeString("have " + c
.getId() + "\n");
375 havesSinceLastContinue
++;
377 if ((31 & havesSent
) != 0) {
378 // We group the have lines into blocks of 32, each marked
379 // with a flush (aka end). This one is within a block so
380 // continue with another have line.
385 if (monitor
.isCancelled())
386 throw new CancelledException();
389 resultsPending
++; // Each end will cause a result to come back.
391 if (havesSent
== 32) {
392 // On the first block we race ahead and try to send
393 // more of the second block while waiting for the
394 // remote to respond to our first block request.
395 // This keeps us one block ahead of the peer.
400 while (resultsPending
> 0) {
401 final PacketLineIn
.AckNackResult anr
;
403 anr
= pckIn
.readACK(ackId
);
405 if (anr
== PacketLineIn
.AckNackResult
.NAK
) {
406 // More have lines are necessary to compute the
407 // pack on the remote side. Keep doing that.
412 if (anr
== PacketLineIn
.AckNackResult
.ACK
) {
413 // The remote side is happy and knows exactly what
414 // to send us. There is no further negotiation and
415 // we can break out immediately.
424 if (anr
== PacketLineIn
.AckNackResult
.ACK_CONTINUE
) {
425 // The server knows this commit (ackId). We don't
426 // need to send any further along its ancestry, but
427 // we need to continue to talk about other parts of
428 // our local history.
430 markCommon(walk
.parseAny(ackId
));
432 receivedContinue
= true;
433 havesSinceLastContinue
= 0;
436 if (monitor
.isCancelled())
437 throw new CancelledException();
440 if (receivedContinue
&& havesSinceLastContinue
> MAX_HAVES
) {
441 // Our history must be really different from the remote's.
442 // We just sent a whole slew of have lines, and it did not
443 // recognize any of them. Avoid sending our entire history
444 // to them by giving up early.
450 // Tell the remote side we have run out of things to talk about.
452 if (monitor
.isCancelled())
453 throw new CancelledException();
454 pckOut
.writeString("done\n");
458 // Apparently if we have never received an ACK earlier
459 // there is one more result expected from the done we
460 // just sent to the remote.
466 while (resultsPending
> 0 || multiAck
) {
467 final PacketLineIn
.AckNackResult anr
;
469 anr
= pckIn
.readACK(ackId
);
472 if (anr
== PacketLineIn
.AckNackResult
.ACK
)
473 break; // commit negotiation is finished.
475 if (anr
== PacketLineIn
.AckNackResult
.ACK_CONTINUE
) {
476 // There must be a normal ACK following this.
481 if (monitor
.isCancelled())
482 throw new CancelledException();
486 private void negotiateBegin() throws IOException
{
487 walk
.resetRetain(REACHABLE
, ADVERTISED
);
488 walk
.markStart(reachableCommits
);
489 walk
.sort(RevSort
.COMMIT_TIME_DESC
);
490 walk
.setRevFilter(new RevFilter() {
492 public RevFilter
clone() {
497 public boolean include(final RevWalk walker
, final RevCommit c
) {
498 final boolean remoteKnowsIsCommon
= c
.has(COMMON
);
499 if (c
.has(ADVERTISED
)) {
500 // Remote advertised this, and we have it, hence common.
501 // Whether or not the remote knows that fact is tested
502 // before we added the flag. If the remote doesn't know
503 // we have to still send them this object.
507 return !remoteKnowsIsCommon
;
512 private void markRefsAdvertised() {
513 for (final Ref r
: getRefs()) {
514 markAdvertised(r
.getObjectId());
515 if (r
.getPeeledObjectId() != null)
516 markAdvertised(r
.getPeeledObjectId());
520 private void markAdvertised(final AnyObjectId id
) {
522 walk
.parseAny(id
).add(ADVERTISED
);
523 } catch (IOException readError
) {
524 // We probably just do not have this object locally.
528 private void markCommon(final RevObject obj
) {
530 if (obj
instanceof RevCommit
)
531 ((RevCommit
) obj
).carry(COMMON
);
534 private void receivePack(final ProgressMonitor monitor
) throws IOException
{
537 ip
= IndexPack
.create(local
, sideband ? pckIn
.sideband(monitor
) : in
);
538 ip
.setFixThin(thinPack
);
540 ip
.renameAndOpenPack();
544 public void close() {
549 } catch (IOException err
) {
550 // Ignore any close errors.
560 } catch (IOException err
) {
561 // Ignore any close errors.
569 private static class CancelledException
extends Exception
{
570 private static final long serialVersionUID
= 1L;