2 * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com>
3 * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
7 * Redistribution and use in source and binary forms, with or
8 * without modification, are permitted provided that the following
11 * - Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
14 * - Redistributions in binary form must reproduce the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer in the documentation and/or other materials provided
17 * with the distribution.
19 * - Neither the name of the Git Development Community nor the
20 * names of its contributors may be used to endorse or promote
21 * products derived from this software without specific prior
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
25 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
26 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
27 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
29 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
31 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
32 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
33 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
34 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
36 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 package org
.spearce
.jgit
.transport
;
41 import java
.io
.IOException
;
42 import java
.util
.Collection
;
43 import java
.util
.Collections
;
44 import java
.util
.Date
;
47 import org
.spearce
.jgit
.errors
.TransportException
;
48 import org
.spearce
.jgit
.lib
.AnyObjectId
;
49 import org
.spearce
.jgit
.lib
.MutableObjectId
;
50 import org
.spearce
.jgit
.lib
.ObjectId
;
51 import org
.spearce
.jgit
.lib
.PackLock
;
52 import org
.spearce
.jgit
.lib
.ProgressMonitor
;
53 import org
.spearce
.jgit
.lib
.Ref
;
54 import org
.spearce
.jgit
.lib
.RepositoryConfig
;
55 import org
.spearce
.jgit
.revwalk
.RevCommit
;
56 import org
.spearce
.jgit
.revwalk
.RevCommitList
;
57 import org
.spearce
.jgit
.revwalk
.RevFlag
;
58 import org
.spearce
.jgit
.revwalk
.RevObject
;
59 import org
.spearce
.jgit
.revwalk
.RevSort
;
60 import org
.spearce
.jgit
.revwalk
.RevWalk
;
61 import org
.spearce
.jgit
.revwalk
.filter
.CommitTimeRevFilter
;
62 import org
.spearce
.jgit
.revwalk
.filter
.RevFilter
;
65 * Fetch implementation using the native Git pack transfer service.
67 * This is the canonical implementation for transferring objects from the remote
68 * repository to the local repository by talking to the 'git-upload-pack'
69 * service. Objects are packed on the remote side into a pack file and then sent
70 * down the pipe to us.
72 * This connection requires only a bi-directional pipe or socket, and thus is
73 * easily wrapped up into a local process pipe, anonymous TCP socket, or a
74 * command executed through an SSH tunnel.
76 * Concrete implementations should just call
77 * {@link #init(java.io.InputStream, java.io.OutputStream)} and
78 * {@link #readAdvertisedRefs()} methods in constructor or before any use. They
79 * should also handle resources releasing in {@link #close()} method if needed.
81 abstract class BasePackFetchConnection
extends BasePackConnection
implements
84 * Maximum number of 'have' lines to send before giving up.
86 * During {@link #negotiate(ProgressMonitor)} we send at most this many
87 * commits to the remote peer as 'have' lines without an ACK response before
90 private static final int MAX_HAVES
= 256;
92 protected static final int MAX_CLIENT_BUFFER
= MAX_HAVES
* 46 + 1024;
94 static final String OPTION_INCLUDE_TAG
= "include-tag";
96 static final String OPTION_MULTI_ACK
= "multi_ack";
98 static final String OPTION_THIN_PACK
= "thin-pack";
100 static final String OPTION_SIDE_BAND
= "side-band";
102 static final String OPTION_SIDE_BAND_64K
= "side-band-64k";
104 static final String OPTION_OFS_DELTA
= "ofs-delta";
106 static final String OPTION_SHALLOW
= "shallow";
108 static final String OPTION_NO_PROGRESS
= "no-progress";
110 private final RevWalk walk
;
112 /** All commits that are immediately reachable by a local ref. */
113 private RevCommitList
<RevCommit
> reachableCommits
;
115 /** Marks an object as having all its dependencies. */
116 final RevFlag REACHABLE
;
118 /** Marks a commit known to both sides of the connection. */
119 final RevFlag COMMON
;
121 /** Marks a commit listed in the advertised refs. */
122 final RevFlag ADVERTISED
;
124 private boolean multiAck
;
126 private boolean thinPack
;
128 private boolean sideband
;
130 private boolean includeTags
;
132 private boolean allowOfsDelta
;
134 private String lockMessage
;
136 private PackLock packLock
;
138 BasePackFetchConnection(final PackTransport packTransport
) {
139 super(packTransport
);
141 final RepositoryConfig cfg
= local
.getConfig();
142 includeTags
= transport
.getTagOpt() != TagOpt
.NO_TAGS
;
143 thinPack
= transport
.isFetchThin();
144 allowOfsDelta
= cfg
.getBoolean("repack", "usedeltabaseoffset", true);
146 walk
= new RevWalk(local
);
147 reachableCommits
= new RevCommitList
<RevCommit
>();
148 REACHABLE
= walk
.newFlag("REACHABLE");
149 COMMON
= walk
.newFlag("COMMON");
150 ADVERTISED
= walk
.newFlag("ADVERTISED");
153 walk
.carry(REACHABLE
);
154 walk
.carry(ADVERTISED
);
157 public final void fetch(final ProgressMonitor monitor
,
158 final Collection
<Ref
> want
, final Set
<ObjectId
> have
)
159 throws TransportException
{
160 markStartedOperation();
161 doFetch(monitor
, want
, have
);
164 public boolean didFetchIncludeTags() {
168 public boolean didFetchTestConnectivity() {
172 public void setPackLockMessage(final String message
) {
173 lockMessage
= message
;
176 public Collection
<PackLock
> getPackLocks() {
177 if (packLock
!= null)
178 return Collections
.singleton(packLock
);
179 return Collections
.<PackLock
> emptyList();
182 protected void doFetch(final ProgressMonitor monitor
,
183 final Collection
<Ref
> want
, final Set
<ObjectId
> have
)
184 throws TransportException
{
186 markRefsAdvertised();
187 markReachable(have
, maxTimeWanted(want
));
189 if (sendWants(want
)) {
193 reachableCommits
= null;
195 receivePack(monitor
);
197 } catch (CancelledException ce
) {
199 return; // Caller should test (or just know) this themselves.
200 } catch (IOException err
) {
202 throw new TransportException(err
.getMessage(), err
);
203 } catch (RuntimeException err
) {
205 throw new TransportException(err
.getMessage(), err
);
209 private int maxTimeWanted(final Collection
<Ref
> wants
) {
211 for (final Ref r
: wants
) {
213 final RevObject obj
= walk
.parseAny(r
.getObjectId());
214 if (obj
instanceof RevCommit
) {
215 final int cTime
= ((RevCommit
) obj
).getCommitTime();
219 } catch (IOException error
) {
220 // We don't have it, but we want to fetch (thus fixing error).
226 private void markReachable(final Set
<ObjectId
> have
, final int maxTime
)
228 for (final Ref r
: local
.getAllRefs().values()) {
230 final RevCommit o
= walk
.parseCommit(r
.getObjectId());
232 reachableCommits
.add(o
);
233 } catch (IOException readError
) {
234 // If we cannot read the value of the ref skip it.
238 for (final ObjectId id
: have
) {
240 final RevCommit o
= walk
.parseCommit(id
);
242 reachableCommits
.add(o
);
243 } catch (IOException readError
) {
244 // If we cannot read the value of the ref skip it.
249 // Mark reachable commits until we reach maxTime. These may
250 // wind up later matching up against things we want and we
251 // can avoid asking for something we already happen to have.
253 final Date maxWhen
= new Date(maxTime
* 1000L);
254 walk
.sort(RevSort
.COMMIT_TIME_DESC
);
255 walk
.markStart(reachableCommits
);
256 walk
.setRevFilter(CommitTimeRevFilter
.after(maxWhen
));
258 final RevCommit c
= walk
.next();
261 if (c
.has(ADVERTISED
) && !c
.has(COMMON
)) {
262 // This is actually going to be a common commit, but
263 // our peer doesn't know that fact yet.
267 reachableCommits
.add(c
);
273 private boolean sendWants(final Collection
<Ref
> want
) throws IOException
{
274 boolean first
= true;
275 for (final Ref r
: want
) {
277 if (walk
.parseAny(r
.getObjectId()).has(REACHABLE
)) {
278 // We already have this object. Asking for it is
279 // not a very good idea.
283 } catch (IOException err
) {
284 // Its OK, we don't have it, but we want to fix that
285 // by fetching the object from the other side.
288 final StringBuilder line
= new StringBuilder(46);
289 line
.append("want ");
290 line
.append(r
.getObjectId().name());
292 line
.append(enableCapabilities());
296 pckOut
.writeString(line
.toString());
303 private String
enableCapabilities() {
304 final StringBuilder line
= new StringBuilder();
306 includeTags
= wantCapability(line
, OPTION_INCLUDE_TAG
);
308 wantCapability(line
, OPTION_OFS_DELTA
);
309 multiAck
= wantCapability(line
, OPTION_MULTI_ACK
);
311 thinPack
= wantCapability(line
, OPTION_THIN_PACK
);
312 if (wantCapability(line
, OPTION_SIDE_BAND_64K
))
314 else if (wantCapability(line
, OPTION_SIDE_BAND
))
316 return line
.toString();
319 private void negotiate(final ProgressMonitor monitor
) throws IOException
,
321 final MutableObjectId ackId
= new MutableObjectId();
322 int resultsPending
= 0;
324 int havesSinceLastContinue
= 0;
325 boolean receivedContinue
= false;
326 boolean receivedAck
= false;
327 boolean sendHaves
= true;
331 final RevCommit c
= walk
.next();
335 pckOut
.writeString("have " + c
.getId().name() + "\n");
337 havesSinceLastContinue
++;
339 if ((31 & havesSent
) != 0) {
340 // We group the have lines into blocks of 32, each marked
341 // with a flush (aka end). This one is within a block so
342 // continue with another have line.
347 if (monitor
.isCancelled())
348 throw new CancelledException();
351 resultsPending
++; // Each end will cause a result to come back.
353 if (havesSent
== 32) {
354 // On the first block we race ahead and try to send
355 // more of the second block while waiting for the
356 // remote to respond to our first block request.
357 // This keeps us one block ahead of the peer.
362 while (resultsPending
> 0) {
363 final PacketLineIn
.AckNackResult anr
;
365 anr
= pckIn
.readACK(ackId
);
367 if (anr
== PacketLineIn
.AckNackResult
.NAK
) {
368 // More have lines are necessary to compute the
369 // pack on the remote side. Keep doing that.
374 if (anr
== PacketLineIn
.AckNackResult
.ACK
) {
375 // The remote side is happy and knows exactly what
376 // to send us. There is no further negotiation and
377 // we can break out immediately.
386 if (anr
== PacketLineIn
.AckNackResult
.ACK_CONTINUE
) {
387 // The server knows this commit (ackId). We don't
388 // need to send any further along its ancestry, but
389 // we need to continue to talk about other parts of
390 // our local history.
392 markCommon(walk
.parseAny(ackId
));
394 receivedContinue
= true;
395 havesSinceLastContinue
= 0;
398 if (monitor
.isCancelled())
399 throw new CancelledException();
402 if (receivedContinue
&& havesSinceLastContinue
> MAX_HAVES
) {
403 // Our history must be really different from the remote's.
404 // We just sent a whole slew of have lines, and it did not
405 // recognize any of them. Avoid sending our entire history
406 // to them by giving up early.
412 // Tell the remote side we have run out of things to talk about.
414 if (monitor
.isCancelled())
415 throw new CancelledException();
416 pckOut
.writeString("done\n");
420 // Apparently if we have never received an ACK earlier
421 // there is one more result expected from the done we
422 // just sent to the remote.
428 while (resultsPending
> 0 || multiAck
) {
429 final PacketLineIn
.AckNackResult anr
;
431 anr
= pckIn
.readACK(ackId
);
434 if (anr
== PacketLineIn
.AckNackResult
.ACK
)
435 break; // commit negotiation is finished.
437 if (anr
== PacketLineIn
.AckNackResult
.ACK_CONTINUE
) {
438 // There must be a normal ACK following this.
443 if (monitor
.isCancelled())
444 throw new CancelledException();
448 private void negotiateBegin() throws IOException
{
449 walk
.resetRetain(REACHABLE
, ADVERTISED
);
450 walk
.markStart(reachableCommits
);
451 walk
.sort(RevSort
.COMMIT_TIME_DESC
);
452 walk
.setRevFilter(new RevFilter() {
454 public RevFilter
clone() {
459 public boolean include(final RevWalk walker
, final RevCommit c
) {
460 final boolean remoteKnowsIsCommon
= c
.has(COMMON
);
461 if (c
.has(ADVERTISED
)) {
462 // Remote advertised this, and we have it, hence common.
463 // Whether or not the remote knows that fact is tested
464 // before we added the flag. If the remote doesn't know
465 // we have to still send them this object.
469 return !remoteKnowsIsCommon
;
474 private void markRefsAdvertised() {
475 for (final Ref r
: getRefs()) {
476 markAdvertised(r
.getObjectId());
477 if (r
.getPeeledObjectId() != null)
478 markAdvertised(r
.getPeeledObjectId());
482 private void markAdvertised(final AnyObjectId id
) {
484 walk
.parseAny(id
).add(ADVERTISED
);
485 } catch (IOException readError
) {
486 // We probably just do not have this object locally.
490 private void markCommon(final RevObject obj
) {
492 if (obj
instanceof RevCommit
)
493 ((RevCommit
) obj
).carry(COMMON
);
496 private void receivePack(final ProgressMonitor monitor
) throws IOException
{
499 ip
= IndexPack
.create(local
, sideband ? pckIn
.sideband(monitor
) : in
);
500 ip
.setFixThin(thinPack
);
501 ip
.setObjectChecking(transport
.isCheckFetchedObjects());
503 packLock
= ip
.renameAndOpenPack(lockMessage
);
506 private static class CancelledException
extends Exception
{
507 private static final long serialVersionUID
= 1L;