Only request advertised include-tags if we want tags
[egit/zawir.git] / org.spearce.jgit / src / org / spearce / jgit / transport / PackFetchConnection.java
blob216a483d4d353132ca89826eaf0aad381b9e6cbd
1 /*
2 * Copyright (C) 2007,2008 Robin Rosenberg
3 * Copyright (C) 2008 Shawn Pearce <spearce@spearce.org>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public
7 * License, version 2, as published by the Free Software Foundation.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
14 * You should have received a copy of the GNU General Public
15 * License along with this library; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
18 package org.spearce.jgit.transport;
20 import java.io.BufferedInputStream;
21 import java.io.BufferedOutputStream;
22 import java.io.EOFException;
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.io.OutputStream;
26 import java.util.Collection;
27 import java.util.Date;
28 import java.util.HashSet;
29 import java.util.LinkedHashMap;
30 import java.util.Set;
32 import org.spearce.jgit.errors.PackProtocolException;
33 import org.spearce.jgit.errors.TransportException;
34 import org.spearce.jgit.lib.AnyObjectId;
35 import org.spearce.jgit.lib.MutableObjectId;
36 import org.spearce.jgit.lib.ObjectId;
37 import org.spearce.jgit.lib.ProgressMonitor;
38 import org.spearce.jgit.lib.Ref;
39 import org.spearce.jgit.lib.Repository;
40 import org.spearce.jgit.revwalk.RevCommit;
41 import org.spearce.jgit.revwalk.RevCommitList;
42 import org.spearce.jgit.revwalk.RevFlag;
43 import org.spearce.jgit.revwalk.RevObject;
44 import org.spearce.jgit.revwalk.RevSort;
45 import org.spearce.jgit.revwalk.RevWalk;
46 import org.spearce.jgit.revwalk.filter.CommitTimeRevFilter;
47 import org.spearce.jgit.revwalk.filter.RevFilter;
49 /**
50 * Fetch implementation using the native Git pack transfer service.
51 * <p>
52 * This is the canonical implementation for transferring objects from the remote
53 * repository to the local repository by talking to the 'git-upload-pack'
54 * service. Objects are packed on the remote side into a pack file and then sent
55 * down the pipe to us.
56 * <p>
57 * This connection requires only a bi-directional pipe or socket, and thus is
58 * easily wrapped up into a local process pipe, anonymous TCP socket, or a
59 * command executed through an SSH tunnel.
61 abstract class PackFetchConnection extends FetchConnection {
62 /**
63 * Maximum number of 'have' lines to send before giving up.
64 * <p>
65 * During {@link #negotiate(ProgressMonitor)} we send at most this many
66 * commits to the remote peer as 'have' lines without an ACK response before
67 * we give up.
69 private static final int MAX_HAVES = 256;
71 static final String OPTION_INCLUDE_TAG = "include-tag";
73 static final String OPTION_MULTI_ACK = "multi_ack";
75 static final String OPTION_THIN_PACK = "thin-pack";
77 static final String OPTION_SIDE_BAND = "side-band";
79 static final String OPTION_SIDE_BAND_64K = "side-band-64k";
81 static final String OPTION_OFS_DELTA = "ofs-delta";
83 static final String OPTION_SHALLOW = "shallow";
85 /** The repository this transport fetches into, or pushes out of. */
86 protected final Repository local;
88 /** Remote repository location. */
89 protected final URIish uri;
91 /** Capability tokens advertised by the remote side. */
92 protected final Set<String> remoteCapablities = new HashSet<String>();
94 /** Buffered input stream reading from the remote. */
95 protected InputStream in;
97 /** Buffered output stream sending to the remote. */
98 protected OutputStream out;
100 /** Packet line decoder around {@link #in}. */
101 protected PacketLineIn pckIn;
103 /** Packet line encoder around {@link #out}. */
104 protected PacketLineOut pckOut;
106 private final RevWalk walk;
108 /** All commits that are immediately reachable by a local ref. */
109 private RevCommitList<RevCommit> reachableCommits;
111 /** Marks an object as having all its dependencies. */
112 final RevFlag REACHABLE;
114 /** Marks a commit known to both sides of the connection. */
115 final RevFlag COMMON;
117 /** Marks a commit listed in the advertised refs. */
118 final RevFlag ADVERTISED;
120 private boolean multiAck;
122 private boolean thinPack;
124 private boolean sideband;
126 private boolean includeTags;
128 PackFetchConnection(final PackTransport packTransport) {
129 local = packTransport.local;
130 uri = packTransport.uri;
131 includeTags = packTransport.getTagOpt() != TagOpt.NO_TAGS;
133 walk = new RevWalk(local);
134 reachableCommits = new RevCommitList<RevCommit>();
135 REACHABLE = walk.newFlag("REACHABLE");
136 COMMON = walk.newFlag("COMMON");
137 ADVERTISED = walk.newFlag("ADVERTISED");
139 walk.carry(COMMON);
140 walk.carry(REACHABLE);
141 walk.carry(ADVERTISED);
144 protected void init(final InputStream myIn, final OutputStream myOut) {
145 in = myIn instanceof BufferedInputStream ? myIn
146 : new BufferedInputStream(myIn);
147 out = myOut instanceof BufferedOutputStream ? myOut
148 : new BufferedOutputStream(myOut);
150 pckIn = new PacketLineIn(in);
151 pckOut = new PacketLineOut(out);
154 protected void readAdvertisedRefs() throws TransportException {
155 try {
156 readAdvertisedRefsImpl();
157 } catch (TransportException err) {
158 close();
159 throw err;
160 } catch (IOException err) {
161 close();
162 throw new TransportException(err.getMessage(), err);
163 } catch (RuntimeException err) {
164 close();
165 throw new TransportException(err.getMessage(), err);
169 private void readAdvertisedRefsImpl() throws IOException {
170 final LinkedHashMap<String, Ref> avail = new LinkedHashMap<String, Ref>();
171 for (;;) {
172 String line;
174 try {
175 line = pckIn.readString();
176 } catch (EOFException eof) {
177 if (avail.isEmpty())
178 throw new TransportException(uri + " not found.");
179 throw eof;
182 if (avail.isEmpty()) {
183 // The first line (if any) may contain "hidden"
184 // capability values after a NUL byte.
186 final int nul = line.indexOf('\0');
187 if (nul >= 0) {
188 for (String c : line.substring(nul + 1).split(" "))
189 remoteCapablities.add(c);
190 line = line.substring(0, nul);
194 if (line.length() == 0)
195 break;
197 String name = line.substring(41, line.length());
198 final ObjectId id = ObjectId.fromString(line.substring(0, 40));
199 if (name.endsWith("^{}")) {
200 name = name.substring(0, name.length() - 3);
201 final Ref prior = avail.get(name);
202 if (prior == null)
203 throw new PackProtocolException("advertisement of " + name
204 + "^{} came before " + name);
206 if (prior.getPeeledObjectId() != null)
207 throw duplicateAdvertisement(name + "^{}");
209 avail.put(name, new Ref(name, prior.getObjectId(), id));
210 } else {
211 final Ref prior = avail.put(name, new Ref(name, id));
212 if (prior != null)
213 throw duplicateAdvertisement(name);
216 available(avail);
219 private PackProtocolException duplicateAdvertisement(final String name) {
220 return new PackProtocolException("duplicate advertisements of " + name);
223 @Override
224 protected void doFetch(final ProgressMonitor monitor,
225 final Collection<Ref> want) throws TransportException {
226 try {
227 markRefsAdvertised();
228 markReachable(maxTimeWanted(want));
230 if (sendWants(want)) {
231 negotiate(monitor);
233 walk.dispose();
234 reachableCommits = null;
236 receivePack(monitor);
238 } catch (CancelledException ce) {
239 close();
240 return; // Caller should test (or just know) this themselves.
241 } catch (IOException err) {
242 close();
243 throw new TransportException(err.getMessage(), err);
244 } catch (RuntimeException err) {
245 close();
246 throw new TransportException(err.getMessage(), err);
250 private int maxTimeWanted(final Collection<Ref> wants) {
251 int maxTime = 0;
252 for (final Ref r : wants) {
253 try {
254 final RevObject obj = walk.parseAny(r.getObjectId());
255 if (obj instanceof RevCommit) {
256 final int cTime = ((RevCommit) obj).getCommitTime();
257 if (maxTime < cTime)
258 maxTime = cTime;
260 } catch (IOException error) {
261 // We don't have it, but we want to fetch (thus fixing error).
264 return maxTime;
267 private void markReachable(final int maxTime) throws IOException {
268 for (final Ref r : local.getAllRefs().values()) {
269 try {
270 final RevCommit o = walk.parseCommit(r.getObjectId());
271 o.add(REACHABLE);
272 reachableCommits.add(o);
273 } catch (IOException readError) {
274 // If we cannot read the value of the ref skip it.
275 } catch (ClassCastException cce) {
276 // Not a commit type.
280 if (maxTime > 0) {
281 // Mark reachable commits until we reach maxTime. These may
282 // wind up later matching up against things we want and we
283 // can avoid asking for something we already happen to have.
285 final Date maxWhen = new Date(maxTime * 1000L);
286 walk.sort(RevSort.COMMIT_TIME_DESC);
287 walk.markStart(reachableCommits);
288 walk.setRevFilter(CommitTimeRevFilter.after(maxWhen));
289 for (;;) {
290 final RevCommit c = walk.next();
291 if (c == null)
292 break;
293 if (c.has(ADVERTISED) && !c.has(COMMON)) {
294 // This is actually going to be a common commit, but
295 // our peer doesn't know that fact yet.
297 c.add(COMMON);
298 c.carry(COMMON);
299 reachableCommits.add(c);
305 private boolean sendWants(final Collection<Ref> want) throws IOException {
306 boolean first = true;
307 for (final Ref r : want) {
308 try {
309 if (walk.parseAny(r.getObjectId()).has(REACHABLE)) {
310 // We already have this object. Asking for it is
311 // not a very good idea.
313 continue;
315 } catch (IOException err) {
316 // Its OK, we don't have it, but we want to fix that
317 // by fetching the object from the other side.
320 final StringBuilder line = new StringBuilder(46);
321 line.append("want ");
322 line.append(r.getObjectId());
323 if (first) {
324 line.append(enableCapabilities());
325 first = false;
327 line.append('\n');
328 pckOut.writeString(line.toString());
330 pckOut.end();
331 return !first;
334 private String enableCapabilities() {
335 final StringBuilder line = new StringBuilder();
336 if (includeTags)
337 includeTags = wantCapability(line, OPTION_INCLUDE_TAG);
338 wantCapability(line, OPTION_OFS_DELTA);
339 multiAck = wantCapability(line, OPTION_MULTI_ACK);
340 thinPack = wantCapability(line, OPTION_THIN_PACK);
341 if (wantCapability(line, OPTION_SIDE_BAND_64K))
342 sideband = true;
343 else if (wantCapability(line, OPTION_SIDE_BAND))
344 sideband = true;
345 return line.toString();
348 private boolean wantCapability(final StringBuilder b, final String option) {
349 if (!remoteCapablities.contains(option))
350 return false;
351 if (b.length() > 0)
352 b.append(' ');
353 b.append(option);
354 return true;
357 private void negotiate(final ProgressMonitor monitor) throws IOException,
358 CancelledException {
359 final MutableObjectId ackId = new MutableObjectId();
360 int resultsPending = 0;
361 int havesSent = 0;
362 int havesSinceLastContinue = 0;
363 boolean receivedContinue = false;
364 boolean receivedAck = false;
365 boolean sendHaves = true;
367 negotiateBegin();
368 while (sendHaves) {
369 final RevCommit c = walk.next();
370 if (c == null)
371 break;
373 pckOut.writeString("have " + c.getId() + "\n");
374 havesSent++;
375 havesSinceLastContinue++;
377 if ((31 & havesSent) != 0) {
378 // We group the have lines into blocks of 32, each marked
379 // with a flush (aka end). This one is within a block so
380 // continue with another have line.
382 continue;
385 if (monitor.isCancelled())
386 throw new CancelledException();
388 pckOut.end();
389 resultsPending++; // Each end will cause a result to come back.
391 if (havesSent == 32) {
392 // On the first block we race ahead and try to send
393 // more of the second block while waiting for the
394 // remote to respond to our first block request.
395 // This keeps us one block ahead of the peer.
397 continue;
400 while (resultsPending > 0) {
401 final PacketLineIn.AckNackResult anr;
403 anr = pckIn.readACK(ackId);
404 resultsPending--;
405 if (anr == PacketLineIn.AckNackResult.NAK) {
406 // More have lines are necessary to compute the
407 // pack on the remote side. Keep doing that.
409 break;
412 if (anr == PacketLineIn.AckNackResult.ACK) {
413 // The remote side is happy and knows exactly what
414 // to send us. There is no further negotiation and
415 // we can break out immediately.
417 multiAck = false;
418 resultsPending = 0;
419 receivedAck = true;
420 sendHaves = false;
421 break;
424 if (anr == PacketLineIn.AckNackResult.ACK_CONTINUE) {
425 // The server knows this commit (ackId). We don't
426 // need to send any further along its ancestry, but
427 // we need to continue to talk about other parts of
428 // our local history.
430 markCommon(walk.parseAny(ackId));
431 receivedAck = true;
432 receivedContinue = true;
433 havesSinceLastContinue = 0;
436 if (monitor.isCancelled())
437 throw new CancelledException();
440 if (receivedContinue && havesSinceLastContinue > MAX_HAVES) {
441 // Our history must be really different from the remote's.
442 // We just sent a whole slew of have lines, and it did not
443 // recognize any of them. Avoid sending our entire history
444 // to them by giving up early.
446 break;
450 // Tell the remote side we have run out of things to talk about.
452 if (monitor.isCancelled())
453 throw new CancelledException();
454 pckOut.writeString("done\n");
455 pckOut.flush();
457 if (!receivedAck) {
458 // Apparently if we have never received an ACK earlier
459 // there is one more result expected from the done we
460 // just sent to the remote.
462 multiAck = false;
463 resultsPending++;
466 while (resultsPending > 0 || multiAck) {
467 final PacketLineIn.AckNackResult anr;
469 anr = pckIn.readACK(ackId);
470 resultsPending--;
472 if (anr == PacketLineIn.AckNackResult.ACK)
473 break; // commit negotiation is finished.
475 if (anr == PacketLineIn.AckNackResult.ACK_CONTINUE) {
476 // There must be a normal ACK following this.
478 multiAck = true;
481 if (monitor.isCancelled())
482 throw new CancelledException();
486 private void negotiateBegin() throws IOException {
487 walk.resetRetain(REACHABLE, ADVERTISED);
488 walk.markStart(reachableCommits);
489 walk.sort(RevSort.COMMIT_TIME_DESC);
490 walk.setRevFilter(new RevFilter() {
491 @Override
492 public RevFilter clone() {
493 return this;
496 @Override
497 public boolean include(final RevWalk walker, final RevCommit c) {
498 final boolean remoteKnowsIsCommon = c.has(COMMON);
499 if (c.has(ADVERTISED)) {
500 // Remote advertised this, and we have it, hence common.
501 // Whether or not the remote knows that fact is tested
502 // before we added the flag. If the remote doesn't know
503 // we have to still send them this object.
505 c.add(COMMON);
507 return !remoteKnowsIsCommon;
512 private void markRefsAdvertised() {
513 for (final Ref r : getRefs()) {
514 markAdvertised(r.getObjectId());
515 if (r.getPeeledObjectId() != null)
516 markAdvertised(r.getPeeledObjectId());
520 private void markAdvertised(final AnyObjectId id) {
521 try {
522 walk.parseAny(id).add(ADVERTISED);
523 } catch (IOException readError) {
524 // We probably just do not have this object locally.
528 private void markCommon(final RevObject obj) {
529 obj.add(COMMON);
530 if (obj instanceof RevCommit)
531 ((RevCommit) obj).carry(COMMON);
534 private void receivePack(final ProgressMonitor monitor) throws IOException {
535 final IndexPack ip;
537 ip = IndexPack.create(local, sideband ? pckIn.sideband(monitor) : in);
538 ip.setFixThin(thinPack);
539 ip.index(monitor);
540 ip.renameAndOpenPack();
543 @Override
544 public void close() {
545 if (out != null) {
546 try {
547 pckOut.end();
548 out.close();
549 } catch (IOException err) {
550 // Ignore any close errors.
551 } finally {
552 out = null;
553 pckOut = null;
557 if (in != null) {
558 try {
559 in.close();
560 } catch (IOException err) {
561 // Ignore any close errors.
562 } finally {
563 in = null;
564 pckIn = null;
569 private static class CancelledException extends Exception {
570 private static final long serialVersionUID = 1L;