?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
everything_local: use "quick" object existence check
[git/debian.git] / fetch-pack.c
blobeb553cb631f9924859ec36523bc128b8c7c44250
1 #include "cache.h"
2 #include "config.h"
3 #include "lockfile.h"
4 #include "refs.h"
5 #include "pkt-line.h"
6 #include "commit.h"
7 #include "tag.h"
8 #include "exec_cmd.h"
9 #include "pack.h"
10 #include "sideband.h"
11 #include "fetch-pack.h"
12 #include "remote.h"
13 #include "run-command.h"
14 #include "connect.h"
15 #include "transport.h"
16 #include "version.h"
17 #include "prio-queue.h"
18 #include "sha1-array.h"
19 #include "oidset.h"
20
21 static int transfer_unpack_limit = -1;
22 static int fetch_unpack_limit = -1;
23 static int unpack_limit = 100;
24 static int prefer_ofs_delta = 1;
25 static int no_done;
26 static int deepen_since_ok;
27 static int deepen_not_ok;
28 static int fetch_fsck_objects = -1;
29 static int transfer_fsck_objects = -1;
30 static int agent_supported;
31 static struct lock_file shallow_lock;
32 static const char *alternate_shallow_file;
33
34 /* Remember to update object flag allocation in object.h */
35 #define COMPLETE (1U << 0)
36 #define COMMON (1U << 1)
37 #define COMMON_REF (1U << 2)
38 #define SEEN (1U << 3)
39 #define POPPED (1U << 4)
40 #define ALTERNATE (1U << 5)
41
42 static int marked;
43
44 /*
45 * After sending this many "have"s if we do not get any new ACK , we
46 * give up traversing our history.
47 */
48 #define MAX_IN_VAIN 256
49
50 static struct prio_queue rev_list = { compare_commits_by_commit_date };
51 static int non_common_revs, multi_ack, use_sideband;
52 /* Allow specifying sha1 if it is a ref tip. */
53 #define ALLOW_TIP_SHA1 01
54 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
55 #define ALLOW_REACHABLE_SHA1 02
56 static unsigned int allow_unadvertised_object_request;
57
58 __attribute__((format (printf, 2, 3)))
59 static inline void print_verbose(const struct fetch_pack_args *args,
60 const char *fmt, ...)
61 {
62 va_list params;
63
64 if (!args->verbose)
65 return;
66
67 va_start(params, fmt);
68 vfprintf(stderr, fmt, params);
69 va_end(params);
70 fputc('\n', stderr);
71 }
72
73 struct alternate_object_cache {
74 struct object **items;
75 size_t nr, alloc;
76 };
77
78 static void cache_one_alternate(const char *refname,
79 const struct object_id *oid,
80 void *vcache)
81 {
82 struct alternate_object_cache *cache = vcache;
83 struct object *obj = parse_object(oid);
84
85 if (!obj || (obj->flags & ALTERNATE))
86 return;
87
88 obj->flags |= ALTERNATE;
89 ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
90 cache->items[cache->nr++] = obj;
91 }
92
93 static void for_each_cached_alternate(void (*cb)(struct object *))
94 {
95 static int initialized;
96 static struct alternate_object_cache cache;
97 size_t i;
98
99 if (!initialized) {
100 for_each_alternate_ref(cache_one_alternate, &cache);
101 initialized = 1;
102 }
103
104 for (i = 0; i < cache.nr; i++)
105 cb(cache.items[i]);
106 }
107
108 static void rev_list_push(struct commit *commit, int mark)
109 {
110 if (!(commit->object.flags & mark)) {
111 commit->object.flags |= mark;
112
113 if (parse_commit(commit))
114 return;
115
116 prio_queue_put(&rev_list, commit);
117
118 if (!(commit->object.flags & COMMON))
119 non_common_revs++;
120 }
121 }
122
123 static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
124 {
125 struct object *o = deref_tag(parse_object(oid), refname, 0);
126
127 if (o && o->type == OBJ_COMMIT)
128 rev_list_push((struct commit *)o, SEEN);
129
130 return 0;
131 }
132
133 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
134 int flag, void *cb_data)
135 {
136 return rev_list_insert_ref(refname, oid);
137 }
138
139 static int clear_marks(const char *refname, const struct object_id *oid,
140 int flag, void *cb_data)
141 {
142 struct object *o = deref_tag(parse_object(oid), refname, 0);
143
144 if (o && o->type == OBJ_COMMIT)
145 clear_commit_marks((struct commit *)o,
146 COMMON | COMMON_REF | SEEN | POPPED);
147 return 0;
148 }
149
150 /*
151 This function marks a rev and its ancestors as common.
152 In some cases, it is desirable to mark only the ancestors (for example
153 when only the server does not yet know that they are common).
154 */
155
156 static void mark_common(struct commit *commit,
157 int ancestors_only, int dont_parse)
158 {
159 if (commit != NULL && !(commit->object.flags & COMMON)) {
160 struct object *o = (struct object *)commit;
161
162 if (!ancestors_only)
163 o->flags |= COMMON;
164
165 if (!(o->flags & SEEN))
166 rev_list_push(commit, SEEN);
167 else {
168 struct commit_list *parents;
169
170 if (!ancestors_only && !(o->flags & POPPED))
171 non_common_revs--;
172 if (!o->parsed && !dont_parse)
173 if (parse_commit(commit))
174 return;
175
176 for (parents = commit->parents;
177 parents;
178 parents = parents->next)
179 mark_common(parents->item, 0, dont_parse);
180 }
181 }
182 }
183
184 /*
185 Get the next rev to send, ignoring the common.
186 */
187
188 static const struct object_id *get_rev(void)
189 {
190 struct commit *commit = NULL;
191
192 while (commit == NULL) {
193 unsigned int mark;
194 struct commit_list *parents;
195
196 if (rev_list.nr == 0 || non_common_revs == 0)
197 return NULL;
198
199 commit = prio_queue_get(&rev_list);
200 parse_commit(commit);
201 parents = commit->parents;
202
203 commit->object.flags |= POPPED;
204 if (!(commit->object.flags & COMMON))
205 non_common_revs--;
206
207 if (commit->object.flags & COMMON) {
208 /* do not send "have", and ignore ancestors */
209 commit = NULL;
210 mark = COMMON | SEEN;
211 } else if (commit->object.flags & COMMON_REF)
212 /* send "have", and ignore ancestors */
213 mark = COMMON | SEEN;
214 else
215 /* send "have", also for its ancestors */
216 mark = SEEN;
217
218 while (parents) {
219 if (!(parents->item->object.flags & SEEN))
220 rev_list_push(parents->item, mark);
221 if (mark & COMMON)
222 mark_common(parents->item, 1, 0);
223 parents = parents->next;
224 }
225 }
226
227 return &commit->object.oid;
228 }
229
230 enum ack_type {
231 NAK = 0,
232 ACK,
233 ACK_continue,
234 ACK_common,
235 ACK_ready
236 };
237
238 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
239 {
240 if (args->stateless_rpc && args->deepen) {
241 /* If we sent a depth we will get back "duplicate"
242 * shallow and unshallow commands every time there
243 * is a block of have lines exchanged.
244 */
245 char *line;
246 while ((line = packet_read_line(fd, NULL))) {
247 if (starts_with(line, "shallow "))
248 continue;
249 if (starts_with(line, "unshallow "))
250 continue;
251 die(_("git fetch-pack: expected shallow list"));
252 }
253 }
254 }
255
256 static enum ack_type get_ack(int fd, struct object_id *result_oid)
257 {
258 int len;
259 char *line = packet_read_line(fd, &len);
260 const char *arg;
261
262 if (!len)
263 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
264 if (!strcmp(line, "NAK"))
265 return NAK;
266 if (skip_prefix(line, "ACK ", &arg)) {
267 if (!get_oid_hex(arg, result_oid)) {
268 arg += 40;
269 len -= arg - line;
270 if (len < 1)
271 return ACK;
272 if (strstr(arg, "continue"))
273 return ACK_continue;
274 if (strstr(arg, "common"))
275 return ACK_common;
276 if (strstr(arg, "ready"))
277 return ACK_ready;
278 return ACK;
279 }
280 }
281 if (skip_prefix(line, "ERR ", &arg))
282 die(_("remote error: %s"), arg);
283 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
284 }
285
286 static void send_request(struct fetch_pack_args *args,
287 int fd, struct strbuf *buf)
288 {
289 if (args->stateless_rpc) {
290 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
291 packet_flush(fd);
292 } else
293 write_or_die(fd, buf->buf, buf->len);
294 }
295
296 static void insert_one_alternate_object(struct object *obj)
297 {
298 rev_list_insert_ref(NULL, &obj->oid);
299 }
300
301 #define INITIAL_FLUSH 16
302 #define PIPESAFE_FLUSH 32
303 #define LARGE_FLUSH 16384
304
305 static int next_flush(struct fetch_pack_args *args, int count)
306 {
307 if (args->stateless_rpc) {
308 if (count < LARGE_FLUSH)
309 count <<= 1;
310 else
311 count = count * 11 / 10;
312 } else {
313 if (count < PIPESAFE_FLUSH)
314 count <<= 1;
315 else
316 count += PIPESAFE_FLUSH;
317 }
318 return count;
319 }
320
321 static int find_common(struct fetch_pack_args *args,
322 int fd[2], struct object_id *result_oid,
323 struct ref *refs)
324 {
325 int fetching;
326 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
327 const struct object_id *oid;
328 unsigned in_vain = 0;
329 int got_continue = 0;
330 int got_ready = 0;
331 struct strbuf req_buf = STRBUF_INIT;
332 size_t state_len = 0;
333
334 if (args->stateless_rpc && multi_ack == 1)
335 die(_("--stateless-rpc requires multi_ack_detailed"));
336 if (marked)
337 for_each_ref(clear_marks, NULL);
338 marked = 1;
339
340 for_each_ref(rev_list_insert_ref_oid, NULL);
341 for_each_cached_alternate(insert_one_alternate_object);
342
343 fetching = 0;
344 for ( ; refs ; refs = refs->next) {
345 struct object_id *remote = &refs->old_oid;
346 const char *remote_hex;
347 struct object *o;
348
349 /*
350 * If that object is complete (i.e. it is an ancestor of a
351 * local ref), we tell them we have it but do not have to
352 * tell them about its ancestors, which they already know
353 * about.
354 *
355 * We use lookup_object here because we are only
356 * interested in the case we *know* the object is
357 * reachable and we have already scanned it.
358 */
359 if (((o = lookup_object(remote->hash)) != NULL) &&
360 (o->flags & COMPLETE)) {
361 continue;
362 }
363
364 remote_hex = oid_to_hex(remote);
365 if (!fetching) {
366 struct strbuf c = STRBUF_INIT;
367 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
368 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
369 if (no_done) strbuf_addstr(&c, " no-done");
370 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
371 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
372 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
373 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
374 if (args->no_progress) strbuf_addstr(&c, " no-progress");
375 if (args->include_tag) strbuf_addstr(&c, " include-tag");
376 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
377 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
378 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
379 if (agent_supported) strbuf_addf(&c, " agent=%s",
380 git_user_agent_sanitized());
381 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
382 strbuf_release(&c);
383 } else
384 packet_buf_write(&req_buf, "want %s\n", remote_hex);
385 fetching++;
386 }
387
388 if (!fetching) {
389 strbuf_release(&req_buf);
390 packet_flush(fd[1]);
391 return 1;
392 }
393
394 if (is_repository_shallow())
395 write_shallow_commits(&req_buf, 1, NULL);
396 if (args->depth > 0)
397 packet_buf_write(&req_buf, "deepen %d", args->depth);
398 if (args->deepen_since) {
399 timestamp_t max_age = approxidate(args->deepen_since);
400 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
401 }
402 if (args->deepen_not) {
403 int i;
404 for (i = 0; i < args->deepen_not->nr; i++) {
405 struct string_list_item *s = args->deepen_not->items + i;
406 packet_buf_write(&req_buf, "deepen-not %s", s->string);
407 }
408 }
409 packet_buf_flush(&req_buf);
410 state_len = req_buf.len;
411
412 if (args->deepen) {
413 char *line;
414 const char *arg;
415 struct object_id oid;
416
417 send_request(args, fd[1], &req_buf);
418 while ((line = packet_read_line(fd[0], NULL))) {
419 if (skip_prefix(line, "shallow ", &arg)) {
420 if (get_oid_hex(arg, &oid))
421 die(_("invalid shallow line: %s"), line);
422 register_shallow(&oid);
423 continue;
424 }
425 if (skip_prefix(line, "unshallow ", &arg)) {
426 if (get_oid_hex(arg, &oid))
427 die(_("invalid unshallow line: %s"), line);
428 if (!lookup_object(oid.hash))
429 die(_("object not found: %s"), line);
430 /* make sure that it is parsed as shallow */
431 if (!parse_object(&oid))
432 die(_("error in object: %s"), line);
433 if (unregister_shallow(&oid))
434 die(_("no shallow found: %s"), line);
435 continue;
436 }
437 die(_("expected shallow/unshallow, got %s"), line);
438 }
439 } else if (!args->stateless_rpc)
440 send_request(args, fd[1], &req_buf);
441
442 if (!args->stateless_rpc) {
443 /* If we aren't using the stateless-rpc interface
444 * we don't need to retain the headers.
445 */
446 strbuf_setlen(&req_buf, 0);
447 state_len = 0;
448 }
449
450 flushes = 0;
451 retval = -1;
452 while ((oid = get_rev())) {
453 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
454 print_verbose(args, "have %s", oid_to_hex(oid));
455 in_vain++;
456 if (flush_at <= ++count) {
457 int ack;
458
459 packet_buf_flush(&req_buf);
460 send_request(args, fd[1], &req_buf);
461 strbuf_setlen(&req_buf, state_len);
462 flushes++;
463 flush_at = next_flush(args, count);
464
465 /*
466 * We keep one window "ahead" of the other side, and
467 * will wait for an ACK only on the next one
468 */
469 if (!args->stateless_rpc && count == INITIAL_FLUSH)
470 continue;
471
472 consume_shallow_list(args, fd[0]);
473 do {
474 ack = get_ack(fd[0], result_oid);
475 if (ack)
476 print_verbose(args, _("got %s %d %s"), "ack",
477 ack, oid_to_hex(result_oid));
478 switch (ack) {
479 case ACK:
480 flushes = 0;
481 multi_ack = 0;
482 retval = 0;
483 goto done;
484 case ACK_common:
485 case ACK_ready:
486 case ACK_continue: {
487 struct commit *commit =
488 lookup_commit(result_oid);
489 if (!commit)
490 die(_("invalid commit %s"), oid_to_hex(result_oid));
491 if (args->stateless_rpc
492 && ack == ACK_common
493 && !(commit->object.flags & COMMON)) {
494 /* We need to replay the have for this object
495 * on the next RPC request so the peer knows
496 * it is in common with us.
497 */
498 const char *hex = oid_to_hex(result_oid);
499 packet_buf_write(&req_buf, "have %s\n", hex);
500 state_len = req_buf.len;
501 /*
502 * Reset in_vain because an ack
503 * for this commit has not been
504 * seen.
505 */
506 in_vain = 0;
507 } else if (!args->stateless_rpc
508 || ack != ACK_common)
509 in_vain = 0;
510 mark_common(commit, 0, 1);
511 retval = 0;
512 got_continue = 1;
513 if (ack == ACK_ready) {
514 clear_prio_queue(&rev_list);
515 got_ready = 1;
516 }
517 break;
518 }
519 }
520 } while (ack);
521 flushes--;
522 if (got_continue && MAX_IN_VAIN < in_vain) {
523 print_verbose(args, _("giving up"));
524 break; /* give up */
525 }
526 }
527 }
528 done:
529 if (!got_ready || !no_done) {
530 packet_buf_write(&req_buf, "done\n");
531 send_request(args, fd[1], &req_buf);
532 }
533 print_verbose(args, _("done"));
534 if (retval != 0) {
535 multi_ack = 0;
536 flushes++;
537 }
538 strbuf_release(&req_buf);
539
540 if (!got_ready || !no_done)
541 consume_shallow_list(args, fd[0]);
542 while (flushes || multi_ack) {
543 int ack = get_ack(fd[0], result_oid);
544 if (ack) {
545 print_verbose(args, _("got %s (%d) %s"), "ack",
546 ack, oid_to_hex(result_oid));
547 if (ack == ACK)
548 return 0;
549 multi_ack = 1;
550 continue;
551 }
552 flushes--;
553 }
554 /* it is no error to fetch into a completely empty repo */
555 return count ? retval : 0;
556 }
557
558 static struct commit_list *complete;
559
560 static int mark_complete(const struct object_id *oid)
561 {
562 struct object *o = parse_object(oid);
563
564 while (o && o->type == OBJ_TAG) {
565 struct tag *t = (struct tag *) o;
566 if (!t->tagged)
567 break; /* broken repository */
568 o->flags |= COMPLETE;
569 o = parse_object(&t->tagged->oid);
570 }
571 if (o && o->type == OBJ_COMMIT) {
572 struct commit *commit = (struct commit *)o;
573 if (!(commit->object.flags & COMPLETE)) {
574 commit->object.flags |= COMPLETE;
575 commit_list_insert(commit, &complete);
576 }
577 }
578 return 0;
579 }
580
581 static int mark_complete_oid(const char *refname, const struct object_id *oid,
582 int flag, void *cb_data)
583 {
584 return mark_complete(oid);
585 }
586
587 static void mark_recent_complete_commits(struct fetch_pack_args *args,
588 timestamp_t cutoff)
589 {
590 while (complete && cutoff <= complete->item->date) {
591 print_verbose(args, _("Marking %s as complete"),
592 oid_to_hex(&complete->item->object.oid));
593 pop_most_recent_commit(&complete, COMPLETE);
594 }
595 }
596
597 static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
598 {
599 for (; refs; refs = refs->next)
600 oidset_insert(oids, &refs->old_oid);
601 }
602
603 static int tip_oids_contain(struct oidset *tip_oids,
604 struct ref *unmatched, struct ref *newlist,
605 const struct object_id *id)
606 {
607 /*
608 * Note that this only looks at the ref lists the first time it's
609 * called. This works out in filter_refs() because even though it may
610 * add to "newlist" between calls, the additions will always be for
611 * oids that are already in the set.
612 */
613 if (!tip_oids->map.tablesize) {
614 add_refs_to_oidset(tip_oids, unmatched);
615 add_refs_to_oidset(tip_oids, newlist);
616 }
617 return oidset_contains(tip_oids, id);
618 }
619
620 static void filter_refs(struct fetch_pack_args *args,
621 struct ref **refs,
622 struct ref **sought, int nr_sought)
623 {
624 struct ref *newlist = NULL;
625 struct ref **newtail = &newlist;
626 struct ref *unmatched = NULL;
627 struct ref *ref, *next;
628 struct oidset tip_oids = OIDSET_INIT;
629 int i;
630
631 i = 0;
632 for (ref = *refs; ref; ref = next) {
633 int keep = 0;
634 next = ref->next;
635
636 if (starts_with(ref->name, "refs/") &&
637 check_refname_format(ref->name, 0))
638 ; /* trash */
639 else {
640 while (i < nr_sought) {
641 int cmp = strcmp(ref->name, sought[i]->name);
642 if (cmp < 0)
643 break; /* definitely do not have it */
644 else if (cmp == 0) {
645 keep = 1; /* definitely have it */
646 sought[i]->match_status = REF_MATCHED;
647 }
648 i++;
649 }
650 }
651
652 if (!keep && args->fetch_all &&
653 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
654 keep = 1;
655
656 if (keep) {
657 *newtail = ref;
658 ref->next = NULL;
659 newtail = &ref->next;
660 } else {
661 ref->next = unmatched;
662 unmatched = ref;
663 }
664 }
665
666 /* Append unmatched requests to the list */
667 for (i = 0; i < nr_sought; i++) {
668 struct object_id oid;
669 const char *p;
670
671 ref = sought[i];
672 if (ref->match_status != REF_NOT_MATCHED)
673 continue;
674 if (parse_oid_hex(ref->name, &oid, &p) ||
675 *p != '\0' ||
676 oidcmp(&oid, &ref->old_oid))
677 continue;
678
679 if ((allow_unadvertised_object_request &
680 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
681 tip_oids_contain(&tip_oids, unmatched, newlist,
682 &ref->old_oid)) {
683 ref->match_status = REF_MATCHED;
684 *newtail = copy_ref(ref);
685 newtail = &(*newtail)->next;
686 } else {
687 ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
688 }
689 }
690
691 oidset_clear(&tip_oids);
692 for (ref = unmatched; ref; ref = next) {
693 next = ref->next;
694 free(ref);
695 }
696
697 *refs = newlist;
698 }
699
700 static void mark_alternate_complete(struct object *obj)
701 {
702 mark_complete(&obj->oid);
703 }
704
705 static int everything_local(struct fetch_pack_args *args,
706 struct ref **refs,
707 struct ref **sought, int nr_sought)
708 {
709 struct ref *ref;
710 int retval;
711 timestamp_t cutoff = 0;
712
713 save_commit_buffer = 0;
714
715 for (ref = *refs; ref; ref = ref->next) {
716 struct object *o;
717
718 if (!has_object_file_with_flags(&ref->old_oid,
719 OBJECT_INFO_QUICK))
720 continue;
721
722 o = parse_object(&ref->old_oid);
723 if (!o)
724 continue;
725
726 /* We already have it -- which may mean that we were
727 * in sync with the other side at some time after
728 * that (it is OK if we guess wrong here).
729 */
730 if (o->type == OBJ_COMMIT) {
731 struct commit *commit = (struct commit *)o;
732 if (!cutoff || cutoff < commit->date)
733 cutoff = commit->date;
734 }
735 }
736
737 if (!args->deepen) {
738 for_each_ref(mark_complete_oid, NULL);
739 for_each_cached_alternate(mark_alternate_complete);
740 commit_list_sort_by_date(&complete);
741 if (cutoff)
742 mark_recent_complete_commits(args, cutoff);
743 }
744
745 /*
746 * Mark all complete remote refs as common refs.
747 * Don't mark them common yet; the server has to be told so first.
748 */
749 for (ref = *refs; ref; ref = ref->next) {
750 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
751 NULL, 0);
752
753 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
754 continue;
755
756 if (!(o->flags & SEEN)) {
757 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
758
759 mark_common((struct commit *)o, 1, 1);
760 }
761 }
762
763 filter_refs(args, refs, sought, nr_sought);
764
765 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
766 const struct object_id *remote = &ref->old_oid;
767 struct object *o;
768
769 o = lookup_object(remote->hash);
770 if (!o || !(o->flags & COMPLETE)) {
771 retval = 0;
772 print_verbose(args, "want %s (%s)", oid_to_hex(remote),
773 ref->name);
774 continue;
775 }
776 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
777 ref->name);
778 }
779 return retval;
780 }
781
782 static int sideband_demux(int in, int out, void *data)
783 {
784 int *xd = data;
785 int ret;
786
787 ret = recv_sideband("fetch-pack", xd[0], out);
788 close(out);
789 return ret;
790 }
791
792 static int get_pack(struct fetch_pack_args *args,
793 int xd[2], char **pack_lockfile)
794 {
795 struct async demux;
796 int do_keep = args->keep_pack;
797 const char *cmd_name;
798 struct pack_header header;
799 int pass_header = 0;
800 struct child_process cmd = CHILD_PROCESS_INIT;
801 int ret;
802
803 memset(&demux, 0, sizeof(demux));
804 if (use_sideband) {
805 /* xd[] is talking with upload-pack; subprocess reads from
806 * xd[0], spits out band#2 to stderr, and feeds us band#1
807 * through demux->out.
808 */
809 demux.proc = sideband_demux;
810 demux.data = xd;
811 demux.out = -1;
812 demux.isolate_sigpipe = 1;
813 if (start_async(&demux))
814 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
815 }
816 else
817 demux.out = xd[0];
818
819 if (!args->keep_pack && unpack_limit) {
820
821 if (read_pack_header(demux.out, &header))
822 die(_("protocol error: bad pack header"));
823 pass_header = 1;
824 if (ntohl(header.hdr_entries) < unpack_limit)
825 do_keep = 0;
826 else
827 do_keep = 1;
828 }
829
830 if (alternate_shallow_file) {
831 argv_array_push(&cmd.args, "--shallow-file");
832 argv_array_push(&cmd.args, alternate_shallow_file);
833 }
834
835 if (do_keep) {
836 if (pack_lockfile)
837 cmd.out = -1;
838 cmd_name = "index-pack";
839 argv_array_push(&cmd.args, cmd_name);
840 argv_array_push(&cmd.args, "--stdin");
841 if (!args->quiet && !args->no_progress)
842 argv_array_push(&cmd.args, "-v");
843 if (args->use_thin_pack)
844 argv_array_push(&cmd.args, "--fix-thin");
845 if (args->lock_pack || unpack_limit) {
846 char hostname[HOST_NAME_MAX + 1];
847 if (xgethostname(hostname, sizeof(hostname)))
848 xsnprintf(hostname, sizeof(hostname), "localhost");
849 argv_array_pushf(&cmd.args,
850 "--keep=fetch-pack %"PRIuMAX " on %s",
851 (uintmax_t)getpid(), hostname);
852 }
853 if (args->check_self_contained_and_connected)
854 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
855 }
856 else {
857 cmd_name = "unpack-objects";
858 argv_array_push(&cmd.args, cmd_name);
859 if (args->quiet || args->no_progress)
860 argv_array_push(&cmd.args, "-q");
861 args->check_self_contained_and_connected = 0;
862 }
863
864 if (pass_header)
865 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
866 ntohl(header.hdr_version),
867 ntohl(header.hdr_entries));
868 if (fetch_fsck_objects >= 0
869 ? fetch_fsck_objects
870 : transfer_fsck_objects >= 0
871 ? transfer_fsck_objects
872 : 0)
873 argv_array_push(&cmd.args, "--strict");
874
875 cmd.in = demux.out;
876 cmd.git_cmd = 1;
877 if (start_command(&cmd))
878 die(_("fetch-pack: unable to fork off %s"), cmd_name);
879 if (do_keep && pack_lockfile) {
880 *pack_lockfile = index_pack_lockfile(cmd.out);
881 close(cmd.out);
882 }
883
884 if (!use_sideband)
885 /* Closed by start_command() */
886 xd[0] = -1;
887
888 ret = finish_command(&cmd);
889 if (!ret || (args->check_self_contained_and_connected && ret == 1))
890 args->self_contained_and_connected =
891 args->check_self_contained_and_connected &&
892 ret == 0;
893 else
894 die(_("%s failed"), cmd_name);
895 if (use_sideband && finish_async(&demux))
896 die(_("error in sideband demultiplexer"));
897 return 0;
898 }
899
900 static int cmp_ref_by_name(const void *a_, const void *b_)
901 {
902 const struct ref *a = *((const struct ref **)a_);
903 const struct ref *b = *((const struct ref **)b_);
904 return strcmp(a->name, b->name);
905 }
906
907 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
908 int fd[2],
909 const struct ref *orig_ref,
910 struct ref **sought, int nr_sought,
911 struct shallow_info *si,
912 char **pack_lockfile)
913 {
914 struct ref *ref = copy_ref_list(orig_ref);
915 struct object_id oid;
916 const char *agent_feature;
917 int agent_len;
918
919 sort_ref_list(&ref, ref_compare_name);
920 QSORT(sought, nr_sought, cmp_ref_by_name);
921
922 if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
923 die(_("Server does not support shallow clients"));
924 if (args->depth > 0 || args->deepen_since || args->deepen_not)
925 args->deepen = 1;
926 if (server_supports("multi_ack_detailed")) {
927 print_verbose(args, _("Server supports multi_ack_detailed"));
928 multi_ack = 2;
929 if (server_supports("no-done")) {
930 print_verbose(args, _("Server supports no-done"));
931 if (args->stateless_rpc)
932 no_done = 1;
933 }
934 }
935 else if (server_supports("multi_ack")) {
936 print_verbose(args, _("Server supports multi_ack"));
937 multi_ack = 1;
938 }
939 if (server_supports("side-band-64k")) {
940 print_verbose(args, _("Server supports side-band-64k"));
941 use_sideband = 2;
942 }
943 else if (server_supports("side-band")) {
944 print_verbose(args, _("Server supports side-band"));
945 use_sideband = 1;
946 }
947 if (server_supports("allow-tip-sha1-in-want")) {
948 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
949 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
950 }
951 if (server_supports("allow-reachable-sha1-in-want")) {
952 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
953 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
954 }
955 if (!server_supports("thin-pack"))
956 args->use_thin_pack = 0;
957 if (!server_supports("no-progress"))
958 args->no_progress = 0;
959 if (!server_supports("include-tag"))
960 args->include_tag = 0;
961 if (server_supports("ofs-delta"))
962 print_verbose(args, _("Server supports ofs-delta"));
963 else
964 prefer_ofs_delta = 0;
965
966 if ((agent_feature = server_feature_value("agent", &agent_len))) {
967 agent_supported = 1;
968 if (agent_len)
969 print_verbose(args, _("Server version is %.*s"),
970 agent_len, agent_feature);
971 }
972 if (server_supports("deepen-since"))
973 deepen_since_ok = 1;
974 else if (args->deepen_since)
975 die(_("Server does not support --shallow-since"));
976 if (server_supports("deepen-not"))
977 deepen_not_ok = 1;
978 else if (args->deepen_not)
979 die(_("Server does not support --shallow-exclude"));
980 if (!server_supports("deepen-relative") && args->deepen_relative)
981 die(_("Server does not support --deepen"));
982
983 if (everything_local(args, &ref, sought, nr_sought)) {
984 packet_flush(fd[1]);
985 goto all_done;
986 }
987 if (find_common(args, fd, &oid, ref) < 0)
988 if (!args->keep_pack)
989 /* When cloning, it is not unusual to have
990 * no common commit.
991 */
992 warning(_("no common commits"));
993
994 if (args->stateless_rpc)
995 packet_flush(fd[1]);
996 if (args->deepen)
997 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
998 NULL);
999 else if (si->nr_ours || si->nr_theirs)
1000 alternate_shallow_file = setup_temporary_shallow(si->shallow);
1001 else
1002 alternate_shallow_file = NULL;
1003 if (get_pack(args, fd, pack_lockfile))
1004 die(_("git fetch-pack: fetch failed."));
1005
1006 all_done:
1007 return ref;
1008 }
1009
1010 static void fetch_pack_config(void)
1011 {
1012 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1013 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1014 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1015 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1016 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1017
1018 git_config(git_default_config, NULL);
1019 }
1020
1021 static void fetch_pack_setup(void)
1022 {
1023 static int did_setup;
1024 if (did_setup)
1025 return;
1026 fetch_pack_config();
1027 if (0 <= transfer_unpack_limit)
1028 unpack_limit = transfer_unpack_limit;
1029 else if (0 <= fetch_unpack_limit)
1030 unpack_limit = fetch_unpack_limit;
1031 did_setup = 1;
1032 }
1033
1034 static int remove_duplicates_in_refs(struct ref **ref, int nr)
1035 {
1036 struct string_list names = STRING_LIST_INIT_NODUP;
1037 int src, dst;
1038
1039 for (src = dst = 0; src < nr; src++) {
1040 struct string_list_item *item;
1041 item = string_list_insert(&names, ref[src]->name);
1042 if (item->util)
1043 continue; /* already have it */
1044 item->util = ref[src];
1045 if (src != dst)
1046 ref[dst] = ref[src];
1047 dst++;
1048 }
1049 for (src = dst; src < nr; src++)
1050 ref[src] = NULL;
1051 string_list_clear(&names, 0);
1052 return dst;
1053 }
1054
1055 static void update_shallow(struct fetch_pack_args *args,
1056 struct ref **sought, int nr_sought,
1057 struct shallow_info *si)
1058 {
1059 struct oid_array ref = OID_ARRAY_INIT;
1060 int *status;
1061 int i;
1062
1063 if (args->deepen && alternate_shallow_file) {
1064 if (*alternate_shallow_file == '\0') { /* --unshallow */
1065 unlink_or_warn(git_path_shallow());
1066 rollback_lock_file(&shallow_lock);
1067 } else
1068 commit_lock_file(&shallow_lock);
1069 return;
1070 }
1071
1072 if (!si->shallow || !si->shallow->nr)
1073 return;
1074
1075 if (args->cloning) {
1076 /*
1077 * remote is shallow, but this is a clone, there are
1078 * no objects in repo to worry about. Accept any
1079 * shallow points that exist in the pack (iow in repo
1080 * after get_pack() and reprepare_packed_git())
1081 */
1082 struct oid_array extra = OID_ARRAY_INIT;
1083 struct object_id *oid = si->shallow->oid;
1084 for (i = 0; i < si->shallow->nr; i++)
1085 if (has_object_file(&oid[i]))
1086 oid_array_append(&extra, &oid[i]);
1087 if (extra.nr) {
1088 setup_alternate_shallow(&shallow_lock,
1089 &alternate_shallow_file,
1090 &extra);
1091 commit_lock_file(&shallow_lock);
1092 }
1093 oid_array_clear(&extra);
1094 return;
1095 }
1096
1097 if (!si->nr_ours && !si->nr_theirs)
1098 return;
1099
1100 remove_nonexistent_theirs_shallow(si);
1101 if (!si->nr_ours && !si->nr_theirs)
1102 return;
1103 for (i = 0; i < nr_sought; i++)
1104 oid_array_append(&ref, &sought[i]->old_oid);
1105 si->ref = &ref;
1106
1107 if (args->update_shallow) {
1108 /*
1109 * remote is also shallow, .git/shallow may be updated
1110 * so all refs can be accepted. Make sure we only add
1111 * shallow roots that are actually reachable from new
1112 * refs.
1113 */
1114 struct oid_array extra = OID_ARRAY_INIT;
1115 struct object_id *oid = si->shallow->oid;
1116 assign_shallow_commits_to_refs(si, NULL, NULL);
1117 if (!si->nr_ours && !si->nr_theirs) {
1118 oid_array_clear(&ref);
1119 return;
1120 }
1121 for (i = 0; i < si->nr_ours; i++)
1122 oid_array_append(&extra, &oid[si->ours[i]]);
1123 for (i = 0; i < si->nr_theirs; i++)
1124 oid_array_append(&extra, &oid[si->theirs[i]]);
1125 setup_alternate_shallow(&shallow_lock,
1126 &alternate_shallow_file,
1127 &extra);
1128 commit_lock_file(&shallow_lock);
1129 oid_array_clear(&extra);
1130 oid_array_clear(&ref);
1131 return;
1132 }
1133
1134 /*
1135 * remote is also shallow, check what ref is safe to update
1136 * without updating .git/shallow
1137 */
1138 status = xcalloc(nr_sought, sizeof(*status));
1139 assign_shallow_commits_to_refs(si, NULL, status);
1140 if (si->nr_ours || si->nr_theirs) {
1141 for (i = 0; i < nr_sought; i++)
1142 if (status[i])
1143 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1144 }
1145 free(status);
1146 oid_array_clear(&ref);
1147 }
1148
1149 struct ref *fetch_pack(struct fetch_pack_args *args,
1150 int fd[], struct child_process *conn,
1151 const struct ref *ref,
1152 const char *dest,
1153 struct ref **sought, int nr_sought,
1154 struct oid_array *shallow,
1155 char **pack_lockfile)
1156 {
1157 struct ref *ref_cpy;
1158 struct shallow_info si;
1159
1160 fetch_pack_setup();
1161 if (nr_sought)
1162 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1163
1164 if (!ref) {
1165 packet_flush(fd[1]);
1166 die(_("no matching remote head"));
1167 }
1168 prepare_shallow_info(&si, shallow);
1169 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1170 &si, pack_lockfile);
1171 reprepare_packed_git();
1172 update_shallow(args, sought, nr_sought, &si);
1173 clear_shallow_info(&si);
1174 return ref_cpy;
1175 }
1176
1177 int report_unmatched_refs(struct ref **sought, int nr_sought)
1178 {
1179 int i, ret = 0;
1180
1181 for (i = 0; i < nr_sought; i++) {
1182 if (!sought[i])
1183 continue;
1184 switch (sought[i]->match_status) {
1185 case REF_MATCHED:
1186 continue;
1187 case REF_NOT_MATCHED:
1188 error(_("no such remote ref %s"), sought[i]->name);
1189 break;
1190 case REF_UNADVERTISED_NOT_ALLOWED:
1191 error(_("Server does not allow request for unadvertised object %s"),
1192 sought[i]->name);
1193 break;
1194 }
1195 ret = 1;
1196 }
1197 return ret;
1198 }
git for Debian