get_packed_ref_cache(): take a `packed_ref_store *` parameter
[git.git] / fetch-pack.c
blob44fa047bf41c25e8be91f91dd71240a4ce92b9db
1 #include "cache.h"
2 #include "lockfile.h"
3 #include "refs.h"
4 #include "pkt-line.h"
5 #include "commit.h"
6 #include "tag.h"
7 #include "exec_cmd.h"
8 #include "pack.h"
9 #include "sideband.h"
10 #include "fetch-pack.h"
11 #include "remote.h"
12 #include "run-command.h"
13 #include "connect.h"
14 #include "transport.h"
15 #include "version.h"
16 #include "prio-queue.h"
17 #include "sha1-array.h"
19 static int transfer_unpack_limit = -1;
20 static int fetch_unpack_limit = -1;
21 static int unpack_limit = 100;
22 static int prefer_ofs_delta = 1;
23 static int no_done;
24 static int deepen_since_ok;
25 static int deepen_not_ok;
26 static int fetch_fsck_objects = -1;
27 static int transfer_fsck_objects = -1;
28 static int agent_supported;
29 static struct lock_file shallow_lock;
30 static const char *alternate_shallow_file;
32 /* Remember to update object flag allocation in object.h */
33 #define COMPLETE (1U << 0)
34 #define COMMON (1U << 1)
35 #define COMMON_REF (1U << 2)
36 #define SEEN (1U << 3)
37 #define POPPED (1U << 4)
38 #define ALTERNATE (1U << 5)
40 static int marked;
43 * After sending this many "have"s if we do not get any new ACK , we
44 * give up traversing our history.
46 #define MAX_IN_VAIN 256
48 static struct prio_queue rev_list = { compare_commits_by_commit_date };
49 static int non_common_revs, multi_ack, use_sideband;
50 /* Allow specifying sha1 if it is a ref tip. */
51 #define ALLOW_TIP_SHA1 01
52 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
53 #define ALLOW_REACHABLE_SHA1 02
54 static unsigned int allow_unadvertised_object_request;
56 __attribute__((format (printf, 2, 3)))
57 static inline void print_verbose(const struct fetch_pack_args *args,
58 const char *fmt, ...)
60 va_list params;
62 if (!args->verbose)
63 return;
65 va_start(params, fmt);
66 vfprintf(stderr, fmt, params);
67 va_end(params);
68 fputc('\n', stderr);
71 struct alternate_object_cache {
72 struct object **items;
73 size_t nr, alloc;
76 static void cache_one_alternate(const char *refname,
77 const struct object_id *oid,
78 void *vcache)
80 struct alternate_object_cache *cache = vcache;
81 struct object *obj = parse_object(oid);
83 if (!obj || (obj->flags & ALTERNATE))
84 return;
86 obj->flags |= ALTERNATE;
87 ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
88 cache->items[cache->nr++] = obj;
91 static void for_each_cached_alternate(void (*cb)(struct object *))
93 static int initialized;
94 static struct alternate_object_cache cache;
95 size_t i;
97 if (!initialized) {
98 for_each_alternate_ref(cache_one_alternate, &cache);
99 initialized = 1;
102 for (i = 0; i < cache.nr; i++)
103 cb(cache.items[i]);
106 static void rev_list_push(struct commit *commit, int mark)
108 if (!(commit->object.flags & mark)) {
109 commit->object.flags |= mark;
111 if (parse_commit(commit))
112 return;
114 prio_queue_put(&rev_list, commit);
116 if (!(commit->object.flags & COMMON))
117 non_common_revs++;
121 static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
123 struct object *o = deref_tag(parse_object(oid), refname, 0);
125 if (o && o->type == OBJ_COMMIT)
126 rev_list_push((struct commit *)o, SEEN);
128 return 0;
131 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
132 int flag, void *cb_data)
134 return rev_list_insert_ref(refname, oid);
137 static int clear_marks(const char *refname, const struct object_id *oid,
138 int flag, void *cb_data)
140 struct object *o = deref_tag(parse_object(oid), refname, 0);
142 if (o && o->type == OBJ_COMMIT)
143 clear_commit_marks((struct commit *)o,
144 COMMON | COMMON_REF | SEEN | POPPED);
145 return 0;
149 This function marks a rev and its ancestors as common.
150 In some cases, it is desirable to mark only the ancestors (for example
151 when only the server does not yet know that they are common).
154 static void mark_common(struct commit *commit,
155 int ancestors_only, int dont_parse)
157 if (commit != NULL && !(commit->object.flags & COMMON)) {
158 struct object *o = (struct object *)commit;
160 if (!ancestors_only)
161 o->flags |= COMMON;
163 if (!(o->flags & SEEN))
164 rev_list_push(commit, SEEN);
165 else {
166 struct commit_list *parents;
168 if (!ancestors_only && !(o->flags & POPPED))
169 non_common_revs--;
170 if (!o->parsed && !dont_parse)
171 if (parse_commit(commit))
172 return;
174 for (parents = commit->parents;
175 parents;
176 parents = parents->next)
177 mark_common(parents->item, 0, dont_parse);
183 Get the next rev to send, ignoring the common.
186 static const struct object_id *get_rev(void)
188 struct commit *commit = NULL;
190 while (commit == NULL) {
191 unsigned int mark;
192 struct commit_list *parents;
194 if (rev_list.nr == 0 || non_common_revs == 0)
195 return NULL;
197 commit = prio_queue_get(&rev_list);
198 parse_commit(commit);
199 parents = commit->parents;
201 commit->object.flags |= POPPED;
202 if (!(commit->object.flags & COMMON))
203 non_common_revs--;
205 if (commit->object.flags & COMMON) {
206 /* do not send "have", and ignore ancestors */
207 commit = NULL;
208 mark = COMMON | SEEN;
209 } else if (commit->object.flags & COMMON_REF)
210 /* send "have", and ignore ancestors */
211 mark = COMMON | SEEN;
212 else
213 /* send "have", also for its ancestors */
214 mark = SEEN;
216 while (parents) {
217 if (!(parents->item->object.flags & SEEN))
218 rev_list_push(parents->item, mark);
219 if (mark & COMMON)
220 mark_common(parents->item, 1, 0);
221 parents = parents->next;
225 return &commit->object.oid;
228 enum ack_type {
229 NAK = 0,
230 ACK,
231 ACK_continue,
232 ACK_common,
233 ACK_ready
236 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
238 if (args->stateless_rpc && args->deepen) {
239 /* If we sent a depth we will get back "duplicate"
240 * shallow and unshallow commands every time there
241 * is a block of have lines exchanged.
243 char *line;
244 while ((line = packet_read_line(fd, NULL))) {
245 if (starts_with(line, "shallow "))
246 continue;
247 if (starts_with(line, "unshallow "))
248 continue;
249 die(_("git fetch-pack: expected shallow list"));
254 static enum ack_type get_ack(int fd, struct object_id *result_oid)
256 int len;
257 char *line = packet_read_line(fd, &len);
258 const char *arg;
260 if (!len)
261 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
262 if (!strcmp(line, "NAK"))
263 return NAK;
264 if (skip_prefix(line, "ACK ", &arg)) {
265 if (!get_oid_hex(arg, result_oid)) {
266 arg += 40;
267 len -= arg - line;
268 if (len < 1)
269 return ACK;
270 if (strstr(arg, "continue"))
271 return ACK_continue;
272 if (strstr(arg, "common"))
273 return ACK_common;
274 if (strstr(arg, "ready"))
275 return ACK_ready;
276 return ACK;
279 if (skip_prefix(line, "ERR ", &arg))
280 die(_("remote error: %s"), arg);
281 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
284 static void send_request(struct fetch_pack_args *args,
285 int fd, struct strbuf *buf)
287 if (args->stateless_rpc) {
288 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
289 packet_flush(fd);
290 } else
291 write_or_die(fd, buf->buf, buf->len);
294 static void insert_one_alternate_object(struct object *obj)
296 rev_list_insert_ref(NULL, &obj->oid);
299 #define INITIAL_FLUSH 16
300 #define PIPESAFE_FLUSH 32
301 #define LARGE_FLUSH 16384
303 static int next_flush(struct fetch_pack_args *args, int count)
305 if (args->stateless_rpc) {
306 if (count < LARGE_FLUSH)
307 count <<= 1;
308 else
309 count = count * 11 / 10;
310 } else {
311 if (count < PIPESAFE_FLUSH)
312 count <<= 1;
313 else
314 count += PIPESAFE_FLUSH;
316 return count;
319 static int find_common(struct fetch_pack_args *args,
320 int fd[2], struct object_id *result_oid,
321 struct ref *refs)
323 int fetching;
324 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
325 const struct object_id *oid;
326 unsigned in_vain = 0;
327 int got_continue = 0;
328 int got_ready = 0;
329 struct strbuf req_buf = STRBUF_INIT;
330 size_t state_len = 0;
332 if (args->stateless_rpc && multi_ack == 1)
333 die(_("--stateless-rpc requires multi_ack_detailed"));
334 if (marked)
335 for_each_ref(clear_marks, NULL);
336 marked = 1;
338 for_each_ref(rev_list_insert_ref_oid, NULL);
339 for_each_cached_alternate(insert_one_alternate_object);
341 fetching = 0;
342 for ( ; refs ; refs = refs->next) {
343 struct object_id *remote = &refs->old_oid;
344 const char *remote_hex;
345 struct object *o;
348 * If that object is complete (i.e. it is an ancestor of a
349 * local ref), we tell them we have it but do not have to
350 * tell them about its ancestors, which they already know
351 * about.
353 * We use lookup_object here because we are only
354 * interested in the case we *know* the object is
355 * reachable and we have already scanned it.
357 if (((o = lookup_object(remote->hash)) != NULL) &&
358 (o->flags & COMPLETE)) {
359 continue;
362 remote_hex = oid_to_hex(remote);
363 if (!fetching) {
364 struct strbuf c = STRBUF_INIT;
365 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
366 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
367 if (no_done) strbuf_addstr(&c, " no-done");
368 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
369 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
370 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
371 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
372 if (args->no_progress) strbuf_addstr(&c, " no-progress");
373 if (args->include_tag) strbuf_addstr(&c, " include-tag");
374 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
375 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
376 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
377 if (agent_supported) strbuf_addf(&c, " agent=%s",
378 git_user_agent_sanitized());
379 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
380 strbuf_release(&c);
381 } else
382 packet_buf_write(&req_buf, "want %s\n", remote_hex);
383 fetching++;
386 if (!fetching) {
387 strbuf_release(&req_buf);
388 packet_flush(fd[1]);
389 return 1;
392 if (is_repository_shallow())
393 write_shallow_commits(&req_buf, 1, NULL);
394 if (args->depth > 0)
395 packet_buf_write(&req_buf, "deepen %d", args->depth);
396 if (args->deepen_since) {
397 timestamp_t max_age = approxidate(args->deepen_since);
398 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
400 if (args->deepen_not) {
401 int i;
402 for (i = 0; i < args->deepen_not->nr; i++) {
403 struct string_list_item *s = args->deepen_not->items + i;
404 packet_buf_write(&req_buf, "deepen-not %s", s->string);
407 packet_buf_flush(&req_buf);
408 state_len = req_buf.len;
410 if (args->deepen) {
411 char *line;
412 const char *arg;
413 struct object_id oid;
415 send_request(args, fd[1], &req_buf);
416 while ((line = packet_read_line(fd[0], NULL))) {
417 if (skip_prefix(line, "shallow ", &arg)) {
418 if (get_oid_hex(arg, &oid))
419 die(_("invalid shallow line: %s"), line);
420 register_shallow(&oid);
421 continue;
423 if (skip_prefix(line, "unshallow ", &arg)) {
424 if (get_oid_hex(arg, &oid))
425 die(_("invalid unshallow line: %s"), line);
426 if (!lookup_object(oid.hash))
427 die(_("object not found: %s"), line);
428 /* make sure that it is parsed as shallow */
429 if (!parse_object(&oid))
430 die(_("error in object: %s"), line);
431 if (unregister_shallow(&oid))
432 die(_("no shallow found: %s"), line);
433 continue;
435 die(_("expected shallow/unshallow, got %s"), line);
437 } else if (!args->stateless_rpc)
438 send_request(args, fd[1], &req_buf);
440 if (!args->stateless_rpc) {
441 /* If we aren't using the stateless-rpc interface
442 * we don't need to retain the headers.
444 strbuf_setlen(&req_buf, 0);
445 state_len = 0;
448 flushes = 0;
449 retval = -1;
450 while ((oid = get_rev())) {
451 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
452 print_verbose(args, "have %s", oid_to_hex(oid));
453 in_vain++;
454 if (flush_at <= ++count) {
455 int ack;
457 packet_buf_flush(&req_buf);
458 send_request(args, fd[1], &req_buf);
459 strbuf_setlen(&req_buf, state_len);
460 flushes++;
461 flush_at = next_flush(args, count);
464 * We keep one window "ahead" of the other side, and
465 * will wait for an ACK only on the next one
467 if (!args->stateless_rpc && count == INITIAL_FLUSH)
468 continue;
470 consume_shallow_list(args, fd[0]);
471 do {
472 ack = get_ack(fd[0], result_oid);
473 if (ack)
474 print_verbose(args, _("got %s %d %s"), "ack",
475 ack, oid_to_hex(result_oid));
476 switch (ack) {
477 case ACK:
478 flushes = 0;
479 multi_ack = 0;
480 retval = 0;
481 goto done;
482 case ACK_common:
483 case ACK_ready:
484 case ACK_continue: {
485 struct commit *commit =
486 lookup_commit(result_oid);
487 if (!commit)
488 die(_("invalid commit %s"), oid_to_hex(result_oid));
489 if (args->stateless_rpc
490 && ack == ACK_common
491 && !(commit->object.flags & COMMON)) {
492 /* We need to replay the have for this object
493 * on the next RPC request so the peer knows
494 * it is in common with us.
496 const char *hex = oid_to_hex(result_oid);
497 packet_buf_write(&req_buf, "have %s\n", hex);
498 state_len = req_buf.len;
500 * Reset in_vain because an ack
501 * for this commit has not been
502 * seen.
504 in_vain = 0;
505 } else if (!args->stateless_rpc
506 || ack != ACK_common)
507 in_vain = 0;
508 mark_common(commit, 0, 1);
509 retval = 0;
510 got_continue = 1;
511 if (ack == ACK_ready) {
512 clear_prio_queue(&rev_list);
513 got_ready = 1;
515 break;
518 } while (ack);
519 flushes--;
520 if (got_continue && MAX_IN_VAIN < in_vain) {
521 print_verbose(args, _("giving up"));
522 break; /* give up */
526 done:
527 if (!got_ready || !no_done) {
528 packet_buf_write(&req_buf, "done\n");
529 send_request(args, fd[1], &req_buf);
531 print_verbose(args, _("done"));
532 if (retval != 0) {
533 multi_ack = 0;
534 flushes++;
536 strbuf_release(&req_buf);
538 if (!got_ready || !no_done)
539 consume_shallow_list(args, fd[0]);
540 while (flushes || multi_ack) {
541 int ack = get_ack(fd[0], result_oid);
542 if (ack) {
543 print_verbose(args, _("got %s (%d) %s"), "ack",
544 ack, oid_to_hex(result_oid));
545 if (ack == ACK)
546 return 0;
547 multi_ack = 1;
548 continue;
550 flushes--;
552 /* it is no error to fetch into a completely empty repo */
553 return count ? retval : 0;
556 static struct commit_list *complete;
558 static int mark_complete(const struct object_id *oid)
560 struct object *o = parse_object(oid);
562 while (o && o->type == OBJ_TAG) {
563 struct tag *t = (struct tag *) o;
564 if (!t->tagged)
565 break; /* broken repository */
566 o->flags |= COMPLETE;
567 o = parse_object(&t->tagged->oid);
569 if (o && o->type == OBJ_COMMIT) {
570 struct commit *commit = (struct commit *)o;
571 if (!(commit->object.flags & COMPLETE)) {
572 commit->object.flags |= COMPLETE;
573 commit_list_insert(commit, &complete);
576 return 0;
579 static int mark_complete_oid(const char *refname, const struct object_id *oid,
580 int flag, void *cb_data)
582 return mark_complete(oid);
585 static void mark_recent_complete_commits(struct fetch_pack_args *args,
586 timestamp_t cutoff)
588 while (complete && cutoff <= complete->item->date) {
589 print_verbose(args, _("Marking %s as complete"),
590 oid_to_hex(&complete->item->object.oid));
591 pop_most_recent_commit(&complete, COMPLETE);
595 static void filter_refs(struct fetch_pack_args *args,
596 struct ref **refs,
597 struct ref **sought, int nr_sought)
599 struct ref *newlist = NULL;
600 struct ref **newtail = &newlist;
601 struct ref *ref, *next;
602 int i;
604 i = 0;
605 for (ref = *refs; ref; ref = next) {
606 int keep = 0;
607 next = ref->next;
609 if (starts_with(ref->name, "refs/") &&
610 check_refname_format(ref->name, 0))
611 ; /* trash */
612 else {
613 while (i < nr_sought) {
614 int cmp = strcmp(ref->name, sought[i]->name);
615 if (cmp < 0)
616 break; /* definitely do not have it */
617 else if (cmp == 0) {
618 keep = 1; /* definitely have it */
619 sought[i]->match_status = REF_MATCHED;
621 i++;
625 if (!keep && args->fetch_all &&
626 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
627 keep = 1;
629 if (keep) {
630 *newtail = ref;
631 ref->next = NULL;
632 newtail = &ref->next;
633 } else {
634 free(ref);
638 /* Append unmatched requests to the list */
639 for (i = 0; i < nr_sought; i++) {
640 struct object_id oid;
641 const char *p;
643 ref = sought[i];
644 if (ref->match_status != REF_NOT_MATCHED)
645 continue;
646 if (parse_oid_hex(ref->name, &oid, &p) ||
647 *p != '\0' ||
648 oidcmp(&oid, &ref->old_oid))
649 continue;
651 if ((allow_unadvertised_object_request &
652 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
653 ref->match_status = REF_MATCHED;
654 *newtail = copy_ref(ref);
655 newtail = &(*newtail)->next;
656 } else {
657 ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
660 *refs = newlist;
663 static void mark_alternate_complete(struct object *obj)
665 mark_complete(&obj->oid);
668 static int everything_local(struct fetch_pack_args *args,
669 struct ref **refs,
670 struct ref **sought, int nr_sought)
672 struct ref *ref;
673 int retval;
674 timestamp_t cutoff = 0;
676 save_commit_buffer = 0;
678 for (ref = *refs; ref; ref = ref->next) {
679 struct object *o;
681 if (!has_object_file(&ref->old_oid))
682 continue;
684 o = parse_object(&ref->old_oid);
685 if (!o)
686 continue;
688 /* We already have it -- which may mean that we were
689 * in sync with the other side at some time after
690 * that (it is OK if we guess wrong here).
692 if (o->type == OBJ_COMMIT) {
693 struct commit *commit = (struct commit *)o;
694 if (!cutoff || cutoff < commit->date)
695 cutoff = commit->date;
699 if (!args->deepen) {
700 for_each_ref(mark_complete_oid, NULL);
701 for_each_cached_alternate(mark_alternate_complete);
702 commit_list_sort_by_date(&complete);
703 if (cutoff)
704 mark_recent_complete_commits(args, cutoff);
708 * Mark all complete remote refs as common refs.
709 * Don't mark them common yet; the server has to be told so first.
711 for (ref = *refs; ref; ref = ref->next) {
712 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
713 NULL, 0);
715 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
716 continue;
718 if (!(o->flags & SEEN)) {
719 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
721 mark_common((struct commit *)o, 1, 1);
725 filter_refs(args, refs, sought, nr_sought);
727 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
728 const struct object_id *remote = &ref->old_oid;
729 struct object *o;
731 o = lookup_object(remote->hash);
732 if (!o || !(o->flags & COMPLETE)) {
733 retval = 0;
734 print_verbose(args, "want %s (%s)", oid_to_hex(remote),
735 ref->name);
736 continue;
738 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
739 ref->name);
741 return retval;
744 static int sideband_demux(int in, int out, void *data)
746 int *xd = data;
747 int ret;
749 ret = recv_sideband("fetch-pack", xd[0], out);
750 close(out);
751 return ret;
754 static int get_pack(struct fetch_pack_args *args,
755 int xd[2], char **pack_lockfile)
757 struct async demux;
758 int do_keep = args->keep_pack;
759 const char *cmd_name;
760 struct pack_header header;
761 int pass_header = 0;
762 struct child_process cmd = CHILD_PROCESS_INIT;
763 int ret;
765 memset(&demux, 0, sizeof(demux));
766 if (use_sideband) {
767 /* xd[] is talking with upload-pack; subprocess reads from
768 * xd[0], spits out band#2 to stderr, and feeds us band#1
769 * through demux->out.
771 demux.proc = sideband_demux;
772 demux.data = xd;
773 demux.out = -1;
774 demux.isolate_sigpipe = 1;
775 if (start_async(&demux))
776 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
778 else
779 demux.out = xd[0];
781 if (!args->keep_pack && unpack_limit) {
783 if (read_pack_header(demux.out, &header))
784 die(_("protocol error: bad pack header"));
785 pass_header = 1;
786 if (ntohl(header.hdr_entries) < unpack_limit)
787 do_keep = 0;
788 else
789 do_keep = 1;
792 if (alternate_shallow_file) {
793 argv_array_push(&cmd.args, "--shallow-file");
794 argv_array_push(&cmd.args, alternate_shallow_file);
797 if (do_keep) {
798 if (pack_lockfile)
799 cmd.out = -1;
800 cmd_name = "index-pack";
801 argv_array_push(&cmd.args, cmd_name);
802 argv_array_push(&cmd.args, "--stdin");
803 if (!args->quiet && !args->no_progress)
804 argv_array_push(&cmd.args, "-v");
805 if (args->use_thin_pack)
806 argv_array_push(&cmd.args, "--fix-thin");
807 if (args->lock_pack || unpack_limit) {
808 char hostname[HOST_NAME_MAX + 1];
809 if (xgethostname(hostname, sizeof(hostname)))
810 xsnprintf(hostname, sizeof(hostname), "localhost");
811 argv_array_pushf(&cmd.args,
812 "--keep=fetch-pack %"PRIuMAX " on %s",
813 (uintmax_t)getpid(), hostname);
815 if (args->check_self_contained_and_connected)
816 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
818 else {
819 cmd_name = "unpack-objects";
820 argv_array_push(&cmd.args, cmd_name);
821 if (args->quiet || args->no_progress)
822 argv_array_push(&cmd.args, "-q");
823 args->check_self_contained_and_connected = 0;
826 if (pass_header)
827 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
828 ntohl(header.hdr_version),
829 ntohl(header.hdr_entries));
830 if (fetch_fsck_objects >= 0
831 ? fetch_fsck_objects
832 : transfer_fsck_objects >= 0
833 ? transfer_fsck_objects
834 : 0)
835 argv_array_push(&cmd.args, "--strict");
837 cmd.in = demux.out;
838 cmd.git_cmd = 1;
839 if (start_command(&cmd))
840 die(_("fetch-pack: unable to fork off %s"), cmd_name);
841 if (do_keep && pack_lockfile) {
842 *pack_lockfile = index_pack_lockfile(cmd.out);
843 close(cmd.out);
846 if (!use_sideband)
847 /* Closed by start_command() */
848 xd[0] = -1;
850 ret = finish_command(&cmd);
851 if (!ret || (args->check_self_contained_and_connected && ret == 1))
852 args->self_contained_and_connected =
853 args->check_self_contained_and_connected &&
854 ret == 0;
855 else
856 die(_("%s failed"), cmd_name);
857 if (use_sideband && finish_async(&demux))
858 die(_("error in sideband demultiplexer"));
859 return 0;
862 static int cmp_ref_by_name(const void *a_, const void *b_)
864 const struct ref *a = *((const struct ref **)a_);
865 const struct ref *b = *((const struct ref **)b_);
866 return strcmp(a->name, b->name);
869 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
870 int fd[2],
871 const struct ref *orig_ref,
872 struct ref **sought, int nr_sought,
873 struct shallow_info *si,
874 char **pack_lockfile)
876 struct ref *ref = copy_ref_list(orig_ref);
877 struct object_id oid;
878 const char *agent_feature;
879 int agent_len;
881 sort_ref_list(&ref, ref_compare_name);
882 QSORT(sought, nr_sought, cmp_ref_by_name);
884 if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
885 die(_("Server does not support shallow clients"));
886 if (args->depth > 0 || args->deepen_since || args->deepen_not)
887 args->deepen = 1;
888 if (server_supports("multi_ack_detailed")) {
889 print_verbose(args, _("Server supports multi_ack_detailed"));
890 multi_ack = 2;
891 if (server_supports("no-done")) {
892 print_verbose(args, _("Server supports no-done"));
893 if (args->stateless_rpc)
894 no_done = 1;
897 else if (server_supports("multi_ack")) {
898 print_verbose(args, _("Server supports multi_ack"));
899 multi_ack = 1;
901 if (server_supports("side-band-64k")) {
902 print_verbose(args, _("Server supports side-band-64k"));
903 use_sideband = 2;
905 else if (server_supports("side-band")) {
906 print_verbose(args, _("Server supports side-band"));
907 use_sideband = 1;
909 if (server_supports("allow-tip-sha1-in-want")) {
910 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
911 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
913 if (server_supports("allow-reachable-sha1-in-want")) {
914 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
915 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
917 if (!server_supports("thin-pack"))
918 args->use_thin_pack = 0;
919 if (!server_supports("no-progress"))
920 args->no_progress = 0;
921 if (!server_supports("include-tag"))
922 args->include_tag = 0;
923 if (server_supports("ofs-delta"))
924 print_verbose(args, _("Server supports ofs-delta"));
925 else
926 prefer_ofs_delta = 0;
928 if ((agent_feature = server_feature_value("agent", &agent_len))) {
929 agent_supported = 1;
930 if (agent_len)
931 print_verbose(args, _("Server version is %.*s"),
932 agent_len, agent_feature);
934 if (server_supports("deepen-since"))
935 deepen_since_ok = 1;
936 else if (args->deepen_since)
937 die(_("Server does not support --shallow-since"));
938 if (server_supports("deepen-not"))
939 deepen_not_ok = 1;
940 else if (args->deepen_not)
941 die(_("Server does not support --shallow-exclude"));
942 if (!server_supports("deepen-relative") && args->deepen_relative)
943 die(_("Server does not support --deepen"));
945 if (everything_local(args, &ref, sought, nr_sought)) {
946 packet_flush(fd[1]);
947 goto all_done;
949 if (find_common(args, fd, &oid, ref) < 0)
950 if (!args->keep_pack)
951 /* When cloning, it is not unusual to have
952 * no common commit.
954 warning(_("no common commits"));
956 if (args->stateless_rpc)
957 packet_flush(fd[1]);
958 if (args->deepen)
959 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
960 NULL);
961 else if (si->nr_ours || si->nr_theirs)
962 alternate_shallow_file = setup_temporary_shallow(si->shallow);
963 else
964 alternate_shallow_file = NULL;
965 if (get_pack(args, fd, pack_lockfile))
966 die(_("git fetch-pack: fetch failed."));
968 all_done:
969 return ref;
972 static void fetch_pack_config(void)
974 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
975 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
976 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
977 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
978 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
980 git_config(git_default_config, NULL);
983 static void fetch_pack_setup(void)
985 static int did_setup;
986 if (did_setup)
987 return;
988 fetch_pack_config();
989 if (0 <= transfer_unpack_limit)
990 unpack_limit = transfer_unpack_limit;
991 else if (0 <= fetch_unpack_limit)
992 unpack_limit = fetch_unpack_limit;
993 did_setup = 1;
996 static int remove_duplicates_in_refs(struct ref **ref, int nr)
998 struct string_list names = STRING_LIST_INIT_NODUP;
999 int src, dst;
1001 for (src = dst = 0; src < nr; src++) {
1002 struct string_list_item *item;
1003 item = string_list_insert(&names, ref[src]->name);
1004 if (item->util)
1005 continue; /* already have it */
1006 item->util = ref[src];
1007 if (src != dst)
1008 ref[dst] = ref[src];
1009 dst++;
1011 for (src = dst; src < nr; src++)
1012 ref[src] = NULL;
1013 string_list_clear(&names, 0);
1014 return dst;
1017 static void update_shallow(struct fetch_pack_args *args,
1018 struct ref **sought, int nr_sought,
1019 struct shallow_info *si)
1021 struct oid_array ref = OID_ARRAY_INIT;
1022 int *status;
1023 int i;
1025 if (args->deepen && alternate_shallow_file) {
1026 if (*alternate_shallow_file == '\0') { /* --unshallow */
1027 unlink_or_warn(git_path_shallow());
1028 rollback_lock_file(&shallow_lock);
1029 } else
1030 commit_lock_file(&shallow_lock);
1031 return;
1034 if (!si->shallow || !si->shallow->nr)
1035 return;
1037 if (args->cloning) {
1039 * remote is shallow, but this is a clone, there are
1040 * no objects in repo to worry about. Accept any
1041 * shallow points that exist in the pack (iow in repo
1042 * after get_pack() and reprepare_packed_git())
1044 struct oid_array extra = OID_ARRAY_INIT;
1045 struct object_id *oid = si->shallow->oid;
1046 for (i = 0; i < si->shallow->nr; i++)
1047 if (has_object_file(&oid[i]))
1048 oid_array_append(&extra, &oid[i]);
1049 if (extra.nr) {
1050 setup_alternate_shallow(&shallow_lock,
1051 &alternate_shallow_file,
1052 &extra);
1053 commit_lock_file(&shallow_lock);
1055 oid_array_clear(&extra);
1056 return;
1059 if (!si->nr_ours && !si->nr_theirs)
1060 return;
1062 remove_nonexistent_theirs_shallow(si);
1063 if (!si->nr_ours && !si->nr_theirs)
1064 return;
1065 for (i = 0; i < nr_sought; i++)
1066 oid_array_append(&ref, &sought[i]->old_oid);
1067 si->ref = &ref;
1069 if (args->update_shallow) {
1071 * remote is also shallow, .git/shallow may be updated
1072 * so all refs can be accepted. Make sure we only add
1073 * shallow roots that are actually reachable from new
1074 * refs.
1076 struct oid_array extra = OID_ARRAY_INIT;
1077 struct object_id *oid = si->shallow->oid;
1078 assign_shallow_commits_to_refs(si, NULL, NULL);
1079 if (!si->nr_ours && !si->nr_theirs) {
1080 oid_array_clear(&ref);
1081 return;
1083 for (i = 0; i < si->nr_ours; i++)
1084 oid_array_append(&extra, &oid[si->ours[i]]);
1085 for (i = 0; i < si->nr_theirs; i++)
1086 oid_array_append(&extra, &oid[si->theirs[i]]);
1087 setup_alternate_shallow(&shallow_lock,
1088 &alternate_shallow_file,
1089 &extra);
1090 commit_lock_file(&shallow_lock);
1091 oid_array_clear(&extra);
1092 oid_array_clear(&ref);
1093 return;
1097 * remote is also shallow, check what ref is safe to update
1098 * without updating .git/shallow
1100 status = xcalloc(nr_sought, sizeof(*status));
1101 assign_shallow_commits_to_refs(si, NULL, status);
1102 if (si->nr_ours || si->nr_theirs) {
1103 for (i = 0; i < nr_sought; i++)
1104 if (status[i])
1105 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1107 free(status);
1108 oid_array_clear(&ref);
1111 struct ref *fetch_pack(struct fetch_pack_args *args,
1112 int fd[], struct child_process *conn,
1113 const struct ref *ref,
1114 const char *dest,
1115 struct ref **sought, int nr_sought,
1116 struct oid_array *shallow,
1117 char **pack_lockfile)
1119 struct ref *ref_cpy;
1120 struct shallow_info si;
1122 fetch_pack_setup();
1123 if (nr_sought)
1124 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1126 if (!ref) {
1127 packet_flush(fd[1]);
1128 die(_("no matching remote head"));
1130 prepare_shallow_info(&si, shallow);
1131 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1132 &si, pack_lockfile);
1133 reprepare_packed_git();
1134 update_shallow(args, sought, nr_sought, &si);
1135 clear_shallow_info(&si);
1136 return ref_cpy;
1139 int report_unmatched_refs(struct ref **sought, int nr_sought)
1141 int i, ret = 0;
1143 for (i = 0; i < nr_sought; i++) {
1144 if (!sought[i])
1145 continue;
1146 switch (sought[i]->match_status) {
1147 case REF_MATCHED:
1148 continue;
1149 case REF_NOT_MATCHED:
1150 error(_("no such remote ref %s"), sought[i]->name);
1151 break;
1152 case REF_UNADVERTISED_NOT_ALLOWED:
1153 error(_("Server does not allow request for unadvertised object %s"),
1154 sought[i]->name);
1155 break;
1157 ret = 1;
1159 return ret;