?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ci/lib: allow running in GitHub Actions
[git/debian.git] / fetch-pack.c
blob1734a573b010dd2f87541154cc70e5bb71b00463
1 #include "cache.h"
2 #include "repository.h"
3 #include "config.h"
4 #include "lockfile.h"
5 #include "refs.h"
6 #include "pkt-line.h"
7 #include "commit.h"
8 #include "tag.h"
9 #include "exec-cmd.h"
10 #include "pack.h"
11 #include "sideband.h"
12 #include "fetch-pack.h"
13 #include "remote.h"
14 #include "run-command.h"
15 #include "connect.h"
16 #include "transport.h"
17 #include "version.h"
18 #include "sha1-array.h"
19 #include "oidset.h"
20 #include "packfile.h"
21 #include "object-store.h"
22 #include "connected.h"
23 #include "fetch-negotiator.h"
24 #include "fsck.h"
25
26 static int transfer_unpack_limit = -1;
27 static int fetch_unpack_limit = -1;
28 static int unpack_limit = 100;
29 static int prefer_ofs_delta = 1;
30 static int no_done;
31 static int deepen_since_ok;
32 static int deepen_not_ok;
33 static int fetch_fsck_objects = -1;
34 static int transfer_fsck_objects = -1;
35 static int agent_supported;
36 static int server_supports_filtering;
37 static struct lock_file shallow_lock;
38 static const char *alternate_shallow_file;
39 static struct strbuf fsck_msg_types = STRBUF_INIT;
40
41 /* Remember to update object flag allocation in object.h */
42 #define COMPLETE (1U << 0)
43 #define ALTERNATE (1U << 1)
44
45 /*
46 * After sending this many "have"s if we do not get any new ACK , we
47 * give up traversing our history.
48 */
49 #define MAX_IN_VAIN 256
50
51 static int multi_ack, use_sideband;
52 /* Allow specifying sha1 if it is a ref tip. */
53 #define ALLOW_TIP_SHA1 01
54 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
55 #define ALLOW_REACHABLE_SHA1 02
56 static unsigned int allow_unadvertised_object_request;
57
58 __attribute__((format (printf, 2, 3)))
59 static inline void print_verbose(const struct fetch_pack_args *args,
60 const char *fmt, ...)
61 {
62 va_list params;
63
64 if (!args->verbose)
65 return;
66
67 va_start(params, fmt);
68 vfprintf(stderr, fmt, params);
69 va_end(params);
70 fputc('\n', stderr);
71 }
72
73 struct alternate_object_cache {
74 struct object **items;
75 size_t nr, alloc;
76 };
77
78 static void cache_one_alternate(const struct object_id *oid,
79 void *vcache)
80 {
81 struct alternate_object_cache *cache = vcache;
82 struct object *obj = parse_object(the_repository, oid);
83
84 if (!obj || (obj->flags & ALTERNATE))
85 return;
86
87 obj->flags |= ALTERNATE;
88 ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
89 cache->items[cache->nr++] = obj;
90 }
91
92 static void for_each_cached_alternate(struct fetch_negotiator *negotiator,
93 void (*cb)(struct fetch_negotiator *,
94 struct object *))
95 {
96 static int initialized;
97 static struct alternate_object_cache cache;
98 size_t i;
99
100 if (!initialized) {
101 for_each_alternate_ref(cache_one_alternate, &cache);
102 initialized = 1;
103 }
104
105 for (i = 0; i < cache.nr; i++)
106 cb(negotiator, cache.items[i]);
107 }
108
109 static int rev_list_insert_ref(struct fetch_negotiator *negotiator,
110 const char *refname,
111 const struct object_id *oid)
112 {
113 struct object *o = deref_tag(the_repository,
114 parse_object(the_repository, oid),
115 refname, 0);
116
117 if (o && o->type == OBJ_COMMIT)
118 negotiator->add_tip(negotiator, (struct commit *)o);
119
120 return 0;
121 }
122
123 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
124 int flag, void *cb_data)
125 {
126 return rev_list_insert_ref(cb_data, refname, oid);
127 }
128
129 enum ack_type {
130 NAK = 0,
131 ACK,
132 ACK_continue,
133 ACK_common,
134 ACK_ready
135 };
136
137 static void consume_shallow_list(struct fetch_pack_args *args,
138 struct packet_reader *reader)
139 {
140 if (args->stateless_rpc && args->deepen) {
141 /* If we sent a depth we will get back "duplicate"
142 * shallow and unshallow commands every time there
143 * is a block of have lines exchanged.
144 */
145 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
146 if (starts_with(reader->line, "shallow "))
147 continue;
148 if (starts_with(reader->line, "unshallow "))
149 continue;
150 die(_("git fetch-pack: expected shallow list"));
151 }
152 if (reader->status != PACKET_READ_FLUSH)
153 die(_("git fetch-pack: expected a flush packet after shallow list"));
154 }
155 }
156
157 static enum ack_type get_ack(struct packet_reader *reader,
158 struct object_id *result_oid)
159 {
160 int len;
161 const char *arg;
162
163 if (packet_reader_read(reader) != PACKET_READ_NORMAL)
164 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
165 len = reader->pktlen;
166
167 if (!strcmp(reader->line, "NAK"))
168 return NAK;
169 if (skip_prefix(reader->line, "ACK ", &arg)) {
170 const char *p;
171 if (!parse_oid_hex(arg, result_oid, &p)) {
172 len -= p - reader->line;
173 if (len < 1)
174 return ACK;
175 if (strstr(p, "continue"))
176 return ACK_continue;
177 if (strstr(p, "common"))
178 return ACK_common;
179 if (strstr(p, "ready"))
180 return ACK_ready;
181 return ACK;
182 }
183 }
184 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), reader->line);
185 }
186
187 static void send_request(struct fetch_pack_args *args,
188 int fd, struct strbuf *buf)
189 {
190 if (args->stateless_rpc) {
191 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
192 packet_flush(fd);
193 } else {
194 if (write_in_full(fd, buf->buf, buf->len) < 0)
195 die_errno(_("unable to write to remote"));
196 }
197 }
198
199 static void insert_one_alternate_object(struct fetch_negotiator *negotiator,
200 struct object *obj)
201 {
202 rev_list_insert_ref(negotiator, NULL, &obj->oid);
203 }
204
205 #define INITIAL_FLUSH 16
206 #define PIPESAFE_FLUSH 32
207 #define LARGE_FLUSH 16384
208
209 static int next_flush(int stateless_rpc, int count)
210 {
211 if (stateless_rpc) {
212 if (count < LARGE_FLUSH)
213 count <<= 1;
214 else
215 count = count * 11 / 10;
216 } else {
217 if (count < PIPESAFE_FLUSH)
218 count <<= 1;
219 else
220 count += PIPESAFE_FLUSH;
221 }
222 return count;
223 }
224
225 static void mark_tips(struct fetch_negotiator *negotiator,
226 const struct oid_array *negotiation_tips)
227 {
228 int i;
229
230 if (!negotiation_tips) {
231 for_each_ref(rev_list_insert_ref_oid, negotiator);
232 return;
233 }
234
235 for (i = 0; i < negotiation_tips->nr; i++)
236 rev_list_insert_ref(negotiator, NULL,
237 &negotiation_tips->oid[i]);
238 return;
239 }
240
241 static int find_common(struct fetch_negotiator *negotiator,
242 struct fetch_pack_args *args,
243 int fd[2], struct object_id *result_oid,
244 struct ref *refs)
245 {
246 int fetching;
247 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
248 const struct object_id *oid;
249 unsigned in_vain = 0;
250 int got_continue = 0;
251 int got_ready = 0;
252 struct strbuf req_buf = STRBUF_INIT;
253 size_t state_len = 0;
254 struct packet_reader reader;
255
256 if (args->stateless_rpc && multi_ack == 1)
257 die(_("--stateless-rpc requires multi_ack_detailed"));
258
259 packet_reader_init(&reader, fd[0], NULL, 0,
260 PACKET_READ_CHOMP_NEWLINE |
261 PACKET_READ_DIE_ON_ERR_PACKET);
262
263 if (!args->no_dependents) {
264 mark_tips(negotiator, args->negotiation_tips);
265 for_each_cached_alternate(negotiator, insert_one_alternate_object);
266 }
267
268 fetching = 0;
269 for ( ; refs ; refs = refs->next) {
270 struct object_id *remote = &refs->old_oid;
271 const char *remote_hex;
272 struct object *o;
273
274 /*
275 * If that object is complete (i.e. it is an ancestor of a
276 * local ref), we tell them we have it but do not have to
277 * tell them about its ancestors, which they already know
278 * about.
279 *
280 * We use lookup_object here because we are only
281 * interested in the case we *know* the object is
282 * reachable and we have already scanned it.
283 *
284 * Do this only if args->no_dependents is false (if it is true,
285 * we cannot trust the object flags).
286 */
287 if (!args->no_dependents &&
288 ((o = lookup_object(the_repository, remote)) != NULL) &&
289 (o->flags & COMPLETE)) {
290 continue;
291 }
292
293 remote_hex = oid_to_hex(remote);
294 if (!fetching) {
295 struct strbuf c = STRBUF_INIT;
296 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
297 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
298 if (no_done) strbuf_addstr(&c, " no-done");
299 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
300 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
301 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
302 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
303 if (args->no_progress) strbuf_addstr(&c, " no-progress");
304 if (args->include_tag) strbuf_addstr(&c, " include-tag");
305 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
306 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
307 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
308 if (agent_supported) strbuf_addf(&c, " agent=%s",
309 git_user_agent_sanitized());
310 if (args->filter_options.choice)
311 strbuf_addstr(&c, " filter");
312 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
313 strbuf_release(&c);
314 } else
315 packet_buf_write(&req_buf, "want %s\n", remote_hex);
316 fetching++;
317 }
318
319 if (!fetching) {
320 strbuf_release(&req_buf);
321 packet_flush(fd[1]);
322 return 1;
323 }
324
325 if (is_repository_shallow(the_repository))
326 write_shallow_commits(&req_buf, 1, NULL);
327 if (args->depth > 0)
328 packet_buf_write(&req_buf, "deepen %d", args->depth);
329 if (args->deepen_since) {
330 timestamp_t max_age = approxidate(args->deepen_since);
331 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
332 }
333 if (args->deepen_not) {
334 int i;
335 for (i = 0; i < args->deepen_not->nr; i++) {
336 struct string_list_item *s = args->deepen_not->items + i;
337 packet_buf_write(&req_buf, "deepen-not %s", s->string);
338 }
339 }
340 if (server_supports_filtering && args->filter_options.choice) {
341 const char *spec =
342 expand_list_objects_filter_spec(&args->filter_options);
343 packet_buf_write(&req_buf, "filter %s", spec);
344 }
345 packet_buf_flush(&req_buf);
346 state_len = req_buf.len;
347
348 if (args->deepen) {
349 const char *arg;
350 struct object_id oid;
351
352 send_request(args, fd[1], &req_buf);
353 while (packet_reader_read(&reader) == PACKET_READ_NORMAL) {
354 if (skip_prefix(reader.line, "shallow ", &arg)) {
355 if (get_oid_hex(arg, &oid))
356 die(_("invalid shallow line: %s"), reader.line);
357 register_shallow(the_repository, &oid);
358 continue;
359 }
360 if (skip_prefix(reader.line, "unshallow ", &arg)) {
361 if (get_oid_hex(arg, &oid))
362 die(_("invalid unshallow line: %s"), reader.line);
363 if (!lookup_object(the_repository, &oid))
364 die(_("object not found: %s"), reader.line);
365 /* make sure that it is parsed as shallow */
366 if (!parse_object(the_repository, &oid))
367 die(_("error in object: %s"), reader.line);
368 if (unregister_shallow(&oid))
369 die(_("no shallow found: %s"), reader.line);
370 continue;
371 }
372 die(_("expected shallow/unshallow, got %s"), reader.line);
373 }
374 } else if (!args->stateless_rpc)
375 send_request(args, fd[1], &req_buf);
376
377 if (!args->stateless_rpc) {
378 /* If we aren't using the stateless-rpc interface
379 * we don't need to retain the headers.
380 */
381 strbuf_setlen(&req_buf, 0);
382 state_len = 0;
383 }
384
385 trace2_region_enter("fetch-pack", "negotiation_v0_v1", the_repository);
386 flushes = 0;
387 retval = -1;
388 if (args->no_dependents)
389 goto done;
390 while ((oid = negotiator->next(negotiator))) {
391 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
392 print_verbose(args, "have %s", oid_to_hex(oid));
393 in_vain++;
394 if (flush_at <= ++count) {
395 int ack;
396
397 packet_buf_flush(&req_buf);
398 send_request(args, fd[1], &req_buf);
399 strbuf_setlen(&req_buf, state_len);
400 flushes++;
401 flush_at = next_flush(args->stateless_rpc, count);
402
403 /*
404 * We keep one window "ahead" of the other side, and
405 * will wait for an ACK only on the next one
406 */
407 if (!args->stateless_rpc && count == INITIAL_FLUSH)
408 continue;
409
410 consume_shallow_list(args, &reader);
411 do {
412 ack = get_ack(&reader, result_oid);
413 if (ack)
414 print_verbose(args, _("got %s %d %s"), "ack",
415 ack, oid_to_hex(result_oid));
416 switch (ack) {
417 case ACK:
418 flushes = 0;
419 multi_ack = 0;
420 retval = 0;
421 goto done;
422 case ACK_common:
423 case ACK_ready:
424 case ACK_continue: {
425 struct commit *commit =
426 lookup_commit(the_repository,
427 result_oid);
428 int was_common;
429
430 if (!commit)
431 die(_("invalid commit %s"), oid_to_hex(result_oid));
432 was_common = negotiator->ack(negotiator, commit);
433 if (args->stateless_rpc
434 && ack == ACK_common
435 && !was_common) {
436 /* We need to replay the have for this object
437 * on the next RPC request so the peer knows
438 * it is in common with us.
439 */
440 const char *hex = oid_to_hex(result_oid);
441 packet_buf_write(&req_buf, "have %s\n", hex);
442 state_len = req_buf.len;
443 /*
444 * Reset in_vain because an ack
445 * for this commit has not been
446 * seen.
447 */
448 in_vain = 0;
449 } else if (!args->stateless_rpc
450 || ack != ACK_common)
451 in_vain = 0;
452 retval = 0;
453 got_continue = 1;
454 if (ack == ACK_ready)
455 got_ready = 1;
456 break;
457 }
458 }
459 } while (ack);
460 flushes--;
461 if (got_continue && MAX_IN_VAIN < in_vain) {
462 print_verbose(args, _("giving up"));
463 break; /* give up */
464 }
465 if (got_ready)
466 break;
467 }
468 }
469 done:
470 trace2_region_leave("fetch-pack", "negotiation_v0_v1", the_repository);
471 if (!got_ready || !no_done) {
472 packet_buf_write(&req_buf, "done\n");
473 send_request(args, fd[1], &req_buf);
474 }
475 print_verbose(args, _("done"));
476 if (retval != 0) {
477 multi_ack = 0;
478 flushes++;
479 }
480 strbuf_release(&req_buf);
481
482 if (!got_ready || !no_done)
483 consume_shallow_list(args, &reader);
484 while (flushes || multi_ack) {
485 int ack = get_ack(&reader, result_oid);
486 if (ack) {
487 print_verbose(args, _("got %s (%d) %s"), "ack",
488 ack, oid_to_hex(result_oid));
489 if (ack == ACK)
490 return 0;
491 multi_ack = 1;
492 continue;
493 }
494 flushes--;
495 }
496 /* it is no error to fetch into a completely empty repo */
497 return count ? retval : 0;
498 }
499
500 static struct commit_list *complete;
501
502 static int mark_complete(const struct object_id *oid)
503 {
504 struct object *o = parse_object(the_repository, oid);
505
506 while (o && o->type == OBJ_TAG) {
507 struct tag *t = (struct tag *) o;
508 if (!t->tagged)
509 break; /* broken repository */
510 o->flags |= COMPLETE;
511 o = parse_object(the_repository, &t->tagged->oid);
512 }
513 if (o && o->type == OBJ_COMMIT) {
514 struct commit *commit = (struct commit *)o;
515 if (!(commit->object.flags & COMPLETE)) {
516 commit->object.flags |= COMPLETE;
517 commit_list_insert(commit, &complete);
518 }
519 }
520 return 0;
521 }
522
523 static int mark_complete_oid(const char *refname, const struct object_id *oid,
524 int flag, void *cb_data)
525 {
526 return mark_complete(oid);
527 }
528
529 static void mark_recent_complete_commits(struct fetch_pack_args *args,
530 timestamp_t cutoff)
531 {
532 while (complete && cutoff <= complete->item->date) {
533 print_verbose(args, _("Marking %s as complete"),
534 oid_to_hex(&complete->item->object.oid));
535 pop_most_recent_commit(&complete, COMPLETE);
536 }
537 }
538
539 static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
540 {
541 for (; refs; refs = refs->next)
542 oidset_insert(oids, &refs->old_oid);
543 }
544
545 static int is_unmatched_ref(const struct ref *ref)
546 {
547 struct object_id oid;
548 const char *p;
549 return ref->match_status == REF_NOT_MATCHED &&
550 !parse_oid_hex(ref->name, &oid, &p) &&
551 *p == '\0' &&
552 oideq(&oid, &ref->old_oid);
553 }
554
555 static void filter_refs(struct fetch_pack_args *args,
556 struct ref **refs,
557 struct ref **sought, int nr_sought)
558 {
559 struct ref *newlist = NULL;
560 struct ref **newtail = &newlist;
561 struct ref *unmatched = NULL;
562 struct ref *ref, *next;
563 struct oidset tip_oids = OIDSET_INIT;
564 int i;
565 int strict = !(allow_unadvertised_object_request &
566 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1));
567
568 i = 0;
569 for (ref = *refs; ref; ref = next) {
570 int keep = 0;
571 next = ref->next;
572
573 if (starts_with(ref->name, "refs/") &&
574 check_refname_format(ref->name, 0)) {
575 /*
576 * trash or a peeled value; do not even add it to
577 * unmatched list
578 */
579 free_one_ref(ref);
580 continue;
581 } else {
582 while (i < nr_sought) {
583 int cmp = strcmp(ref->name, sought[i]->name);
584 if (cmp < 0)
585 break; /* definitely do not have it */
586 else if (cmp == 0) {
587 keep = 1; /* definitely have it */
588 sought[i]->match_status = REF_MATCHED;
589 }
590 i++;
591 }
592
593 if (!keep && args->fetch_all &&
594 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
595 keep = 1;
596 }
597
598 if (keep) {
599 *newtail = ref;
600 ref->next = NULL;
601 newtail = &ref->next;
602 } else {
603 ref->next = unmatched;
604 unmatched = ref;
605 }
606 }
607
608 if (strict) {
609 for (i = 0; i < nr_sought; i++) {
610 ref = sought[i];
611 if (!is_unmatched_ref(ref))
612 continue;
613
614 add_refs_to_oidset(&tip_oids, unmatched);
615 add_refs_to_oidset(&tip_oids, newlist);
616 break;
617 }
618 }
619
620 /* Append unmatched requests to the list */
621 for (i = 0; i < nr_sought; i++) {
622 ref = sought[i];
623 if (!is_unmatched_ref(ref))
624 continue;
625
626 if (!strict || oidset_contains(&tip_oids, &ref->old_oid)) {
627 ref->match_status = REF_MATCHED;
628 *newtail = copy_ref(ref);
629 newtail = &(*newtail)->next;
630 } else {
631 ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
632 }
633 }
634
635 oidset_clear(&tip_oids);
636 free_refs(unmatched);
637
638 *refs = newlist;
639 }
640
641 static void mark_alternate_complete(struct fetch_negotiator *unused,
642 struct object *obj)
643 {
644 mark_complete(&obj->oid);
645 }
646
647 struct loose_object_iter {
648 struct oidset *loose_object_set;
649 struct ref *refs;
650 };
651
652 /*
653 * Mark recent commits available locally and reachable from a local ref as
654 * COMPLETE. If args->no_dependents is false, also mark COMPLETE remote refs as
655 * COMMON_REF (otherwise, we are not planning to participate in negotiation, and
656 * thus do not need COMMON_REF marks).
657 *
658 * The cutoff time for recency is determined by this heuristic: it is the
659 * earliest commit time of the objects in refs that are commits and that we know
660 * the commit time of.
661 */
662 static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
663 struct fetch_pack_args *args,
664 struct ref **refs)
665 {
666 struct ref *ref;
667 int old_save_commit_buffer = save_commit_buffer;
668 timestamp_t cutoff = 0;
669
670 save_commit_buffer = 0;
671
672 trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
673 for (ref = *refs; ref; ref = ref->next) {
674 struct object *o;
675
676 if (!has_object_file_with_flags(&ref->old_oid,
677 OBJECT_INFO_QUICK |
678 OBJECT_INFO_SKIP_FETCH_OBJECT))
679 continue;
680 o = parse_object(the_repository, &ref->old_oid);
681 if (!o)
682 continue;
683
684 /*
685 * We already have it -- which may mean that we were
686 * in sync with the other side at some time after
687 * that (it is OK if we guess wrong here).
688 */
689 if (o->type == OBJ_COMMIT) {
690 struct commit *commit = (struct commit *)o;
691 if (!cutoff || cutoff < commit->date)
692 cutoff = commit->date;
693 }
694 }
695 trace2_region_leave("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
696
697 /*
698 * This block marks all local refs as COMPLETE, and then recursively marks all
699 * parents of those refs as COMPLETE.
700 */
701 trace2_region_enter("fetch-pack", "mark_complete_local_refs", NULL);
702 if (!args->deepen) {
703 for_each_ref(mark_complete_oid, NULL);
704 for_each_cached_alternate(NULL, mark_alternate_complete);
705 commit_list_sort_by_date(&complete);
706 if (cutoff)
707 mark_recent_complete_commits(args, cutoff);
708 }
709 trace2_region_leave("fetch-pack", "mark_complete_local_refs", NULL);
710
711 /*
712 * Mark all complete remote refs as common refs.
713 * Don't mark them common yet; the server has to be told so first.
714 */
715 trace2_region_enter("fetch-pack", "mark_common_remote_refs", NULL);
716 for (ref = *refs; ref; ref = ref->next) {
717 struct object *o = deref_tag(the_repository,
718 lookup_object(the_repository,
719 &ref->old_oid),
720 NULL, 0);
721
722 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
723 continue;
724
725 negotiator->known_common(negotiator,
726 (struct commit *)o);
727 }
728 trace2_region_leave("fetch-pack", "mark_common_remote_refs", NULL);
729
730 save_commit_buffer = old_save_commit_buffer;
731 }
732
733 /*
734 * Returns 1 if every object pointed to by the given remote refs is available
735 * locally and reachable from a local ref, and 0 otherwise.
736 */
737 static int everything_local(struct fetch_pack_args *args,
738 struct ref **refs)
739 {
740 struct ref *ref;
741 int retval;
742
743 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
744 const struct object_id *remote = &ref->old_oid;
745 struct object *o;
746
747 o = lookup_object(the_repository, remote);
748 if (!o || !(o->flags & COMPLETE)) {
749 retval = 0;
750 print_verbose(args, "want %s (%s)", oid_to_hex(remote),
751 ref->name);
752 continue;
753 }
754 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
755 ref->name);
756 }
757
758 return retval;
759 }
760
761 static int sideband_demux(int in, int out, void *data)
762 {
763 int *xd = data;
764 int ret;
765
766 ret = recv_sideband("fetch-pack", xd[0], out);
767 close(out);
768 return ret;
769 }
770
771 static void write_promisor_file(const char *keep_name,
772 struct ref **sought, int nr_sought)
773 {
774 struct strbuf promisor_name = STRBUF_INIT;
775 int suffix_stripped;
776 FILE *output;
777 int i;
778
779 strbuf_addstr(&promisor_name, keep_name);
780 suffix_stripped = strbuf_strip_suffix(&promisor_name, ".keep");
781 if (!suffix_stripped)
782 BUG("name of pack lockfile should end with .keep (was '%s')",
783 keep_name);
784 strbuf_addstr(&promisor_name, ".promisor");
785
786 output = xfopen(promisor_name.buf, "w");
787 for (i = 0; i < nr_sought; i++)
788 fprintf(output, "%s %s\n", oid_to_hex(&sought[i]->old_oid),
789 sought[i]->name);
790 fclose(output);
791
792 strbuf_release(&promisor_name);
793 }
794
795 static int get_pack(struct fetch_pack_args *args,
796 int xd[2], char **pack_lockfile,
797 struct ref **sought, int nr_sought)
798 {
799 struct async demux;
800 int do_keep = args->keep_pack;
801 const char *cmd_name;
802 struct pack_header header;
803 int pass_header = 0;
804 struct child_process cmd = CHILD_PROCESS_INIT;
805 int ret;
806
807 memset(&demux, 0, sizeof(demux));
808 if (use_sideband) {
809 /* xd[] is talking with upload-pack; subprocess reads from
810 * xd[0], spits out band#2 to stderr, and feeds us band#1
811 * through demux->out.
812 */
813 demux.proc = sideband_demux;
814 demux.data = xd;
815 demux.out = -1;
816 demux.isolate_sigpipe = 1;
817 if (start_async(&demux))
818 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
819 }
820 else
821 demux.out = xd[0];
822
823 if (!args->keep_pack && unpack_limit) {
824
825 if (read_pack_header(demux.out, &header))
826 die(_("protocol error: bad pack header"));
827 pass_header = 1;
828 if (ntohl(header.hdr_entries) < unpack_limit)
829 do_keep = 0;
830 else
831 do_keep = 1;
832 }
833
834 if (alternate_shallow_file) {
835 argv_array_push(&cmd.args, "--shallow-file");
836 argv_array_push(&cmd.args, alternate_shallow_file);
837 }
838
839 if (do_keep || args->from_promisor) {
840 if (pack_lockfile)
841 cmd.out = -1;
842 cmd_name = "index-pack";
843 argv_array_push(&cmd.args, cmd_name);
844 argv_array_push(&cmd.args, "--stdin");
845 if (!args->quiet && !args->no_progress)
846 argv_array_push(&cmd.args, "-v");
847 if (args->use_thin_pack)
848 argv_array_push(&cmd.args, "--fix-thin");
849 if (do_keep && (args->lock_pack || unpack_limit)) {
850 char hostname[HOST_NAME_MAX + 1];
851 if (xgethostname(hostname, sizeof(hostname)))
852 xsnprintf(hostname, sizeof(hostname), "localhost");
853 argv_array_pushf(&cmd.args,
854 "--keep=fetch-pack %"PRIuMAX " on %s",
855 (uintmax_t)getpid(), hostname);
856 }
857 if (args->check_self_contained_and_connected)
858 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
859 /*
860 * If we're obtaining the filename of a lockfile, we'll use
861 * that filename to write a .promisor file with more
862 * information below. If not, we need index-pack to do it for
863 * us.
864 */
865 if (!(do_keep && pack_lockfile) && args->from_promisor)
866 argv_array_push(&cmd.args, "--promisor");
867 }
868 else {
869 cmd_name = "unpack-objects";
870 argv_array_push(&cmd.args, cmd_name);
871 if (args->quiet || args->no_progress)
872 argv_array_push(&cmd.args, "-q");
873 args->check_self_contained_and_connected = 0;
874 }
875
876 if (pass_header)
877 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
878 ntohl(header.hdr_version),
879 ntohl(header.hdr_entries));
880 if (fetch_fsck_objects >= 0
881 ? fetch_fsck_objects
882 : transfer_fsck_objects >= 0
883 ? transfer_fsck_objects
884 : 0) {
885 if (args->from_promisor)
886 /*
887 * We cannot use --strict in index-pack because it
888 * checks both broken objects and links, but we only
889 * want to check for broken objects.
890 */
891 argv_array_push(&cmd.args, "--fsck-objects");
892 else
893 argv_array_pushf(&cmd.args, "--strict%s",
894 fsck_msg_types.buf);
895 }
896
897 cmd.in = demux.out;
898 cmd.git_cmd = 1;
899 if (start_command(&cmd))
900 die(_("fetch-pack: unable to fork off %s"), cmd_name);
901 if (do_keep && pack_lockfile) {
902 *pack_lockfile = index_pack_lockfile(cmd.out);
903 close(cmd.out);
904 }
905
906 if (!use_sideband)
907 /* Closed by start_command() */
908 xd[0] = -1;
909
910 ret = finish_command(&cmd);
911 if (!ret || (args->check_self_contained_and_connected && ret == 1))
912 args->self_contained_and_connected =
913 args->check_self_contained_and_connected &&
914 ret == 0;
915 else
916 die(_("%s failed"), cmd_name);
917 if (use_sideband && finish_async(&demux))
918 die(_("error in sideband demultiplexer"));
919
920 /*
921 * Now that index-pack has succeeded, write the promisor file using the
922 * obtained .keep filename if necessary
923 */
924 if (do_keep && pack_lockfile && args->from_promisor)
925 write_promisor_file(*pack_lockfile, sought, nr_sought);
926
927 return 0;
928 }
929
930 static int cmp_ref_by_name(const void *a_, const void *b_)
931 {
932 const struct ref *a = *((const struct ref **)a_);
933 const struct ref *b = *((const struct ref **)b_);
934 return strcmp(a->name, b->name);
935 }
936
937 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
938 int fd[2],
939 const struct ref *orig_ref,
940 struct ref **sought, int nr_sought,
941 struct shallow_info *si,
942 char **pack_lockfile)
943 {
944 struct repository *r = the_repository;
945 struct ref *ref = copy_ref_list(orig_ref);
946 struct object_id oid;
947 const char *agent_feature;
948 int agent_len;
949 struct fetch_negotiator negotiator_alloc;
950 struct fetch_negotiator *negotiator;
951
952 if (args->no_dependents) {
953 negotiator = NULL;
954 } else {
955 negotiator = &negotiator_alloc;
956 fetch_negotiator_init(r, negotiator);
957 }
958
959 sort_ref_list(&ref, ref_compare_name);
960 QSORT(sought, nr_sought, cmp_ref_by_name);
961
962 if ((agent_feature = server_feature_value("agent", &agent_len))) {
963 agent_supported = 1;
964 if (agent_len)
965 print_verbose(args, _("Server version is %.*s"),
966 agent_len, agent_feature);
967 }
968
969 if (server_supports("shallow"))
970 print_verbose(args, _("Server supports %s"), "shallow");
971 else if (args->depth > 0 || is_repository_shallow(r))
972 die(_("Server does not support shallow clients"));
973 if (args->depth > 0 || args->deepen_since || args->deepen_not)
974 args->deepen = 1;
975 if (server_supports("multi_ack_detailed")) {
976 print_verbose(args, _("Server supports %s"), "multi_ack_detailed");
977 multi_ack = 2;
978 if (server_supports("no-done")) {
979 print_verbose(args, _("Server supports %s"), "no-done");
980 if (args->stateless_rpc)
981 no_done = 1;
982 }
983 }
984 else if (server_supports("multi_ack")) {
985 print_verbose(args, _("Server supports %s"), "multi_ack");
986 multi_ack = 1;
987 }
988 if (server_supports("side-band-64k")) {
989 print_verbose(args, _("Server supports %s"), "side-band-64k");
990 use_sideband = 2;
991 }
992 else if (server_supports("side-band")) {
993 print_verbose(args, _("Server supports %s"), "side-band");
994 use_sideband = 1;
995 }
996 if (server_supports("allow-tip-sha1-in-want")) {
997 print_verbose(args, _("Server supports %s"), "allow-tip-sha1-in-want");
998 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
999 }
1000 if (server_supports("allow-reachable-sha1-in-want")) {
1001 print_verbose(args, _("Server supports %s"), "allow-reachable-sha1-in-want");
1002 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1003 }
1004 if (server_supports("thin-pack"))
1005 print_verbose(args, _("Server supports %s"), "thin-pack");
1006 else
1007 args->use_thin_pack = 0;
1008 if (server_supports("no-progress"))
1009 print_verbose(args, _("Server supports %s"), "no-progress");
1010 else
1011 args->no_progress = 0;
1012 if (server_supports("include-tag"))
1013 print_verbose(args, _("Server supports %s"), "include-tag");
1014 else
1015 args->include_tag = 0;
1016 if (server_supports("ofs-delta"))
1017 print_verbose(args, _("Server supports %s"), "ofs-delta");
1018 else
1019 prefer_ofs_delta = 0;
1020
1021 if (server_supports("filter")) {
1022 server_supports_filtering = 1;
1023 print_verbose(args, _("Server supports %s"), "filter");
1024 } else if (args->filter_options.choice) {
1025 warning("filtering not recognized by server, ignoring");
1026 }
1027
1028 if (server_supports("deepen-since")) {
1029 print_verbose(args, _("Server supports %s"), "deepen-since");
1030 deepen_since_ok = 1;
1031 } else if (args->deepen_since)
1032 die(_("Server does not support --shallow-since"));
1033 if (server_supports("deepen-not")) {
1034 print_verbose(args, _("Server supports %s"), "deepen-not");
1035 deepen_not_ok = 1;
1036 } else if (args->deepen_not)
1037 die(_("Server does not support --shallow-exclude"));
1038 if (server_supports("deepen-relative"))
1039 print_verbose(args, _("Server supports %s"), "deepen-relative");
1040 else if (args->deepen_relative)
1041 die(_("Server does not support --deepen"));
1042
1043 if (!args->no_dependents) {
1044 mark_complete_and_common_ref(negotiator, args, &ref);
1045 filter_refs(args, &ref, sought, nr_sought);
1046 if (everything_local(args, &ref)) {
1047 packet_flush(fd[1]);
1048 goto all_done;
1049 }
1050 } else {
1051 filter_refs(args, &ref, sought, nr_sought);
1052 }
1053 if (find_common(negotiator, args, fd, &oid, ref) < 0)
1054 if (!args->keep_pack)
1055 /* When cloning, it is not unusual to have
1056 * no common commit.
1057 */
1058 warning(_("no common commits"));
1059
1060 if (args->stateless_rpc)
1061 packet_flush(fd[1]);
1062 if (args->deepen)
1063 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1064 NULL);
1065 else if (si->nr_ours || si->nr_theirs)
1066 alternate_shallow_file = setup_temporary_shallow(si->shallow);
1067 else
1068 alternate_shallow_file = NULL;
1069 if (get_pack(args, fd, pack_lockfile, sought, nr_sought))
1070 die(_("git fetch-pack: fetch failed."));
1071
1072 all_done:
1073 if (negotiator)
1074 negotiator->release(negotiator);
1075 return ref;
1076 }
1077
1078 static void add_shallow_requests(struct strbuf *req_buf,
1079 const struct fetch_pack_args *args)
1080 {
1081 if (is_repository_shallow(the_repository))
1082 write_shallow_commits(req_buf, 1, NULL);
1083 if (args->depth > 0)
1084 packet_buf_write(req_buf, "deepen %d", args->depth);
1085 if (args->deepen_since) {
1086 timestamp_t max_age = approxidate(args->deepen_since);
1087 packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1088 }
1089 if (args->deepen_not) {
1090 int i;
1091 for (i = 0; i < args->deepen_not->nr; i++) {
1092 struct string_list_item *s = args->deepen_not->items + i;
1093 packet_buf_write(req_buf, "deepen-not %s", s->string);
1094 }
1095 }
1096 if (args->deepen_relative)
1097 packet_buf_write(req_buf, "deepen-relative\n");
1098 }
1099
1100 static void add_wants(int no_dependents, const struct ref *wants, struct strbuf *req_buf)
1101 {
1102 int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0);
1103
1104 for ( ; wants ; wants = wants->next) {
1105 const struct object_id *remote = &wants->old_oid;
1106 struct object *o;
1107
1108 /*
1109 * If that object is complete (i.e. it is an ancestor of a
1110 * local ref), we tell them we have it but do not have to
1111 * tell them about its ancestors, which they already know
1112 * about.
1113 *
1114 * We use lookup_object here because we are only
1115 * interested in the case we *know* the object is
1116 * reachable and we have already scanned it.
1117 *
1118 * Do this only if args->no_dependents is false (if it is true,
1119 * we cannot trust the object flags).
1120 */
1121 if (!no_dependents &&
1122 ((o = lookup_object(the_repository, remote)) != NULL) &&
1123 (o->flags & COMPLETE)) {
1124 continue;
1125 }
1126
1127 if (!use_ref_in_want || wants->exact_oid)
1128 packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote));
1129 else
1130 packet_buf_write(req_buf, "want-ref %s\n", wants->name);
1131 }
1132 }
1133
1134 static void add_common(struct strbuf *req_buf, struct oidset *common)
1135 {
1136 struct oidset_iter iter;
1137 const struct object_id *oid;
1138 oidset_iter_init(common, &iter);
1139
1140 while ((oid = oidset_iter_next(&iter))) {
1141 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1142 }
1143 }
1144
1145 static int add_haves(struct fetch_negotiator *negotiator,
1146 struct strbuf *req_buf,
1147 int *haves_to_send, int *in_vain)
1148 {
1149 int ret = 0;
1150 int haves_added = 0;
1151 const struct object_id *oid;
1152
1153 while ((oid = negotiator->next(negotiator))) {
1154 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1155 if (++haves_added >= *haves_to_send)
1156 break;
1157 }
1158
1159 *in_vain += haves_added;
1160 if (!haves_added || *in_vain >= MAX_IN_VAIN) {
1161 /* Send Done */
1162 packet_buf_write(req_buf, "done\n");
1163 ret = 1;
1164 }
1165
1166 /* Increase haves to send on next round */
1167 *haves_to_send = next_flush(1, *haves_to_send);
1168
1169 return ret;
1170 }
1171
1172 static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
1173 struct fetch_pack_args *args,
1174 const struct ref *wants, struct oidset *common,
1175 int *haves_to_send, int *in_vain,
1176 int sideband_all)
1177 {
1178 int ret = 0;
1179 struct strbuf req_buf = STRBUF_INIT;
1180
1181 if (server_supports_v2("fetch", 1))
1182 packet_buf_write(&req_buf, "command=fetch");
1183 if (server_supports_v2("agent", 0))
1184 packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
1185 if (args->server_options && args->server_options->nr &&
1186 server_supports_v2("server-option", 1)) {
1187 int i;
1188 for (i = 0; i < args->server_options->nr; i++)
1189 packet_buf_write(&req_buf, "server-option=%s",
1190 args->server_options->items[i].string);
1191 }
1192
1193 packet_buf_delim(&req_buf);
1194 if (args->use_thin_pack)
1195 packet_buf_write(&req_buf, "thin-pack");
1196 if (args->no_progress)
1197 packet_buf_write(&req_buf, "no-progress");
1198 if (args->include_tag)
1199 packet_buf_write(&req_buf, "include-tag");
1200 if (prefer_ofs_delta)
1201 packet_buf_write(&req_buf, "ofs-delta");
1202 if (sideband_all)
1203 packet_buf_write(&req_buf, "sideband-all");
1204
1205 /* Add shallow-info and deepen request */
1206 if (server_supports_feature("fetch", "shallow", 0))
1207 add_shallow_requests(&req_buf, args);
1208 else if (is_repository_shallow(the_repository) || args->deepen)
1209 die(_("Server does not support shallow requests"));
1210
1211 /* Add filter */
1212 if (server_supports_feature("fetch", "filter", 0) &&
1213 args->filter_options.choice) {
1214 const char *spec =
1215 expand_list_objects_filter_spec(&args->filter_options);
1216 print_verbose(args, _("Server supports filter"));
1217 packet_buf_write(&req_buf, "filter %s", spec);
1218 } else if (args->filter_options.choice) {
1219 warning("filtering not recognized by server, ignoring");
1220 }
1221
1222 /* add wants */
1223 add_wants(args->no_dependents, wants, &req_buf);
1224
1225 if (args->no_dependents) {
1226 packet_buf_write(&req_buf, "done");
1227 ret = 1;
1228 } else {
1229 /* Add all of the common commits we've found in previous rounds */
1230 add_common(&req_buf, common);
1231
1232 /* Add initial haves */
1233 ret = add_haves(negotiator, &req_buf, haves_to_send, in_vain);
1234 }
1235
1236 /* Send request */
1237 packet_buf_flush(&req_buf);
1238 if (write_in_full(fd_out, req_buf.buf, req_buf.len) < 0)
1239 die_errno(_("unable to write request to remote"));
1240
1241 strbuf_release(&req_buf);
1242 return ret;
1243 }
1244
1245 /*
1246 * Processes a section header in a server's response and checks if it matches
1247 * `section`. If the value of `peek` is 1, the header line will be peeked (and
1248 * not consumed); if 0, the line will be consumed and the function will die if
1249 * the section header doesn't match what was expected.
1250 */
1251 static int process_section_header(struct packet_reader *reader,
1252 const char *section, int peek)
1253 {
1254 int ret;
1255
1256 if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
1257 die(_("error reading section header '%s'"), section);
1258
1259 ret = !strcmp(reader->line, section);
1260
1261 if (!peek) {
1262 if (!ret)
1263 die(_("expected '%s', received '%s'"),
1264 section, reader->line);
1265 packet_reader_read(reader);
1266 }
1267
1268 return ret;
1269 }
1270
1271 static int process_acks(struct fetch_negotiator *negotiator,
1272 struct packet_reader *reader,
1273 struct oidset *common)
1274 {
1275 /* received */
1276 int received_ready = 0;
1277 int received_ack = 0;
1278
1279 process_section_header(reader, "acknowledgments", 0);
1280 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1281 const char *arg;
1282
1283 if (!strcmp(reader->line, "NAK"))
1284 continue;
1285
1286 if (skip_prefix(reader->line, "ACK ", &arg)) {
1287 struct object_id oid;
1288 if (!get_oid_hex(arg, &oid)) {
1289 struct commit *commit;
1290 oidset_insert(common, &oid);
1291 commit = lookup_commit(the_repository, &oid);
1292 if (negotiator)
1293 negotiator->ack(negotiator, commit);
1294 }
1295 continue;
1296 }
1297
1298 if (!strcmp(reader->line, "ready")) {
1299 received_ready = 1;
1300 continue;
1301 }
1302
1303 die(_("unexpected acknowledgment line: '%s'"), reader->line);
1304 }
1305
1306 if (reader->status != PACKET_READ_FLUSH &&
1307 reader->status != PACKET_READ_DELIM)
1308 die(_("error processing acks: %d"), reader->status);
1309
1310 /*
1311 * If an "acknowledgments" section is sent, a packfile is sent if and
1312 * only if "ready" was sent in this section. The other sections
1313 * ("shallow-info" and "wanted-refs") are sent only if a packfile is
1314 * sent. Therefore, a DELIM is expected if "ready" is sent, and a FLUSH
1315 * otherwise.
1316 */
1317 if (received_ready && reader->status != PACKET_READ_DELIM)
1318 die(_("expected packfile to be sent after 'ready'"));
1319 if (!received_ready && reader->status != PACKET_READ_FLUSH)
1320 die(_("expected no other sections to be sent after no 'ready'"));
1321
1322 /* return 0 if no common, 1 if there are common, or 2 if ready */
1323 return received_ready ? 2 : (received_ack ? 1 : 0);
1324 }
1325
1326 static void receive_shallow_info(struct fetch_pack_args *args,
1327 struct packet_reader *reader,
1328 struct oid_array *shallows,
1329 struct shallow_info *si)
1330 {
1331 int unshallow_received = 0;
1332
1333 process_section_header(reader, "shallow-info", 0);
1334 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1335 const char *arg;
1336 struct object_id oid;
1337
1338 if (skip_prefix(reader->line, "shallow ", &arg)) {
1339 if (get_oid_hex(arg, &oid))
1340 die(_("invalid shallow line: %s"), reader->line);
1341 oid_array_append(shallows, &oid);
1342 continue;
1343 }
1344 if (skip_prefix(reader->line, "unshallow ", &arg)) {
1345 if (get_oid_hex(arg, &oid))
1346 die(_("invalid unshallow line: %s"), reader->line);
1347 if (!lookup_object(the_repository, &oid))
1348 die(_("object not found: %s"), reader->line);
1349 /* make sure that it is parsed as shallow */
1350 if (!parse_object(the_repository, &oid))
1351 die(_("error in object: %s"), reader->line);
1352 if (unregister_shallow(&oid))
1353 die(_("no shallow found: %s"), reader->line);
1354 unshallow_received = 1;
1355 continue;
1356 }
1357 die(_("expected shallow/unshallow, got %s"), reader->line);
1358 }
1359
1360 if (reader->status != PACKET_READ_FLUSH &&
1361 reader->status != PACKET_READ_DELIM)
1362 die(_("error processing shallow info: %d"), reader->status);
1363
1364 if (args->deepen || unshallow_received) {
1365 /*
1366 * Treat these as shallow lines caused by our depth settings.
1367 * In v0, these lines cannot cause refs to be rejected; do the
1368 * same.
1369 */
1370 int i;
1371
1372 for (i = 0; i < shallows->nr; i++)
1373 register_shallow(the_repository, &shallows->oid[i]);
1374 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1375 NULL);
1376 args->deepen = 1;
1377 } else if (shallows->nr) {
1378 /*
1379 * Treat these as shallow lines caused by the remote being
1380 * shallow. In v0, remote refs that reach these objects are
1381 * rejected (unless --update-shallow is set); do the same.
1382 */
1383 prepare_shallow_info(si, shallows);
1384 if (si->nr_ours || si->nr_theirs)
1385 alternate_shallow_file =
1386 setup_temporary_shallow(si->shallow);
1387 else
1388 alternate_shallow_file = NULL;
1389 } else {
1390 alternate_shallow_file = NULL;
1391 }
1392 }
1393
1394 static int cmp_name_ref(const void *name, const void *ref)
1395 {
1396 return strcmp(name, (*(struct ref **)ref)->name);
1397 }
1398
1399 static void receive_wanted_refs(struct packet_reader *reader,
1400 struct ref **sought, int nr_sought)
1401 {
1402 process_section_header(reader, "wanted-refs", 0);
1403 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1404 struct object_id oid;
1405 const char *end;
1406 struct ref **found;
1407
1408 if (parse_oid_hex(reader->line, &oid, &end) || *end++ != ' ')
1409 die(_("expected wanted-ref, got '%s'"), reader->line);
1410
1411 found = bsearch(end, sought, nr_sought, sizeof(*sought),
1412 cmp_name_ref);
1413 if (!found)
1414 die(_("unexpected wanted-ref: '%s'"), reader->line);
1415 oidcpy(&(*found)->old_oid, &oid);
1416 }
1417
1418 if (reader->status != PACKET_READ_DELIM)
1419 die(_("error processing wanted refs: %d"), reader->status);
1420 }
1421
1422 enum fetch_state {
1423 FETCH_CHECK_LOCAL = 0,
1424 FETCH_SEND_REQUEST,
1425 FETCH_PROCESS_ACKS,
1426 FETCH_GET_PACK,
1427 FETCH_DONE,
1428 };
1429
1430 static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1431 int fd[2],
1432 const struct ref *orig_ref,
1433 struct ref **sought, int nr_sought,
1434 struct oid_array *shallows,
1435 struct shallow_info *si,
1436 char **pack_lockfile)
1437 {
1438 struct repository *r = the_repository;
1439 struct ref *ref = copy_ref_list(orig_ref);
1440 enum fetch_state state = FETCH_CHECK_LOCAL;
1441 struct oidset common = OIDSET_INIT;
1442 struct packet_reader reader;
1443 int in_vain = 0, negotiation_started = 0;
1444 int haves_to_send = INITIAL_FLUSH;
1445 struct fetch_negotiator negotiator_alloc;
1446 struct fetch_negotiator *negotiator;
1447
1448 if (args->no_dependents) {
1449 negotiator = NULL;
1450 } else {
1451 negotiator = &negotiator_alloc;
1452 fetch_negotiator_init(r, negotiator);
1453 }
1454
1455 packet_reader_init(&reader, fd[0], NULL, 0,
1456 PACKET_READ_CHOMP_NEWLINE |
1457 PACKET_READ_DIE_ON_ERR_PACKET);
1458 if (git_env_bool("GIT_TEST_SIDEBAND_ALL", 1) &&
1459 server_supports_feature("fetch", "sideband-all", 0)) {
1460 reader.use_sideband = 1;
1461 reader.me = "fetch-pack";
1462 }
1463
1464 while (state != FETCH_DONE) {
1465 switch (state) {
1466 case FETCH_CHECK_LOCAL:
1467 sort_ref_list(&ref, ref_compare_name);
1468 QSORT(sought, nr_sought, cmp_ref_by_name);
1469
1470 /* v2 supports these by default */
1471 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1472 use_sideband = 2;
1473 if (args->depth > 0 || args->deepen_since || args->deepen_not)
1474 args->deepen = 1;
1475
1476 /* Filter 'ref' by 'sought' and those that aren't local */
1477 if (!args->no_dependents) {
1478 mark_complete_and_common_ref(negotiator, args, &ref);
1479 filter_refs(args, &ref, sought, nr_sought);
1480 if (everything_local(args, &ref))
1481 state = FETCH_DONE;
1482 else
1483 state = FETCH_SEND_REQUEST;
1484
1485 mark_tips(negotiator, args->negotiation_tips);
1486 for_each_cached_alternate(negotiator,
1487 insert_one_alternate_object);
1488 } else {
1489 filter_refs(args, &ref, sought, nr_sought);
1490 state = FETCH_SEND_REQUEST;
1491 }
1492 break;
1493 case FETCH_SEND_REQUEST:
1494 if (!negotiation_started) {
1495 negotiation_started = 1;
1496 trace2_region_enter("fetch-pack",
1497 "negotiation_v2",
1498 the_repository);
1499 }
1500 if (send_fetch_request(negotiator, fd[1], args, ref,
1501 &common,
1502 &haves_to_send, &in_vain,
1503 reader.use_sideband))
1504 state = FETCH_GET_PACK;
1505 else
1506 state = FETCH_PROCESS_ACKS;
1507 break;
1508 case FETCH_PROCESS_ACKS:
1509 /* Process ACKs/NAKs */
1510 switch (process_acks(negotiator, &reader, &common)) {
1511 case 2:
1512 state = FETCH_GET_PACK;
1513 break;
1514 case 1:
1515 in_vain = 0;
1516 /* fallthrough */
1517 default:
1518 state = FETCH_SEND_REQUEST;
1519 break;
1520 }
1521 break;
1522 case FETCH_GET_PACK:
1523 trace2_region_leave("fetch-pack",
1524 "negotiation_v2",
1525 the_repository);
1526 /* Check for shallow-info section */
1527 if (process_section_header(&reader, "shallow-info", 1))
1528 receive_shallow_info(args, &reader, shallows, si);
1529
1530 if (process_section_header(&reader, "wanted-refs", 1))
1531 receive_wanted_refs(&reader, sought, nr_sought);
1532
1533 /* get the pack */
1534 process_section_header(&reader, "packfile", 0);
1535 if (get_pack(args, fd, pack_lockfile, sought, nr_sought))
1536 die(_("git fetch-pack: fetch failed."));
1537
1538 state = FETCH_DONE;
1539 break;
1540 case FETCH_DONE:
1541 continue;
1542 }
1543 }
1544
1545 if (negotiator)
1546 negotiator->release(negotiator);
1547 oidset_clear(&common);
1548 return ref;
1549 }
1550
1551 static int fetch_pack_config_cb(const char *var, const char *value, void *cb)
1552 {
1553 if (strcmp(var, "fetch.fsck.skiplist") == 0) {
1554 const char *path;
1555
1556 if (git_config_pathname(&path, var, value))
1557 return 1;
1558 strbuf_addf(&fsck_msg_types, "%cskiplist=%s",
1559 fsck_msg_types.len ? ',' : '=', path);
1560 free((char *)path);
1561 return 0;
1562 }
1563
1564 if (skip_prefix(var, "fetch.fsck.", &var)) {
1565 if (is_valid_msg_type(var, value))
1566 strbuf_addf(&fsck_msg_types, "%c%s=%s",
1567 fsck_msg_types.len ? ',' : '=', var, value);
1568 else
1569 warning("Skipping unknown msg id '%s'", var);
1570 return 0;
1571 }
1572
1573 return git_default_config(var, value, cb);
1574 }
1575
1576 static void fetch_pack_config(void)
1577 {
1578 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1579 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1580 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1581 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1582 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1583
1584 git_config(fetch_pack_config_cb, NULL);
1585 }
1586
1587 static void fetch_pack_setup(void)
1588 {
1589 static int did_setup;
1590 if (did_setup)
1591 return;
1592 fetch_pack_config();
1593 if (0 <= transfer_unpack_limit)
1594 unpack_limit = transfer_unpack_limit;
1595 else if (0 <= fetch_unpack_limit)
1596 unpack_limit = fetch_unpack_limit;
1597 did_setup = 1;
1598 }
1599
1600 static int remove_duplicates_in_refs(struct ref **ref, int nr)
1601 {
1602 struct string_list names = STRING_LIST_INIT_NODUP;
1603 int src, dst;
1604
1605 for (src = dst = 0; src < nr; src++) {
1606 struct string_list_item *item;
1607 item = string_list_insert(&names, ref[src]->name);
1608 if (item->util)
1609 continue; /* already have it */
1610 item->util = ref[src];
1611 if (src != dst)
1612 ref[dst] = ref[src];
1613 dst++;
1614 }
1615 for (src = dst; src < nr; src++)
1616 ref[src] = NULL;
1617 string_list_clear(&names, 0);
1618 return dst;
1619 }
1620
1621 static void update_shallow(struct fetch_pack_args *args,
1622 struct ref **sought, int nr_sought,
1623 struct shallow_info *si)
1624 {
1625 struct oid_array ref = OID_ARRAY_INIT;
1626 int *status;
1627 int i;
1628
1629 if (args->deepen && alternate_shallow_file) {
1630 if (*alternate_shallow_file == '\0') { /* --unshallow */
1631 unlink_or_warn(git_path_shallow(the_repository));
1632 rollback_lock_file(&shallow_lock);
1633 } else
1634 commit_lock_file(&shallow_lock);
1635 alternate_shallow_file = NULL;
1636 return;
1637 }
1638
1639 if (!si->shallow || !si->shallow->nr)
1640 return;
1641
1642 if (args->cloning) {
1643 /*
1644 * remote is shallow, but this is a clone, there are
1645 * no objects in repo to worry about. Accept any
1646 * shallow points that exist in the pack (iow in repo
1647 * after get_pack() and reprepare_packed_git())
1648 */
1649 struct oid_array extra = OID_ARRAY_INIT;
1650 struct object_id *oid = si->shallow->oid;
1651 for (i = 0; i < si->shallow->nr; i++)
1652 if (has_object_file(&oid[i]))
1653 oid_array_append(&extra, &oid[i]);
1654 if (extra.nr) {
1655 setup_alternate_shallow(&shallow_lock,
1656 &alternate_shallow_file,
1657 &extra);
1658 commit_lock_file(&shallow_lock);
1659 alternate_shallow_file = NULL;
1660 }
1661 oid_array_clear(&extra);
1662 return;
1663 }
1664
1665 if (!si->nr_ours && !si->nr_theirs)
1666 return;
1667
1668 remove_nonexistent_theirs_shallow(si);
1669 if (!si->nr_ours && !si->nr_theirs)
1670 return;
1671 for (i = 0; i < nr_sought; i++)
1672 oid_array_append(&ref, &sought[i]->old_oid);
1673 si->ref = &ref;
1674
1675 if (args->update_shallow) {
1676 /*
1677 * remote is also shallow, .git/shallow may be updated
1678 * so all refs can be accepted. Make sure we only add
1679 * shallow roots that are actually reachable from new
1680 * refs.
1681 */
1682 struct oid_array extra = OID_ARRAY_INIT;
1683 struct object_id *oid = si->shallow->oid;
1684 assign_shallow_commits_to_refs(si, NULL, NULL);
1685 if (!si->nr_ours && !si->nr_theirs) {
1686 oid_array_clear(&ref);
1687 return;
1688 }
1689 for (i = 0; i < si->nr_ours; i++)
1690 oid_array_append(&extra, &oid[si->ours[i]]);
1691 for (i = 0; i < si->nr_theirs; i++)
1692 oid_array_append(&extra, &oid[si->theirs[i]]);
1693 setup_alternate_shallow(&shallow_lock,
1694 &alternate_shallow_file,
1695 &extra);
1696 commit_lock_file(&shallow_lock);
1697 oid_array_clear(&extra);
1698 oid_array_clear(&ref);
1699 alternate_shallow_file = NULL;
1700 return;
1701 }
1702
1703 /*
1704 * remote is also shallow, check what ref is safe to update
1705 * without updating .git/shallow
1706 */
1707 status = xcalloc(nr_sought, sizeof(*status));
1708 assign_shallow_commits_to_refs(si, NULL, status);
1709 if (si->nr_ours || si->nr_theirs) {
1710 for (i = 0; i < nr_sought; i++)
1711 if (status[i])
1712 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1713 }
1714 free(status);
1715 oid_array_clear(&ref);
1716 }
1717
1718 static int iterate_ref_map(void *cb_data, struct object_id *oid)
1719 {
1720 struct ref **rm = cb_data;
1721 struct ref *ref = *rm;
1722
1723 if (!ref)
1724 return -1; /* end of the list */
1725 *rm = ref->next;
1726 oidcpy(oid, &ref->old_oid);
1727 return 0;
1728 }
1729
1730 struct ref *fetch_pack(struct fetch_pack_args *args,
1731 int fd[],
1732 const struct ref *ref,
1733 struct ref **sought, int nr_sought,
1734 struct oid_array *shallow,
1735 char **pack_lockfile,
1736 enum protocol_version version)
1737 {
1738 struct ref *ref_cpy;
1739 struct shallow_info si;
1740 struct oid_array shallows_scratch = OID_ARRAY_INIT;
1741
1742 fetch_pack_setup();
1743 if (nr_sought)
1744 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1745
1746 if (args->no_dependents && !args->filter_options.choice) {
1747 /*
1748 * The protocol does not support requesting that only the
1749 * wanted objects be sent, so approximate this by setting a
1750 * "blob:none" filter if no filter is already set. This works
1751 * for all object types: note that wanted blobs will still be
1752 * sent because they are directly specified as a "want".
1753 *
1754 * NEEDSWORK: Add an option in the protocol to request that
1755 * only the wanted objects be sent, and implement it.
1756 */
1757 parse_list_objects_filter(&args->filter_options, "blob:none");
1758 }
1759
1760 if (version != protocol_v2 && !ref) {
1761 packet_flush(fd[1]);
1762 die(_("no matching remote head"));
1763 }
1764 if (version == protocol_v2) {
1765 if (shallow->nr)
1766 BUG("Protocol V2 does not provide shallows at this point in the fetch");
1767 memset(&si, 0, sizeof(si));
1768 ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1769 &shallows_scratch, &si,
1770 pack_lockfile);
1771 } else {
1772 prepare_shallow_info(&si, shallow);
1773 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1774 &si, pack_lockfile);
1775 }
1776 reprepare_packed_git(the_repository);
1777
1778 if (!args->cloning && args->deepen) {
1779 struct check_connected_options opt = CHECK_CONNECTED_INIT;
1780 struct ref *iterator = ref_cpy;
1781 opt.shallow_file = alternate_shallow_file;
1782 if (args->deepen)
1783 opt.is_deepening_fetch = 1;
1784 if (check_connected(iterate_ref_map, &iterator, &opt)) {
1785 error(_("remote did not send all necessary objects"));
1786 free_refs(ref_cpy);
1787 ref_cpy = NULL;
1788 rollback_lock_file(&shallow_lock);
1789 goto cleanup;
1790 }
1791 args->connectivity_checked = 1;
1792 }
1793
1794 update_shallow(args, sought, nr_sought, &si);
1795 cleanup:
1796 clear_shallow_info(&si);
1797 oid_array_clear(&shallows_scratch);
1798 return ref_cpy;
1799 }
1800
1801 int report_unmatched_refs(struct ref **sought, int nr_sought)
1802 {
1803 int i, ret = 0;
1804
1805 for (i = 0; i < nr_sought; i++) {
1806 if (!sought[i])
1807 continue;
1808 switch (sought[i]->match_status) {
1809 case REF_MATCHED:
1810 continue;
1811 case REF_NOT_MATCHED:
1812 error(_("no such remote ref %s"), sought[i]->name);
1813 break;
1814 case REF_UNADVERTISED_NOT_ALLOWED:
1815 error(_("Server does not allow request for unadvertised object %s"),
1816 sought[i]->name);
1817 break;
1818 }
1819 ret = 1;
1820 }
1821 return ret;
1822 }
git for Debian