?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
t9834: remove use of `test_might_fail p4`
[git/debian.git] / fetch-pack.c
blob80fb3bd89987671c1c696c72b84cb25e946882b9
1 #include "cache.h"
2 #include "repository.h"
3 #include "config.h"
4 #include "lockfile.h"
5 #include "refs.h"
6 #include "pkt-line.h"
7 #include "commit.h"
8 #include "tag.h"
9 #include "exec-cmd.h"
10 #include "pack.h"
11 #include "sideband.h"
12 #include "fetch-pack.h"
13 #include "remote.h"
14 #include "run-command.h"
15 #include "connect.h"
16 #include "transport.h"
17 #include "version.h"
18 #include "oid-array.h"
19 #include "oidset.h"
20 #include "packfile.h"
21 #include "object-store.h"
22 #include "connected.h"
23 #include "fetch-negotiator.h"
24 #include "fsck.h"
25 #include "shallow.h"
26
27 static int transfer_unpack_limit = -1;
28 static int fetch_unpack_limit = -1;
29 static int unpack_limit = 100;
30 static int prefer_ofs_delta = 1;
31 static int no_done;
32 static int deepen_since_ok;
33 static int deepen_not_ok;
34 static int fetch_fsck_objects = -1;
35 static int transfer_fsck_objects = -1;
36 static int agent_supported;
37 static int server_supports_filtering;
38 static struct shallow_lock shallow_lock;
39 static const char *alternate_shallow_file;
40 static struct strbuf fsck_msg_types = STRBUF_INIT;
41 static struct string_list uri_protocols = STRING_LIST_INIT_DUP;
42
43 /* Remember to update object flag allocation in object.h */
44 #define COMPLETE (1U << 0)
45 #define ALTERNATE (1U << 1)
46
47 /*
48 * After sending this many "have"s if we do not get any new ACK , we
49 * give up traversing our history.
50 */
51 #define MAX_IN_VAIN 256
52
53 static int multi_ack, use_sideband;
54 /* Allow specifying sha1 if it is a ref tip. */
55 #define ALLOW_TIP_SHA1 01
56 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
57 #define ALLOW_REACHABLE_SHA1 02
58 static unsigned int allow_unadvertised_object_request;
59
60 __attribute__((format (printf, 2, 3)))
61 static inline void print_verbose(const struct fetch_pack_args *args,
62 const char *fmt, ...)
63 {
64 va_list params;
65
66 if (!args->verbose)
67 return;
68
69 va_start(params, fmt);
70 vfprintf(stderr, fmt, params);
71 va_end(params);
72 fputc('\n', stderr);
73 }
74
75 struct alternate_object_cache {
76 struct object **items;
77 size_t nr, alloc;
78 };
79
80 static void cache_one_alternate(const struct object_id *oid,
81 void *vcache)
82 {
83 struct alternate_object_cache *cache = vcache;
84 struct object *obj = parse_object(the_repository, oid);
85
86 if (!obj || (obj->flags & ALTERNATE))
87 return;
88
89 obj->flags |= ALTERNATE;
90 ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
91 cache->items[cache->nr++] = obj;
92 }
93
94 static void for_each_cached_alternate(struct fetch_negotiator *negotiator,
95 void (*cb)(struct fetch_negotiator *,
96 struct object *))
97 {
98 static int initialized;
99 static struct alternate_object_cache cache;
100 size_t i;
101
102 if (!initialized) {
103 for_each_alternate_ref(cache_one_alternate, &cache);
104 initialized = 1;
105 }
106
107 for (i = 0; i < cache.nr; i++)
108 cb(negotiator, cache.items[i]);
109 }
110
111 static int rev_list_insert_ref(struct fetch_negotiator *negotiator,
112 const char *refname,
113 const struct object_id *oid)
114 {
115 struct object *o = deref_tag(the_repository,
116 parse_object(the_repository, oid),
117 refname, 0);
118
119 if (o && o->type == OBJ_COMMIT)
120 negotiator->add_tip(negotiator, (struct commit *)o);
121
122 return 0;
123 }
124
125 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
126 int flag, void *cb_data)
127 {
128 return rev_list_insert_ref(cb_data, refname, oid);
129 }
130
131 enum ack_type {
132 NAK = 0,
133 ACK,
134 ACK_continue,
135 ACK_common,
136 ACK_ready
137 };
138
139 static void consume_shallow_list(struct fetch_pack_args *args,
140 struct packet_reader *reader)
141 {
142 if (args->stateless_rpc && args->deepen) {
143 /* If we sent a depth we will get back "duplicate"
144 * shallow and unshallow commands every time there
145 * is a block of have lines exchanged.
146 */
147 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
148 if (starts_with(reader->line, "shallow "))
149 continue;
150 if (starts_with(reader->line, "unshallow "))
151 continue;
152 die(_("git fetch-pack: expected shallow list"));
153 }
154 if (reader->status != PACKET_READ_FLUSH)
155 die(_("git fetch-pack: expected a flush packet after shallow list"));
156 }
157 }
158
159 static enum ack_type get_ack(struct packet_reader *reader,
160 struct object_id *result_oid)
161 {
162 int len;
163 const char *arg;
164
165 if (packet_reader_read(reader) != PACKET_READ_NORMAL)
166 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
167 len = reader->pktlen;
168
169 if (!strcmp(reader->line, "NAK"))
170 return NAK;
171 if (skip_prefix(reader->line, "ACK ", &arg)) {
172 const char *p;
173 if (!parse_oid_hex(arg, result_oid, &p)) {
174 len -= p - reader->line;
175 if (len < 1)
176 return ACK;
177 if (strstr(p, "continue"))
178 return ACK_continue;
179 if (strstr(p, "common"))
180 return ACK_common;
181 if (strstr(p, "ready"))
182 return ACK_ready;
183 return ACK;
184 }
185 }
186 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), reader->line);
187 }
188
189 static void send_request(struct fetch_pack_args *args,
190 int fd, struct strbuf *buf)
191 {
192 if (args->stateless_rpc) {
193 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
194 packet_flush(fd);
195 } else {
196 if (write_in_full(fd, buf->buf, buf->len) < 0)
197 die_errno(_("unable to write to remote"));
198 }
199 }
200
201 static void insert_one_alternate_object(struct fetch_negotiator *negotiator,
202 struct object *obj)
203 {
204 rev_list_insert_ref(negotiator, NULL, &obj->oid);
205 }
206
207 #define INITIAL_FLUSH 16
208 #define PIPESAFE_FLUSH 32
209 #define LARGE_FLUSH 16384
210
211 static int next_flush(int stateless_rpc, int count)
212 {
213 if (stateless_rpc) {
214 if (count < LARGE_FLUSH)
215 count <<= 1;
216 else
217 count = count * 11 / 10;
218 } else {
219 if (count < PIPESAFE_FLUSH)
220 count <<= 1;
221 else
222 count += PIPESAFE_FLUSH;
223 }
224 return count;
225 }
226
227 static void mark_tips(struct fetch_negotiator *negotiator,
228 const struct oid_array *negotiation_tips)
229 {
230 int i;
231
232 if (!negotiation_tips) {
233 for_each_ref(rev_list_insert_ref_oid, negotiator);
234 return;
235 }
236
237 for (i = 0; i < negotiation_tips->nr; i++)
238 rev_list_insert_ref(negotiator, NULL,
239 &negotiation_tips->oid[i]);
240 return;
241 }
242
243 static int find_common(struct fetch_negotiator *negotiator,
244 struct fetch_pack_args *args,
245 int fd[2], struct object_id *result_oid,
246 struct ref *refs)
247 {
248 int fetching;
249 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
250 const struct object_id *oid;
251 unsigned in_vain = 0;
252 int got_continue = 0;
253 int got_ready = 0;
254 struct strbuf req_buf = STRBUF_INIT;
255 size_t state_len = 0;
256 struct packet_reader reader;
257
258 if (args->stateless_rpc && multi_ack == 1)
259 die(_("--stateless-rpc requires multi_ack_detailed"));
260
261 packet_reader_init(&reader, fd[0], NULL, 0,
262 PACKET_READ_CHOMP_NEWLINE |
263 PACKET_READ_DIE_ON_ERR_PACKET);
264
265 if (!args->no_dependents) {
266 mark_tips(negotiator, args->negotiation_tips);
267 for_each_cached_alternate(negotiator, insert_one_alternate_object);
268 }
269
270 fetching = 0;
271 for ( ; refs ; refs = refs->next) {
272 struct object_id *remote = &refs->old_oid;
273 const char *remote_hex;
274 struct object *o;
275
276 /*
277 * If that object is complete (i.e. it is an ancestor of a
278 * local ref), we tell them we have it but do not have to
279 * tell them about its ancestors, which they already know
280 * about.
281 *
282 * We use lookup_object here because we are only
283 * interested in the case we *know* the object is
284 * reachable and we have already scanned it.
285 *
286 * Do this only if args->no_dependents is false (if it is true,
287 * we cannot trust the object flags).
288 */
289 if (!args->no_dependents &&
290 ((o = lookup_object(the_repository, remote)) != NULL) &&
291 (o->flags & COMPLETE)) {
292 continue;
293 }
294
295 remote_hex = oid_to_hex(remote);
296 if (!fetching) {
297 struct strbuf c = STRBUF_INIT;
298 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
299 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
300 if (no_done) strbuf_addstr(&c, " no-done");
301 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
302 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
303 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
304 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
305 if (args->no_progress) strbuf_addstr(&c, " no-progress");
306 if (args->include_tag) strbuf_addstr(&c, " include-tag");
307 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
308 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
309 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
310 if (agent_supported) strbuf_addf(&c, " agent=%s",
311 git_user_agent_sanitized());
312 if (args->filter_options.choice)
313 strbuf_addstr(&c, " filter");
314 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
315 strbuf_release(&c);
316 } else
317 packet_buf_write(&req_buf, "want %s\n", remote_hex);
318 fetching++;
319 }
320
321 if (!fetching) {
322 strbuf_release(&req_buf);
323 packet_flush(fd[1]);
324 return 1;
325 }
326
327 if (is_repository_shallow(the_repository))
328 write_shallow_commits(&req_buf, 1, NULL);
329 if (args->depth > 0)
330 packet_buf_write(&req_buf, "deepen %d", args->depth);
331 if (args->deepen_since) {
332 timestamp_t max_age = approxidate(args->deepen_since);
333 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
334 }
335 if (args->deepen_not) {
336 int i;
337 for (i = 0; i < args->deepen_not->nr; i++) {
338 struct string_list_item *s = args->deepen_not->items + i;
339 packet_buf_write(&req_buf, "deepen-not %s", s->string);
340 }
341 }
342 if (server_supports_filtering && args->filter_options.choice) {
343 const char *spec =
344 expand_list_objects_filter_spec(&args->filter_options);
345 packet_buf_write(&req_buf, "filter %s", spec);
346 }
347 packet_buf_flush(&req_buf);
348 state_len = req_buf.len;
349
350 if (args->deepen) {
351 const char *arg;
352 struct object_id oid;
353
354 send_request(args, fd[1], &req_buf);
355 while (packet_reader_read(&reader) == PACKET_READ_NORMAL) {
356 if (skip_prefix(reader.line, "shallow ", &arg)) {
357 if (get_oid_hex(arg, &oid))
358 die(_("invalid shallow line: %s"), reader.line);
359 register_shallow(the_repository, &oid);
360 continue;
361 }
362 if (skip_prefix(reader.line, "unshallow ", &arg)) {
363 if (get_oid_hex(arg, &oid))
364 die(_("invalid unshallow line: %s"), reader.line);
365 if (!lookup_object(the_repository, &oid))
366 die(_("object not found: %s"), reader.line);
367 /* make sure that it is parsed as shallow */
368 if (!parse_object(the_repository, &oid))
369 die(_("error in object: %s"), reader.line);
370 if (unregister_shallow(&oid))
371 die(_("no shallow found: %s"), reader.line);
372 continue;
373 }
374 die(_("expected shallow/unshallow, got %s"), reader.line);
375 }
376 } else if (!args->stateless_rpc)
377 send_request(args, fd[1], &req_buf);
378
379 if (!args->stateless_rpc) {
380 /* If we aren't using the stateless-rpc interface
381 * we don't need to retain the headers.
382 */
383 strbuf_setlen(&req_buf, 0);
384 state_len = 0;
385 }
386
387 trace2_region_enter("fetch-pack", "negotiation_v0_v1", the_repository);
388 flushes = 0;
389 retval = -1;
390 if (args->no_dependents)
391 goto done;
392 while ((oid = negotiator->next(negotiator))) {
393 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
394 print_verbose(args, "have %s", oid_to_hex(oid));
395 in_vain++;
396 if (flush_at <= ++count) {
397 int ack;
398
399 packet_buf_flush(&req_buf);
400 send_request(args, fd[1], &req_buf);
401 strbuf_setlen(&req_buf, state_len);
402 flushes++;
403 flush_at = next_flush(args->stateless_rpc, count);
404
405 /*
406 * We keep one window "ahead" of the other side, and
407 * will wait for an ACK only on the next one
408 */
409 if (!args->stateless_rpc && count == INITIAL_FLUSH)
410 continue;
411
412 consume_shallow_list(args, &reader);
413 do {
414 ack = get_ack(&reader, result_oid);
415 if (ack)
416 print_verbose(args, _("got %s %d %s"), "ack",
417 ack, oid_to_hex(result_oid));
418 switch (ack) {
419 case ACK:
420 flushes = 0;
421 multi_ack = 0;
422 retval = 0;
423 goto done;
424 case ACK_common:
425 case ACK_ready:
426 case ACK_continue: {
427 struct commit *commit =
428 lookup_commit(the_repository,
429 result_oid);
430 int was_common;
431
432 if (!commit)
433 die(_("invalid commit %s"), oid_to_hex(result_oid));
434 was_common = negotiator->ack(negotiator, commit);
435 if (args->stateless_rpc
436 && ack == ACK_common
437 && !was_common) {
438 /* We need to replay the have for this object
439 * on the next RPC request so the peer knows
440 * it is in common with us.
441 */
442 const char *hex = oid_to_hex(result_oid);
443 packet_buf_write(&req_buf, "have %s\n", hex);
444 state_len = req_buf.len;
445 /*
446 * Reset in_vain because an ack
447 * for this commit has not been
448 * seen.
449 */
450 in_vain = 0;
451 } else if (!args->stateless_rpc
452 || ack != ACK_common)
453 in_vain = 0;
454 retval = 0;
455 got_continue = 1;
456 if (ack == ACK_ready)
457 got_ready = 1;
458 break;
459 }
460 }
461 } while (ack);
462 flushes--;
463 if (got_continue && MAX_IN_VAIN < in_vain) {
464 print_verbose(args, _("giving up"));
465 break; /* give up */
466 }
467 if (got_ready)
468 break;
469 }
470 }
471 done:
472 trace2_region_leave("fetch-pack", "negotiation_v0_v1", the_repository);
473 if (!got_ready || !no_done) {
474 packet_buf_write(&req_buf, "done\n");
475 send_request(args, fd[1], &req_buf);
476 }
477 print_verbose(args, _("done"));
478 if (retval != 0) {
479 multi_ack = 0;
480 flushes++;
481 }
482 strbuf_release(&req_buf);
483
484 if (!got_ready || !no_done)
485 consume_shallow_list(args, &reader);
486 while (flushes || multi_ack) {
487 int ack = get_ack(&reader, result_oid);
488 if (ack) {
489 print_verbose(args, _("got %s (%d) %s"), "ack",
490 ack, oid_to_hex(result_oid));
491 if (ack == ACK)
492 return 0;
493 multi_ack = 1;
494 continue;
495 }
496 flushes--;
497 }
498 /* it is no error to fetch into a completely empty repo */
499 return count ? retval : 0;
500 }
501
502 static struct commit_list *complete;
503
504 static int mark_complete(const struct object_id *oid)
505 {
506 struct object *o = parse_object(the_repository, oid);
507
508 while (o && o->type == OBJ_TAG) {
509 struct tag *t = (struct tag *) o;
510 if (!t->tagged)
511 break; /* broken repository */
512 o->flags |= COMPLETE;
513 o = parse_object(the_repository, &t->tagged->oid);
514 }
515 if (o && o->type == OBJ_COMMIT) {
516 struct commit *commit = (struct commit *)o;
517 if (!(commit->object.flags & COMPLETE)) {
518 commit->object.flags |= COMPLETE;
519 commit_list_insert(commit, &complete);
520 }
521 }
522 return 0;
523 }
524
525 static int mark_complete_oid(const char *refname, const struct object_id *oid,
526 int flag, void *cb_data)
527 {
528 return mark_complete(oid);
529 }
530
531 static void mark_recent_complete_commits(struct fetch_pack_args *args,
532 timestamp_t cutoff)
533 {
534 while (complete && cutoff <= complete->item->date) {
535 print_verbose(args, _("Marking %s as complete"),
536 oid_to_hex(&complete->item->object.oid));
537 pop_most_recent_commit(&complete, COMPLETE);
538 }
539 }
540
541 static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
542 {
543 for (; refs; refs = refs->next)
544 oidset_insert(oids, &refs->old_oid);
545 }
546
547 static int is_unmatched_ref(const struct ref *ref)
548 {
549 struct object_id oid;
550 const char *p;
551 return ref->match_status == REF_NOT_MATCHED &&
552 !parse_oid_hex(ref->name, &oid, &p) &&
553 *p == '\0' &&
554 oideq(&oid, &ref->old_oid);
555 }
556
557 static void filter_refs(struct fetch_pack_args *args,
558 struct ref **refs,
559 struct ref **sought, int nr_sought)
560 {
561 struct ref *newlist = NULL;
562 struct ref **newtail = &newlist;
563 struct ref *unmatched = NULL;
564 struct ref *ref, *next;
565 struct oidset tip_oids = OIDSET_INIT;
566 int i;
567 int strict = !(allow_unadvertised_object_request &
568 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1));
569
570 i = 0;
571 for (ref = *refs; ref; ref = next) {
572 int keep = 0;
573 next = ref->next;
574
575 if (starts_with(ref->name, "refs/") &&
576 check_refname_format(ref->name, 0)) {
577 /*
578 * trash or a peeled value; do not even add it to
579 * unmatched list
580 */
581 free_one_ref(ref);
582 continue;
583 } else {
584 while (i < nr_sought) {
585 int cmp = strcmp(ref->name, sought[i]->name);
586 if (cmp < 0)
587 break; /* definitely do not have it */
588 else if (cmp == 0) {
589 keep = 1; /* definitely have it */
590 sought[i]->match_status = REF_MATCHED;
591 }
592 i++;
593 }
594
595 if (!keep && args->fetch_all &&
596 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
597 keep = 1;
598 }
599
600 if (keep) {
601 *newtail = ref;
602 ref->next = NULL;
603 newtail = &ref->next;
604 } else {
605 ref->next = unmatched;
606 unmatched = ref;
607 }
608 }
609
610 if (strict) {
611 for (i = 0; i < nr_sought; i++) {
612 ref = sought[i];
613 if (!is_unmatched_ref(ref))
614 continue;
615
616 add_refs_to_oidset(&tip_oids, unmatched);
617 add_refs_to_oidset(&tip_oids, newlist);
618 break;
619 }
620 }
621
622 /* Append unmatched requests to the list */
623 for (i = 0; i < nr_sought; i++) {
624 ref = sought[i];
625 if (!is_unmatched_ref(ref))
626 continue;
627
628 if (!strict || oidset_contains(&tip_oids, &ref->old_oid)) {
629 ref->match_status = REF_MATCHED;
630 *newtail = copy_ref(ref);
631 newtail = &(*newtail)->next;
632 } else {
633 ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
634 }
635 }
636
637 oidset_clear(&tip_oids);
638 free_refs(unmatched);
639
640 *refs = newlist;
641 }
642
643 static void mark_alternate_complete(struct fetch_negotiator *unused,
644 struct object *obj)
645 {
646 mark_complete(&obj->oid);
647 }
648
649 struct loose_object_iter {
650 struct oidset *loose_object_set;
651 struct ref *refs;
652 };
653
654 /*
655 * Mark recent commits available locally and reachable from a local ref as
656 * COMPLETE. If args->no_dependents is false, also mark COMPLETE remote refs as
657 * COMMON_REF (otherwise, we are not planning to participate in negotiation, and
658 * thus do not need COMMON_REF marks).
659 *
660 * The cutoff time for recency is determined by this heuristic: it is the
661 * earliest commit time of the objects in refs that are commits and that we know
662 * the commit time of.
663 */
664 static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
665 struct fetch_pack_args *args,
666 struct ref **refs)
667 {
668 struct ref *ref;
669 int old_save_commit_buffer = save_commit_buffer;
670 timestamp_t cutoff = 0;
671
672 save_commit_buffer = 0;
673
674 trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
675 for (ref = *refs; ref; ref = ref->next) {
676 struct object *o;
677
678 if (!has_object_file_with_flags(&ref->old_oid,
679 OBJECT_INFO_QUICK |
680 OBJECT_INFO_SKIP_FETCH_OBJECT))
681 continue;
682 o = parse_object(the_repository, &ref->old_oid);
683 if (!o)
684 continue;
685
686 /*
687 * We already have it -- which may mean that we were
688 * in sync with the other side at some time after
689 * that (it is OK if we guess wrong here).
690 */
691 if (o->type == OBJ_COMMIT) {
692 struct commit *commit = (struct commit *)o;
693 if (!cutoff || cutoff < commit->date)
694 cutoff = commit->date;
695 }
696 }
697 trace2_region_leave("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
698
699 /*
700 * This block marks all local refs as COMPLETE, and then recursively marks all
701 * parents of those refs as COMPLETE.
702 */
703 trace2_region_enter("fetch-pack", "mark_complete_local_refs", NULL);
704 if (!args->deepen) {
705 for_each_ref(mark_complete_oid, NULL);
706 for_each_cached_alternate(NULL, mark_alternate_complete);
707 commit_list_sort_by_date(&complete);
708 if (cutoff)
709 mark_recent_complete_commits(args, cutoff);
710 }
711 trace2_region_leave("fetch-pack", "mark_complete_local_refs", NULL);
712
713 /*
714 * Mark all complete remote refs as common refs.
715 * Don't mark them common yet; the server has to be told so first.
716 */
717 trace2_region_enter("fetch-pack", "mark_common_remote_refs", NULL);
718 for (ref = *refs; ref; ref = ref->next) {
719 struct object *o = deref_tag(the_repository,
720 lookup_object(the_repository,
721 &ref->old_oid),
722 NULL, 0);
723
724 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
725 continue;
726
727 negotiator->known_common(negotiator,
728 (struct commit *)o);
729 }
730 trace2_region_leave("fetch-pack", "mark_common_remote_refs", NULL);
731
732 save_commit_buffer = old_save_commit_buffer;
733 }
734
735 /*
736 * Returns 1 if every object pointed to by the given remote refs is available
737 * locally and reachable from a local ref, and 0 otherwise.
738 */
739 static int everything_local(struct fetch_pack_args *args,
740 struct ref **refs)
741 {
742 struct ref *ref;
743 int retval;
744
745 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
746 const struct object_id *remote = &ref->old_oid;
747 struct object *o;
748
749 o = lookup_object(the_repository, remote);
750 if (!o || !(o->flags & COMPLETE)) {
751 retval = 0;
752 print_verbose(args, "want %s (%s)", oid_to_hex(remote),
753 ref->name);
754 continue;
755 }
756 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
757 ref->name);
758 }
759
760 return retval;
761 }
762
763 static int sideband_demux(int in, int out, void *data)
764 {
765 int *xd = data;
766 int ret;
767
768 ret = recv_sideband("fetch-pack", xd[0], out);
769 close(out);
770 return ret;
771 }
772
773 static void write_promisor_file(const char *keep_name,
774 struct ref **sought, int nr_sought)
775 {
776 struct strbuf promisor_name = STRBUF_INIT;
777 int suffix_stripped;
778 FILE *output;
779 int i;
780
781 strbuf_addstr(&promisor_name, keep_name);
782 suffix_stripped = strbuf_strip_suffix(&promisor_name, ".keep");
783 if (!suffix_stripped)
784 BUG("name of pack lockfile should end with .keep (was '%s')",
785 keep_name);
786 strbuf_addstr(&promisor_name, ".promisor");
787
788 output = xfopen(promisor_name.buf, "w");
789 for (i = 0; i < nr_sought; i++)
790 fprintf(output, "%s %s\n", oid_to_hex(&sought[i]->old_oid),
791 sought[i]->name);
792 fclose(output);
793
794 strbuf_release(&promisor_name);
795 }
796
797 static int get_pack(struct fetch_pack_args *args,
798 int xd[2], struct string_list *pack_lockfiles,
799 int only_packfile,
800 struct ref **sought, int nr_sought)
801 {
802 struct async demux;
803 int do_keep = args->keep_pack;
804 const char *cmd_name;
805 struct pack_header header;
806 int pass_header = 0;
807 struct child_process cmd = CHILD_PROCESS_INIT;
808 int ret;
809
810 memset(&demux, 0, sizeof(demux));
811 if (use_sideband) {
812 /* xd[] is talking with upload-pack; subprocess reads from
813 * xd[0], spits out band#2 to stderr, and feeds us band#1
814 * through demux->out.
815 */
816 demux.proc = sideband_demux;
817 demux.data = xd;
818 demux.out = -1;
819 demux.isolate_sigpipe = 1;
820 if (start_async(&demux))
821 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
822 }
823 else
824 demux.out = xd[0];
825
826 if (!args->keep_pack && unpack_limit) {
827
828 if (read_pack_header(demux.out, &header))
829 die(_("protocol error: bad pack header"));
830 pass_header = 1;
831 if (ntohl(header.hdr_entries) < unpack_limit)
832 do_keep = 0;
833 else
834 do_keep = 1;
835 }
836
837 if (alternate_shallow_file) {
838 argv_array_push(&cmd.args, "--shallow-file");
839 argv_array_push(&cmd.args, alternate_shallow_file);
840 }
841
842 if (do_keep || args->from_promisor) {
843 if (pack_lockfiles)
844 cmd.out = -1;
845 cmd_name = "index-pack";
846 argv_array_push(&cmd.args, cmd_name);
847 argv_array_push(&cmd.args, "--stdin");
848 if (!args->quiet && !args->no_progress)
849 argv_array_push(&cmd.args, "-v");
850 if (args->use_thin_pack)
851 argv_array_push(&cmd.args, "--fix-thin");
852 if (do_keep && (args->lock_pack || unpack_limit)) {
853 char hostname[HOST_NAME_MAX + 1];
854 if (xgethostname(hostname, sizeof(hostname)))
855 xsnprintf(hostname, sizeof(hostname), "localhost");
856 argv_array_pushf(&cmd.args,
857 "--keep=fetch-pack %"PRIuMAX " on %s",
858 (uintmax_t)getpid(), hostname);
859 }
860 if (only_packfile && args->check_self_contained_and_connected)
861 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
862 else
863 /*
864 * We cannot perform any connectivity checks because
865 * not all packs have been downloaded; let the caller
866 * have this responsibility.
867 */
868 args->check_self_contained_and_connected = 0;
869 /*
870 * If we're obtaining the filename of a lockfile, we'll use
871 * that filename to write a .promisor file with more
872 * information below. If not, we need index-pack to do it for
873 * us.
874 */
875 if (!(do_keep && pack_lockfiles) && args->from_promisor)
876 argv_array_push(&cmd.args, "--promisor");
877 }
878 else {
879 cmd_name = "unpack-objects";
880 argv_array_push(&cmd.args, cmd_name);
881 if (args->quiet || args->no_progress)
882 argv_array_push(&cmd.args, "-q");
883 args->check_self_contained_and_connected = 0;
884 }
885
886 if (pass_header)
887 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
888 ntohl(header.hdr_version),
889 ntohl(header.hdr_entries));
890 if (fetch_fsck_objects >= 0
891 ? fetch_fsck_objects
892 : transfer_fsck_objects >= 0
893 ? transfer_fsck_objects
894 : 0) {
895 if (args->from_promisor)
896 /*
897 * We cannot use --strict in index-pack because it
898 * checks both broken objects and links, but we only
899 * want to check for broken objects.
900 */
901 argv_array_push(&cmd.args, "--fsck-objects");
902 else
903 argv_array_pushf(&cmd.args, "--strict%s",
904 fsck_msg_types.buf);
905 }
906
907 cmd.in = demux.out;
908 cmd.git_cmd = 1;
909 if (start_command(&cmd))
910 die(_("fetch-pack: unable to fork off %s"), cmd_name);
911 if (do_keep && pack_lockfiles) {
912 string_list_append_nodup(pack_lockfiles,
913 index_pack_lockfile(cmd.out));
914 close(cmd.out);
915 }
916
917 if (!use_sideband)
918 /* Closed by start_command() */
919 xd[0] = -1;
920
921 ret = finish_command(&cmd);
922 if (!ret || (args->check_self_contained_and_connected && ret == 1))
923 args->self_contained_and_connected =
924 args->check_self_contained_and_connected &&
925 ret == 0;
926 else
927 die(_("%s failed"), cmd_name);
928 if (use_sideband && finish_async(&demux))
929 die(_("error in sideband demultiplexer"));
930
931 /*
932 * Now that index-pack has succeeded, write the promisor file using the
933 * obtained .keep filename if necessary
934 */
935 if (do_keep && pack_lockfiles && pack_lockfiles->nr && args->from_promisor)
936 write_promisor_file(pack_lockfiles->items[0].string, sought, nr_sought);
937
938 return 0;
939 }
940
941 static int cmp_ref_by_name(const void *a_, const void *b_)
942 {
943 const struct ref *a = *((const struct ref **)a_);
944 const struct ref *b = *((const struct ref **)b_);
945 return strcmp(a->name, b->name);
946 }
947
948 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
949 int fd[2],
950 const struct ref *orig_ref,
951 struct ref **sought, int nr_sought,
952 struct shallow_info *si,
953 struct string_list *pack_lockfiles)
954 {
955 struct repository *r = the_repository;
956 struct ref *ref = copy_ref_list(orig_ref);
957 struct object_id oid;
958 const char *agent_feature;
959 int agent_len;
960 struct fetch_negotiator negotiator_alloc;
961 struct fetch_negotiator *negotiator;
962
963 if (args->no_dependents) {
964 negotiator = NULL;
965 } else {
966 negotiator = &negotiator_alloc;
967 fetch_negotiator_init(r, negotiator);
968 }
969
970 sort_ref_list(&ref, ref_compare_name);
971 QSORT(sought, nr_sought, cmp_ref_by_name);
972
973 if ((agent_feature = server_feature_value("agent", &agent_len))) {
974 agent_supported = 1;
975 if (agent_len)
976 print_verbose(args, _("Server version is %.*s"),
977 agent_len, agent_feature);
978 }
979
980 if (server_supports("shallow"))
981 print_verbose(args, _("Server supports %s"), "shallow");
982 else if (args->depth > 0 || is_repository_shallow(r))
983 die(_("Server does not support shallow clients"));
984 if (args->depth > 0 || args->deepen_since || args->deepen_not)
985 args->deepen = 1;
986 if (server_supports("multi_ack_detailed")) {
987 print_verbose(args, _("Server supports %s"), "multi_ack_detailed");
988 multi_ack = 2;
989 if (server_supports("no-done")) {
990 print_verbose(args, _("Server supports %s"), "no-done");
991 if (args->stateless_rpc)
992 no_done = 1;
993 }
994 }
995 else if (server_supports("multi_ack")) {
996 print_verbose(args, _("Server supports %s"), "multi_ack");
997 multi_ack = 1;
998 }
999 if (server_supports("side-band-64k")) {
1000 print_verbose(args, _("Server supports %s"), "side-band-64k");
1001 use_sideband = 2;
1002 }
1003 else if (server_supports("side-band")) {
1004 print_verbose(args, _("Server supports %s"), "side-band");
1005 use_sideband = 1;
1006 }
1007 if (server_supports("allow-tip-sha1-in-want")) {
1008 print_verbose(args, _("Server supports %s"), "allow-tip-sha1-in-want");
1009 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1010 }
1011 if (server_supports("allow-reachable-sha1-in-want")) {
1012 print_verbose(args, _("Server supports %s"), "allow-reachable-sha1-in-want");
1013 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1014 }
1015 if (server_supports("thin-pack"))
1016 print_verbose(args, _("Server supports %s"), "thin-pack");
1017 else
1018 args->use_thin_pack = 0;
1019 if (server_supports("no-progress"))
1020 print_verbose(args, _("Server supports %s"), "no-progress");
1021 else
1022 args->no_progress = 0;
1023 if (server_supports("include-tag"))
1024 print_verbose(args, _("Server supports %s"), "include-tag");
1025 else
1026 args->include_tag = 0;
1027 if (server_supports("ofs-delta"))
1028 print_verbose(args, _("Server supports %s"), "ofs-delta");
1029 else
1030 prefer_ofs_delta = 0;
1031
1032 if (server_supports("filter")) {
1033 server_supports_filtering = 1;
1034 print_verbose(args, _("Server supports %s"), "filter");
1035 } else if (args->filter_options.choice) {
1036 warning("filtering not recognized by server, ignoring");
1037 }
1038
1039 if (server_supports("deepen-since")) {
1040 print_verbose(args, _("Server supports %s"), "deepen-since");
1041 deepen_since_ok = 1;
1042 } else if (args->deepen_since)
1043 die(_("Server does not support --shallow-since"));
1044 if (server_supports("deepen-not")) {
1045 print_verbose(args, _("Server supports %s"), "deepen-not");
1046 deepen_not_ok = 1;
1047 } else if (args->deepen_not)
1048 die(_("Server does not support --shallow-exclude"));
1049 if (server_supports("deepen-relative"))
1050 print_verbose(args, _("Server supports %s"), "deepen-relative");
1051 else if (args->deepen_relative)
1052 die(_("Server does not support --deepen"));
1053 if (!server_supports_hash(the_hash_algo->name, NULL))
1054 die(_("Server does not support this repository's object format"));
1055
1056 if (!args->no_dependents) {
1057 mark_complete_and_common_ref(negotiator, args, &ref);
1058 filter_refs(args, &ref, sought, nr_sought);
1059 if (everything_local(args, &ref)) {
1060 packet_flush(fd[1]);
1061 goto all_done;
1062 }
1063 } else {
1064 filter_refs(args, &ref, sought, nr_sought);
1065 }
1066 if (find_common(negotiator, args, fd, &oid, ref) < 0)
1067 if (!args->keep_pack)
1068 /* When cloning, it is not unusual to have
1069 * no common commit.
1070 */
1071 warning(_("no common commits"));
1072
1073 if (args->stateless_rpc)
1074 packet_flush(fd[1]);
1075 if (args->deepen)
1076 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1077 NULL);
1078 else if (si->nr_ours || si->nr_theirs)
1079 alternate_shallow_file = setup_temporary_shallow(si->shallow);
1080 else
1081 alternate_shallow_file = NULL;
1082 if (get_pack(args, fd, pack_lockfiles, 1, sought, nr_sought))
1083 die(_("git fetch-pack: fetch failed."));
1084
1085 all_done:
1086 if (negotiator)
1087 negotiator->release(negotiator);
1088 return ref;
1089 }
1090
1091 static void add_shallow_requests(struct strbuf *req_buf,
1092 const struct fetch_pack_args *args)
1093 {
1094 if (is_repository_shallow(the_repository))
1095 write_shallow_commits(req_buf, 1, NULL);
1096 if (args->depth > 0)
1097 packet_buf_write(req_buf, "deepen %d", args->depth);
1098 if (args->deepen_since) {
1099 timestamp_t max_age = approxidate(args->deepen_since);
1100 packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1101 }
1102 if (args->deepen_not) {
1103 int i;
1104 for (i = 0; i < args->deepen_not->nr; i++) {
1105 struct string_list_item *s = args->deepen_not->items + i;
1106 packet_buf_write(req_buf, "deepen-not %s", s->string);
1107 }
1108 }
1109 if (args->deepen_relative)
1110 packet_buf_write(req_buf, "deepen-relative\n");
1111 }
1112
1113 static void add_wants(int no_dependents, const struct ref *wants, struct strbuf *req_buf)
1114 {
1115 int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0);
1116
1117 for ( ; wants ; wants = wants->next) {
1118 const struct object_id *remote = &wants->old_oid;
1119 struct object *o;
1120
1121 /*
1122 * If that object is complete (i.e. it is an ancestor of a
1123 * local ref), we tell them we have it but do not have to
1124 * tell them about its ancestors, which they already know
1125 * about.
1126 *
1127 * We use lookup_object here because we are only
1128 * interested in the case we *know* the object is
1129 * reachable and we have already scanned it.
1130 *
1131 * Do this only if args->no_dependents is false (if it is true,
1132 * we cannot trust the object flags).
1133 */
1134 if (!no_dependents &&
1135 ((o = lookup_object(the_repository, remote)) != NULL) &&
1136 (o->flags & COMPLETE)) {
1137 continue;
1138 }
1139
1140 if (!use_ref_in_want || wants->exact_oid)
1141 packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote));
1142 else
1143 packet_buf_write(req_buf, "want-ref %s\n", wants->name);
1144 }
1145 }
1146
1147 static void add_common(struct strbuf *req_buf, struct oidset *common)
1148 {
1149 struct oidset_iter iter;
1150 const struct object_id *oid;
1151 oidset_iter_init(common, &iter);
1152
1153 while ((oid = oidset_iter_next(&iter))) {
1154 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1155 }
1156 }
1157
1158 static int add_haves(struct fetch_negotiator *negotiator,
1159 int seen_ack,
1160 struct strbuf *req_buf,
1161 int *haves_to_send, int *in_vain)
1162 {
1163 int ret = 0;
1164 int haves_added = 0;
1165 const struct object_id *oid;
1166
1167 while ((oid = negotiator->next(negotiator))) {
1168 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1169 if (++haves_added >= *haves_to_send)
1170 break;
1171 }
1172
1173 *in_vain += haves_added;
1174 if (!haves_added || (seen_ack && *in_vain >= MAX_IN_VAIN)) {
1175 /* Send Done */
1176 packet_buf_write(req_buf, "done\n");
1177 ret = 1;
1178 }
1179
1180 /* Increase haves to send on next round */
1181 *haves_to_send = next_flush(1, *haves_to_send);
1182
1183 return ret;
1184 }
1185
1186 static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
1187 struct fetch_pack_args *args,
1188 const struct ref *wants, struct oidset *common,
1189 int *haves_to_send, int *in_vain,
1190 int sideband_all, int seen_ack)
1191 {
1192 int ret = 0;
1193 const char *hash_name;
1194 struct strbuf req_buf = STRBUF_INIT;
1195
1196 if (server_supports_v2("fetch", 1))
1197 packet_buf_write(&req_buf, "command=fetch");
1198 if (server_supports_v2("agent", 0))
1199 packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
1200 if (args->server_options && args->server_options->nr &&
1201 server_supports_v2("server-option", 1)) {
1202 int i;
1203 for (i = 0; i < args->server_options->nr; i++)
1204 packet_buf_write(&req_buf, "server-option=%s",
1205 args->server_options->items[i].string);
1206 }
1207
1208 if (server_feature_v2("object-format", &hash_name)) {
1209 int hash_algo = hash_algo_by_name(hash_name);
1210 if (hash_algo_by_ptr(the_hash_algo) != hash_algo)
1211 die(_("mismatched algorithms: client %s; server %s"),
1212 the_hash_algo->name, hash_name);
1213 packet_write_fmt(fd_out, "object-format=%s", the_hash_algo->name);
1214 } else if (hash_algo_by_ptr(the_hash_algo) != GIT_HASH_SHA1) {
1215 die(_("the server does not support algorithm '%s'"),
1216 the_hash_algo->name);
1217 }
1218
1219 packet_buf_delim(&req_buf);
1220 if (args->use_thin_pack)
1221 packet_buf_write(&req_buf, "thin-pack");
1222 if (args->no_progress)
1223 packet_buf_write(&req_buf, "no-progress");
1224 if (args->include_tag)
1225 packet_buf_write(&req_buf, "include-tag");
1226 if (prefer_ofs_delta)
1227 packet_buf_write(&req_buf, "ofs-delta");
1228 if (sideband_all)
1229 packet_buf_write(&req_buf, "sideband-all");
1230
1231 /* Add shallow-info and deepen request */
1232 if (server_supports_feature("fetch", "shallow", 0))
1233 add_shallow_requests(&req_buf, args);
1234 else if (is_repository_shallow(the_repository) || args->deepen)
1235 die(_("Server does not support shallow requests"));
1236
1237 /* Add filter */
1238 if (server_supports_feature("fetch", "filter", 0) &&
1239 args->filter_options.choice) {
1240 const char *spec =
1241 expand_list_objects_filter_spec(&args->filter_options);
1242 print_verbose(args, _("Server supports filter"));
1243 packet_buf_write(&req_buf, "filter %s", spec);
1244 } else if (args->filter_options.choice) {
1245 warning("filtering not recognized by server, ignoring");
1246 }
1247
1248 if (server_supports_feature("fetch", "packfile-uris", 0)) {
1249 int i;
1250 struct strbuf to_send = STRBUF_INIT;
1251
1252 for (i = 0; i < uri_protocols.nr; i++) {
1253 const char *s = uri_protocols.items[i].string;
1254
1255 if (!strcmp(s, "https") || !strcmp(s, "http")) {
1256 if (to_send.len)
1257 strbuf_addch(&to_send, ',');
1258 strbuf_addstr(&to_send, s);
1259 }
1260 }
1261 if (to_send.len) {
1262 packet_buf_write(&req_buf, "packfile-uris %s",
1263 to_send.buf);
1264 strbuf_release(&to_send);
1265 }
1266 }
1267
1268 /* add wants */
1269 add_wants(args->no_dependents, wants, &req_buf);
1270
1271 if (args->no_dependents) {
1272 packet_buf_write(&req_buf, "done");
1273 ret = 1;
1274 } else {
1275 /* Add all of the common commits we've found in previous rounds */
1276 add_common(&req_buf, common);
1277
1278 /* Add initial haves */
1279 ret = add_haves(negotiator, seen_ack, &req_buf,
1280 haves_to_send, in_vain);
1281 }
1282
1283 /* Send request */
1284 packet_buf_flush(&req_buf);
1285 if (write_in_full(fd_out, req_buf.buf, req_buf.len) < 0)
1286 die_errno(_("unable to write request to remote"));
1287
1288 strbuf_release(&req_buf);
1289 return ret;
1290 }
1291
1292 /*
1293 * Processes a section header in a server's response and checks if it matches
1294 * `section`. If the value of `peek` is 1, the header line will be peeked (and
1295 * not consumed); if 0, the line will be consumed and the function will die if
1296 * the section header doesn't match what was expected.
1297 */
1298 static int process_section_header(struct packet_reader *reader,
1299 const char *section, int peek)
1300 {
1301 int ret;
1302
1303 if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
1304 die(_("error reading section header '%s'"), section);
1305
1306 ret = !strcmp(reader->line, section);
1307
1308 if (!peek) {
1309 if (!ret)
1310 die(_("expected '%s', received '%s'"),
1311 section, reader->line);
1312 packet_reader_read(reader);
1313 }
1314
1315 return ret;
1316 }
1317
1318 enum common_found {
1319 /*
1320 * No commit was found to be possessed by both the client and the
1321 * server, and "ready" was not received.
1322 */
1323 NO_COMMON_FOUND,
1324
1325 /*
1326 * At least one commit was found to be possessed by both the client and
1327 * the server, and "ready" was not received.
1328 */
1329 COMMON_FOUND,
1330
1331 /*
1332 * "ready" was received, indicating that the server is ready to send
1333 * the packfile without any further negotiation.
1334 */
1335 READY
1336 };
1337
1338 static enum common_found process_acks(struct fetch_negotiator *negotiator,
1339 struct packet_reader *reader,
1340 struct oidset *common)
1341 {
1342 /* received */
1343 int received_ready = 0;
1344 int received_ack = 0;
1345
1346 process_section_header(reader, "acknowledgments", 0);
1347 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1348 const char *arg;
1349
1350 if (!strcmp(reader->line, "NAK"))
1351 continue;
1352
1353 if (skip_prefix(reader->line, "ACK ", &arg)) {
1354 struct object_id oid;
1355 received_ack = 1;
1356 if (!get_oid_hex(arg, &oid)) {
1357 struct commit *commit;
1358 oidset_insert(common, &oid);
1359 commit = lookup_commit(the_repository, &oid);
1360 if (negotiator)
1361 negotiator->ack(negotiator, commit);
1362 }
1363 continue;
1364 }
1365
1366 if (!strcmp(reader->line, "ready")) {
1367 received_ready = 1;
1368 continue;
1369 }
1370
1371 die(_("unexpected acknowledgment line: '%s'"), reader->line);
1372 }
1373
1374 if (reader->status != PACKET_READ_FLUSH &&
1375 reader->status != PACKET_READ_DELIM)
1376 die(_("error processing acks: %d"), reader->status);
1377
1378 /*
1379 * If an "acknowledgments" section is sent, a packfile is sent if and
1380 * only if "ready" was sent in this section. The other sections
1381 * ("shallow-info" and "wanted-refs") are sent only if a packfile is
1382 * sent. Therefore, a DELIM is expected if "ready" is sent, and a FLUSH
1383 * otherwise.
1384 */
1385 if (received_ready && reader->status != PACKET_READ_DELIM)
1386 die(_("expected packfile to be sent after 'ready'"));
1387 if (!received_ready && reader->status != PACKET_READ_FLUSH)
1388 die(_("expected no other sections to be sent after no 'ready'"));
1389
1390 return received_ready ? READY :
1391 (received_ack ? COMMON_FOUND : NO_COMMON_FOUND);
1392 }
1393
1394 static void receive_shallow_info(struct fetch_pack_args *args,
1395 struct packet_reader *reader,
1396 struct oid_array *shallows,
1397 struct shallow_info *si)
1398 {
1399 int unshallow_received = 0;
1400
1401 process_section_header(reader, "shallow-info", 0);
1402 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1403 const char *arg;
1404 struct object_id oid;
1405
1406 if (skip_prefix(reader->line, "shallow ", &arg)) {
1407 if (get_oid_hex(arg, &oid))
1408 die(_("invalid shallow line: %s"), reader->line);
1409 oid_array_append(shallows, &oid);
1410 continue;
1411 }
1412 if (skip_prefix(reader->line, "unshallow ", &arg)) {
1413 if (get_oid_hex(arg, &oid))
1414 die(_("invalid unshallow line: %s"), reader->line);
1415 if (!lookup_object(the_repository, &oid))
1416 die(_("object not found: %s"), reader->line);
1417 /* make sure that it is parsed as shallow */
1418 if (!parse_object(the_repository, &oid))
1419 die(_("error in object: %s"), reader->line);
1420 if (unregister_shallow(&oid))
1421 die(_("no shallow found: %s"), reader->line);
1422 unshallow_received = 1;
1423 continue;
1424 }
1425 die(_("expected shallow/unshallow, got %s"), reader->line);
1426 }
1427
1428 if (reader->status != PACKET_READ_FLUSH &&
1429 reader->status != PACKET_READ_DELIM)
1430 die(_("error processing shallow info: %d"), reader->status);
1431
1432 if (args->deepen || unshallow_received) {
1433 /*
1434 * Treat these as shallow lines caused by our depth settings.
1435 * In v0, these lines cannot cause refs to be rejected; do the
1436 * same.
1437 */
1438 int i;
1439
1440 for (i = 0; i < shallows->nr; i++)
1441 register_shallow(the_repository, &shallows->oid[i]);
1442 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1443 NULL);
1444 args->deepen = 1;
1445 } else if (shallows->nr) {
1446 /*
1447 * Treat these as shallow lines caused by the remote being
1448 * shallow. In v0, remote refs that reach these objects are
1449 * rejected (unless --update-shallow is set); do the same.
1450 */
1451 prepare_shallow_info(si, shallows);
1452 if (si->nr_ours || si->nr_theirs)
1453 alternate_shallow_file =
1454 setup_temporary_shallow(si->shallow);
1455 else
1456 alternate_shallow_file = NULL;
1457 } else {
1458 alternate_shallow_file = NULL;
1459 }
1460 }
1461
1462 static int cmp_name_ref(const void *name, const void *ref)
1463 {
1464 return strcmp(name, (*(struct ref **)ref)->name);
1465 }
1466
1467 static void receive_wanted_refs(struct packet_reader *reader,
1468 struct ref **sought, int nr_sought)
1469 {
1470 process_section_header(reader, "wanted-refs", 0);
1471 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1472 struct object_id oid;
1473 const char *end;
1474 struct ref **found;
1475
1476 if (parse_oid_hex(reader->line, &oid, &end) || *end++ != ' ')
1477 die(_("expected wanted-ref, got '%s'"), reader->line);
1478
1479 found = bsearch(end, sought, nr_sought, sizeof(*sought),
1480 cmp_name_ref);
1481 if (!found)
1482 die(_("unexpected wanted-ref: '%s'"), reader->line);
1483 oidcpy(&(*found)->old_oid, &oid);
1484 }
1485
1486 if (reader->status != PACKET_READ_DELIM)
1487 die(_("error processing wanted refs: %d"), reader->status);
1488 }
1489
1490 static void receive_packfile_uris(struct packet_reader *reader,
1491 struct string_list *uris)
1492 {
1493 process_section_header(reader, "packfile-uris", 0);
1494 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1495 if (reader->pktlen < the_hash_algo->hexsz ||
1496 reader->line[the_hash_algo->hexsz] != ' ')
1497 die("expected '<hash> <uri>', got: %s\n", reader->line);
1498
1499 string_list_append(uris, reader->line);
1500 }
1501 if (reader->status != PACKET_READ_DELIM)
1502 die("expected DELIM");
1503 }
1504
1505 enum fetch_state {
1506 FETCH_CHECK_LOCAL = 0,
1507 FETCH_SEND_REQUEST,
1508 FETCH_PROCESS_ACKS,
1509 FETCH_GET_PACK,
1510 FETCH_DONE,
1511 };
1512
1513 static void do_check_stateless_delimiter(const struct fetch_pack_args *args,
1514 struct packet_reader *reader)
1515 {
1516 check_stateless_delimiter(args->stateless_rpc, reader,
1517 _("git fetch-pack: expected response end packet"));
1518 }
1519
1520 static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1521 int fd[2],
1522 const struct ref *orig_ref,
1523 struct ref **sought, int nr_sought,
1524 struct oid_array *shallows,
1525 struct shallow_info *si,
1526 struct string_list *pack_lockfiles)
1527 {
1528 struct repository *r = the_repository;
1529 struct ref *ref = copy_ref_list(orig_ref);
1530 enum fetch_state state = FETCH_CHECK_LOCAL;
1531 struct oidset common = OIDSET_INIT;
1532 struct packet_reader reader;
1533 int in_vain = 0, negotiation_started = 0;
1534 int haves_to_send = INITIAL_FLUSH;
1535 struct fetch_negotiator negotiator_alloc;
1536 struct fetch_negotiator *negotiator;
1537 int seen_ack = 0;
1538 struct string_list packfile_uris = STRING_LIST_INIT_DUP;
1539 int i;
1540
1541 if (args->no_dependents) {
1542 negotiator = NULL;
1543 } else {
1544 negotiator = &negotiator_alloc;
1545 fetch_negotiator_init(r, negotiator);
1546 }
1547
1548 packet_reader_init(&reader, fd[0], NULL, 0,
1549 PACKET_READ_CHOMP_NEWLINE |
1550 PACKET_READ_DIE_ON_ERR_PACKET);
1551 if (git_env_bool("GIT_TEST_SIDEBAND_ALL", 1) &&
1552 server_supports_feature("fetch", "sideband-all", 0)) {
1553 reader.use_sideband = 1;
1554 reader.me = "fetch-pack";
1555 }
1556
1557 while (state != FETCH_DONE) {
1558 switch (state) {
1559 case FETCH_CHECK_LOCAL:
1560 sort_ref_list(&ref, ref_compare_name);
1561 QSORT(sought, nr_sought, cmp_ref_by_name);
1562
1563 /* v2 supports these by default */
1564 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1565 use_sideband = 2;
1566 if (args->depth > 0 || args->deepen_since || args->deepen_not)
1567 args->deepen = 1;
1568
1569 /* Filter 'ref' by 'sought' and those that aren't local */
1570 if (!args->no_dependents) {
1571 mark_complete_and_common_ref(negotiator, args, &ref);
1572 filter_refs(args, &ref, sought, nr_sought);
1573 if (everything_local(args, &ref))
1574 state = FETCH_DONE;
1575 else
1576 state = FETCH_SEND_REQUEST;
1577
1578 mark_tips(negotiator, args->negotiation_tips);
1579 for_each_cached_alternate(negotiator,
1580 insert_one_alternate_object);
1581 } else {
1582 filter_refs(args, &ref, sought, nr_sought);
1583 state = FETCH_SEND_REQUEST;
1584 }
1585 break;
1586 case FETCH_SEND_REQUEST:
1587 if (!negotiation_started) {
1588 negotiation_started = 1;
1589 trace2_region_enter("fetch-pack",
1590 "negotiation_v2",
1591 the_repository);
1592 }
1593 if (send_fetch_request(negotiator, fd[1], args, ref,
1594 &common,
1595 &haves_to_send, &in_vain,
1596 reader.use_sideband,
1597 seen_ack))
1598 state = FETCH_GET_PACK;
1599 else
1600 state = FETCH_PROCESS_ACKS;
1601 break;
1602 case FETCH_PROCESS_ACKS:
1603 /* Process ACKs/NAKs */
1604 switch (process_acks(negotiator, &reader, &common)) {
1605 case READY:
1606 /*
1607 * Don't check for response delimiter; get_pack() will
1608 * read the rest of this response.
1609 */
1610 state = FETCH_GET_PACK;
1611 break;
1612 case COMMON_FOUND:
1613 in_vain = 0;
1614 seen_ack = 1;
1615 /* fallthrough */
1616 case NO_COMMON_FOUND:
1617 do_check_stateless_delimiter(args, &reader);
1618 state = FETCH_SEND_REQUEST;
1619 break;
1620 }
1621 break;
1622 case FETCH_GET_PACK:
1623 trace2_region_leave("fetch-pack",
1624 "negotiation_v2",
1625 the_repository);
1626 /* Check for shallow-info section */
1627 if (process_section_header(&reader, "shallow-info", 1))
1628 receive_shallow_info(args, &reader, shallows, si);
1629
1630 if (process_section_header(&reader, "wanted-refs", 1))
1631 receive_wanted_refs(&reader, sought, nr_sought);
1632
1633 /* get the pack(s) */
1634 if (process_section_header(&reader, "packfile-uris", 1))
1635 receive_packfile_uris(&reader, &packfile_uris);
1636 process_section_header(&reader, "packfile", 0);
1637 if (get_pack(args, fd, pack_lockfiles,
1638 !packfile_uris.nr, sought, nr_sought))
1639 die(_("git fetch-pack: fetch failed."));
1640 do_check_stateless_delimiter(args, &reader);
1641
1642 state = FETCH_DONE;
1643 break;
1644 case FETCH_DONE:
1645 continue;
1646 }
1647 }
1648
1649 for (i = 0; i < packfile_uris.nr; i++) {
1650 struct child_process cmd = CHILD_PROCESS_INIT;
1651 char packname[GIT_MAX_HEXSZ + 1];
1652 const char *uri = packfile_uris.items[i].string +
1653 the_hash_algo->hexsz + 1;
1654
1655 argv_array_push(&cmd.args, "http-fetch");
1656 argv_array_pushf(&cmd.args, "--packfile=%.*s",
1657 (int) the_hash_algo->hexsz,
1658 packfile_uris.items[i].string);
1659 argv_array_push(&cmd.args, uri);
1660 cmd.git_cmd = 1;
1661 cmd.no_stdin = 1;
1662 cmd.out = -1;
1663 if (start_command(&cmd))
1664 die("fetch-pack: unable to spawn http-fetch");
1665
1666 if (read_in_full(cmd.out, packname, 5) < 0 ||
1667 memcmp(packname, "keep\t", 5))
1668 die("fetch-pack: expected keep then TAB at start of http-fetch output");
1669
1670 if (read_in_full(cmd.out, packname,
1671 the_hash_algo->hexsz + 1) < 0 ||
1672 packname[the_hash_algo->hexsz] != '\n')
1673 die("fetch-pack: expected hash then LF at end of http-fetch output");
1674
1675 packname[the_hash_algo->hexsz] = '\0';
1676
1677 close(cmd.out);
1678
1679 if (finish_command(&cmd))
1680 die("fetch-pack: unable to finish http-fetch");
1681
1682 if (memcmp(packfile_uris.items[i].string, packname,
1683 the_hash_algo->hexsz))
1684 die("fetch-pack: pack downloaded from %s does not match expected hash %.*s",
1685 uri, (int) the_hash_algo->hexsz,
1686 packfile_uris.items[i].string);
1687
1688 string_list_append_nodup(pack_lockfiles,
1689 xstrfmt("%s/pack/pack-%s.keep",
1690 get_object_directory(),
1691 packname));
1692 }
1693 string_list_clear(&packfile_uris, 0);
1694
1695 if (negotiator)
1696 negotiator->release(negotiator);
1697
1698 oidset_clear(&common);
1699 return ref;
1700 }
1701
1702 static int fetch_pack_config_cb(const char *var, const char *value, void *cb)
1703 {
1704 if (strcmp(var, "fetch.fsck.skiplist") == 0) {
1705 const char *path;
1706
1707 if (git_config_pathname(&path, var, value))
1708 return 1;
1709 strbuf_addf(&fsck_msg_types, "%cskiplist=%s",
1710 fsck_msg_types.len ? ',' : '=', path);
1711 free((char *)path);
1712 return 0;
1713 }
1714
1715 if (skip_prefix(var, "fetch.fsck.", &var)) {
1716 if (is_valid_msg_type(var, value))
1717 strbuf_addf(&fsck_msg_types, "%c%s=%s",
1718 fsck_msg_types.len ? ',' : '=', var, value);
1719 else
1720 warning("Skipping unknown msg id '%s'", var);
1721 return 0;
1722 }
1723
1724 return git_default_config(var, value, cb);
1725 }
1726
1727 static void fetch_pack_config(void)
1728 {
1729 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1730 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1731 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1732 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1733 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1734 if (!uri_protocols.nr) {
1735 char *str;
1736
1737 if (!git_config_get_string("fetch.uriprotocols", &str) && str) {
1738 string_list_split(&uri_protocols, str, ',', -1);
1739 free(str);
1740 }
1741 }
1742
1743 git_config(fetch_pack_config_cb, NULL);
1744 }
1745
1746 static void fetch_pack_setup(void)
1747 {
1748 static int did_setup;
1749 if (did_setup)
1750 return;
1751 fetch_pack_config();
1752 if (0 <= transfer_unpack_limit)
1753 unpack_limit = transfer_unpack_limit;
1754 else if (0 <= fetch_unpack_limit)
1755 unpack_limit = fetch_unpack_limit;
1756 did_setup = 1;
1757 }
1758
1759 static int remove_duplicates_in_refs(struct ref **ref, int nr)
1760 {
1761 struct string_list names = STRING_LIST_INIT_NODUP;
1762 int src, dst;
1763
1764 for (src = dst = 0; src < nr; src++) {
1765 struct string_list_item *item;
1766 item = string_list_insert(&names, ref[src]->name);
1767 if (item->util)
1768 continue; /* already have it */
1769 item->util = ref[src];
1770 if (src != dst)
1771 ref[dst] = ref[src];
1772 dst++;
1773 }
1774 for (src = dst; src < nr; src++)
1775 ref[src] = NULL;
1776 string_list_clear(&names, 0);
1777 return dst;
1778 }
1779
1780 static void update_shallow(struct fetch_pack_args *args,
1781 struct ref **sought, int nr_sought,
1782 struct shallow_info *si)
1783 {
1784 struct oid_array ref = OID_ARRAY_INIT;
1785 int *status;
1786 int i;
1787
1788 if (args->deepen && alternate_shallow_file) {
1789 if (*alternate_shallow_file == '\0') { /* --unshallow */
1790 unlink_or_warn(git_path_shallow(the_repository));
1791 rollback_shallow_file(the_repository, &shallow_lock);
1792 } else
1793 commit_shallow_file(the_repository, &shallow_lock);
1794 alternate_shallow_file = NULL;
1795 return;
1796 }
1797
1798 if (!si->shallow || !si->shallow->nr)
1799 return;
1800
1801 if (args->cloning) {
1802 /*
1803 * remote is shallow, but this is a clone, there are
1804 * no objects in repo to worry about. Accept any
1805 * shallow points that exist in the pack (iow in repo
1806 * after get_pack() and reprepare_packed_git())
1807 */
1808 struct oid_array extra = OID_ARRAY_INIT;
1809 struct object_id *oid = si->shallow->oid;
1810 for (i = 0; i < si->shallow->nr; i++)
1811 if (has_object_file(&oid[i]))
1812 oid_array_append(&extra, &oid[i]);
1813 if (extra.nr) {
1814 setup_alternate_shallow(&shallow_lock,
1815 &alternate_shallow_file,
1816 &extra);
1817 commit_shallow_file(the_repository, &shallow_lock);
1818 alternate_shallow_file = NULL;
1819 }
1820 oid_array_clear(&extra);
1821 return;
1822 }
1823
1824 if (!si->nr_ours && !si->nr_theirs)
1825 return;
1826
1827 remove_nonexistent_theirs_shallow(si);
1828 if (!si->nr_ours && !si->nr_theirs)
1829 return;
1830 for (i = 0; i < nr_sought; i++)
1831 oid_array_append(&ref, &sought[i]->old_oid);
1832 si->ref = &ref;
1833
1834 if (args->update_shallow) {
1835 /*
1836 * remote is also shallow, .git/shallow may be updated
1837 * so all refs can be accepted. Make sure we only add
1838 * shallow roots that are actually reachable from new
1839 * refs.
1840 */
1841 struct oid_array extra = OID_ARRAY_INIT;
1842 struct object_id *oid = si->shallow->oid;
1843 assign_shallow_commits_to_refs(si, NULL, NULL);
1844 if (!si->nr_ours && !si->nr_theirs) {
1845 oid_array_clear(&ref);
1846 return;
1847 }
1848 for (i = 0; i < si->nr_ours; i++)
1849 oid_array_append(&extra, &oid[si->ours[i]]);
1850 for (i = 0; i < si->nr_theirs; i++)
1851 oid_array_append(&extra, &oid[si->theirs[i]]);
1852 setup_alternate_shallow(&shallow_lock,
1853 &alternate_shallow_file,
1854 &extra);
1855 commit_shallow_file(the_repository, &shallow_lock);
1856 oid_array_clear(&extra);
1857 oid_array_clear(&ref);
1858 alternate_shallow_file = NULL;
1859 return;
1860 }
1861
1862 /*
1863 * remote is also shallow, check what ref is safe to update
1864 * without updating .git/shallow
1865 */
1866 status = xcalloc(nr_sought, sizeof(*status));
1867 assign_shallow_commits_to_refs(si, NULL, status);
1868 if (si->nr_ours || si->nr_theirs) {
1869 for (i = 0; i < nr_sought; i++)
1870 if (status[i])
1871 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1872 }
1873 free(status);
1874 oid_array_clear(&ref);
1875 }
1876
1877 static int iterate_ref_map(void *cb_data, struct object_id *oid)
1878 {
1879 struct ref **rm = cb_data;
1880 struct ref *ref = *rm;
1881
1882 if (!ref)
1883 return -1; /* end of the list */
1884 *rm = ref->next;
1885 oidcpy(oid, &ref->old_oid);
1886 return 0;
1887 }
1888
1889 struct ref *fetch_pack(struct fetch_pack_args *args,
1890 int fd[],
1891 const struct ref *ref,
1892 struct ref **sought, int nr_sought,
1893 struct oid_array *shallow,
1894 struct string_list *pack_lockfiles,
1895 enum protocol_version version)
1896 {
1897 struct ref *ref_cpy;
1898 struct shallow_info si;
1899 struct oid_array shallows_scratch = OID_ARRAY_INIT;
1900
1901 fetch_pack_setup();
1902 if (nr_sought)
1903 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1904
1905 if (args->no_dependents && !args->filter_options.choice) {
1906 /*
1907 * The protocol does not support requesting that only the
1908 * wanted objects be sent, so approximate this by setting a
1909 * "blob:none" filter if no filter is already set. This works
1910 * for all object types: note that wanted blobs will still be
1911 * sent because they are directly specified as a "want".
1912 *
1913 * NEEDSWORK: Add an option in the protocol to request that
1914 * only the wanted objects be sent, and implement it.
1915 */
1916 parse_list_objects_filter(&args->filter_options, "blob:none");
1917 }
1918
1919 if (version != protocol_v2 && !ref) {
1920 packet_flush(fd[1]);
1921 die(_("no matching remote head"));
1922 }
1923 if (version == protocol_v2) {
1924 if (shallow->nr)
1925 BUG("Protocol V2 does not provide shallows at this point in the fetch");
1926 memset(&si, 0, sizeof(si));
1927 ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1928 &shallows_scratch, &si,
1929 pack_lockfiles);
1930 } else {
1931 prepare_shallow_info(&si, shallow);
1932 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1933 &si, pack_lockfiles);
1934 }
1935 reprepare_packed_git(the_repository);
1936
1937 if (!args->cloning && args->deepen) {
1938 struct check_connected_options opt = CHECK_CONNECTED_INIT;
1939 struct ref *iterator = ref_cpy;
1940 opt.shallow_file = alternate_shallow_file;
1941 if (args->deepen)
1942 opt.is_deepening_fetch = 1;
1943 if (check_connected(iterate_ref_map, &iterator, &opt)) {
1944 error(_("remote did not send all necessary objects"));
1945 free_refs(ref_cpy);
1946 ref_cpy = NULL;
1947 rollback_shallow_file(the_repository, &shallow_lock);
1948 goto cleanup;
1949 }
1950 args->connectivity_checked = 1;
1951 }
1952
1953 update_shallow(args, sought, nr_sought, &si);
1954 cleanup:
1955 clear_shallow_info(&si);
1956 oid_array_clear(&shallows_scratch);
1957 return ref_cpy;
1958 }
1959
1960 int report_unmatched_refs(struct ref **sought, int nr_sought)
1961 {
1962 int i, ret = 0;
1963
1964 for (i = 0; i < nr_sought; i++) {
1965 if (!sought[i])
1966 continue;
1967 switch (sought[i]->match_status) {
1968 case REF_MATCHED:
1969 continue;
1970 case REF_NOT_MATCHED:
1971 error(_("no such remote ref %s"), sought[i]->name);
1972 break;
1973 case REF_UNADVERTISED_NOT_ALLOWED:
1974 error(_("Server does not allow request for unadvertised object %s"),
1975 sought[i]->name);
1976 break;
1977 }
1978 ret = 1;
1979 }
1980 return ret;
1981 }
git for Debian