2 #include "repository.h"
12 #include "fetch-pack.h"
14 #include "run-command.h"
16 #include "transport.h"
18 #include "prio-queue.h"
19 #include "sha1-array.h"
22 #include "object-store.h"
24 static int transfer_unpack_limit
= -1;
25 static int fetch_unpack_limit
= -1;
26 static int unpack_limit
= 100;
27 static int prefer_ofs_delta
= 1;
29 static int deepen_since_ok
;
30 static int deepen_not_ok
;
31 static int fetch_fsck_objects
= -1;
32 static int transfer_fsck_objects
= -1;
33 static int agent_supported
;
34 static int server_supports_filtering
;
35 static struct lock_file shallow_lock
;
36 static const char *alternate_shallow_file
;
38 /* Remember to update object flag allocation in object.h */
39 #define COMPLETE (1U << 0)
40 #define COMMON (1U << 1)
41 #define COMMON_REF (1U << 2)
42 #define SEEN (1U << 3)
43 #define POPPED (1U << 4)
44 #define ALTERNATE (1U << 5)
49 * After sending this many "have"s if we do not get any new ACK , we
50 * give up traversing our history.
52 #define MAX_IN_VAIN 256
54 static struct prio_queue rev_list
= { compare_commits_by_commit_date
};
55 static int non_common_revs
, multi_ack
, use_sideband
;
56 /* Allow specifying sha1 if it is a ref tip. */
57 #define ALLOW_TIP_SHA1 01
58 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
59 #define ALLOW_REACHABLE_SHA1 02
60 static unsigned int allow_unadvertised_object_request
;
62 __attribute__((format (printf
, 2, 3)))
63 static inline void print_verbose(const struct fetch_pack_args
*args
,
71 va_start(params
, fmt
);
72 vfprintf(stderr
, fmt
, params
);
77 struct alternate_object_cache
{
78 struct object
**items
;
82 static void cache_one_alternate(const char *refname
,
83 const struct object_id
*oid
,
86 struct alternate_object_cache
*cache
= vcache
;
87 struct object
*obj
= parse_object(the_repository
, oid
);
89 if (!obj
|| (obj
->flags
& ALTERNATE
))
92 obj
->flags
|= ALTERNATE
;
93 ALLOC_GROW(cache
->items
, cache
->nr
+ 1, cache
->alloc
);
94 cache
->items
[cache
->nr
++] = obj
;
97 static void for_each_cached_alternate(void (*cb
)(struct object
*))
99 static int initialized
;
100 static struct alternate_object_cache cache
;
104 for_each_alternate_ref(cache_one_alternate
, &cache
);
108 for (i
= 0; i
< cache
.nr
; i
++)
112 static void rev_list_push(struct commit
*commit
, int mark
)
114 if (!(commit
->object
.flags
& mark
)) {
115 commit
->object
.flags
|= mark
;
117 if (parse_commit(commit
))
120 prio_queue_put(&rev_list
, commit
);
122 if (!(commit
->object
.flags
& COMMON
))
127 static int rev_list_insert_ref(const char *refname
, const struct object_id
*oid
)
129 struct object
*o
= deref_tag(parse_object(the_repository
, oid
),
132 if (o
&& o
->type
== OBJ_COMMIT
)
133 rev_list_push((struct commit
*)o
, SEEN
);
138 static int rev_list_insert_ref_oid(const char *refname
, const struct object_id
*oid
,
139 int flag
, void *cb_data
)
141 return rev_list_insert_ref(refname
, oid
);
144 static int clear_marks(const char *refname
, const struct object_id
*oid
,
145 int flag
, void *cb_data
)
147 struct object
*o
= deref_tag(parse_object(the_repository
, oid
),
150 if (o
&& o
->type
== OBJ_COMMIT
)
151 clear_commit_marks((struct commit
*)o
,
152 COMMON
| COMMON_REF
| SEEN
| POPPED
);
157 This function marks a rev and its ancestors as common.
158 In some cases, it is desirable to mark only the ancestors (for example
159 when only the server does not yet know that they are common).
162 static void mark_common(struct commit
*commit
,
163 int ancestors_only
, int dont_parse
)
165 if (commit
!= NULL
&& !(commit
->object
.flags
& COMMON
)) {
166 struct object
*o
= (struct object
*)commit
;
171 if (!(o
->flags
& SEEN
))
172 rev_list_push(commit
, SEEN
);
174 struct commit_list
*parents
;
176 if (!ancestors_only
&& !(o
->flags
& POPPED
))
178 if (!o
->parsed
&& !dont_parse
)
179 if (parse_commit(commit
))
182 for (parents
= commit
->parents
;
184 parents
= parents
->next
)
185 mark_common(parents
->item
, 0, dont_parse
);
191 Get the next rev to send, ignoring the common.
194 static const struct object_id
*get_rev(void)
196 struct commit
*commit
= NULL
;
198 while (commit
== NULL
) {
200 struct commit_list
*parents
;
202 if (rev_list
.nr
== 0 || non_common_revs
== 0)
205 commit
= prio_queue_get(&rev_list
);
206 parse_commit(commit
);
207 parents
= commit
->parents
;
209 commit
->object
.flags
|= POPPED
;
210 if (!(commit
->object
.flags
& COMMON
))
213 if (commit
->object
.flags
& COMMON
) {
214 /* do not send "have", and ignore ancestors */
216 mark
= COMMON
| SEEN
;
217 } else if (commit
->object
.flags
& COMMON_REF
)
218 /* send "have", and ignore ancestors */
219 mark
= COMMON
| SEEN
;
221 /* send "have", also for its ancestors */
225 if (!(parents
->item
->object
.flags
& SEEN
))
226 rev_list_push(parents
->item
, mark
);
228 mark_common(parents
->item
, 1, 0);
229 parents
= parents
->next
;
233 return &commit
->object
.oid
;
244 static void consume_shallow_list(struct fetch_pack_args
*args
, int fd
)
246 if (args
->stateless_rpc
&& args
->deepen
) {
247 /* If we sent a depth we will get back "duplicate"
248 * shallow and unshallow commands every time there
249 * is a block of have lines exchanged.
252 while ((line
= packet_read_line(fd
, NULL
))) {
253 if (starts_with(line
, "shallow "))
255 if (starts_with(line
, "unshallow "))
257 die(_("git fetch-pack: expected shallow list"));
262 static enum ack_type
get_ack(int fd
, struct object_id
*result_oid
)
265 char *line
= packet_read_line(fd
, &len
);
269 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
270 if (!strcmp(line
, "NAK"))
272 if (skip_prefix(line
, "ACK ", &arg
)) {
273 if (!get_oid_hex(arg
, result_oid
)) {
278 if (strstr(arg
, "continue"))
280 if (strstr(arg
, "common"))
282 if (strstr(arg
, "ready"))
287 if (skip_prefix(line
, "ERR ", &arg
))
288 die(_("remote error: %s"), arg
);
289 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line
);
292 static void send_request(struct fetch_pack_args
*args
,
293 int fd
, struct strbuf
*buf
)
295 if (args
->stateless_rpc
) {
296 send_sideband(fd
, -1, buf
->buf
, buf
->len
, LARGE_PACKET_MAX
);
299 write_or_die(fd
, buf
->buf
, buf
->len
);
302 static void insert_one_alternate_object(struct object
*obj
)
304 rev_list_insert_ref(NULL
, &obj
->oid
);
307 #define INITIAL_FLUSH 16
308 #define PIPESAFE_FLUSH 32
309 #define LARGE_FLUSH 16384
311 static int next_flush(int stateless_rpc
, int count
)
314 if (count
< LARGE_FLUSH
)
317 count
= count
* 11 / 10;
319 if (count
< PIPESAFE_FLUSH
)
322 count
+= PIPESAFE_FLUSH
;
327 static int find_common(struct fetch_pack_args
*args
,
328 int fd
[2], struct object_id
*result_oid
,
332 int count
= 0, flushes
= 0, flush_at
= INITIAL_FLUSH
, retval
;
333 const struct object_id
*oid
;
334 unsigned in_vain
= 0;
335 int got_continue
= 0;
337 struct strbuf req_buf
= STRBUF_INIT
;
338 size_t state_len
= 0;
340 if (args
->stateless_rpc
&& multi_ack
== 1)
341 die(_("--stateless-rpc requires multi_ack_detailed"));
343 for_each_ref(clear_marks
, NULL
);
346 for_each_ref(rev_list_insert_ref_oid
, NULL
);
347 for_each_cached_alternate(insert_one_alternate_object
);
350 for ( ; refs
; refs
= refs
->next
) {
351 struct object_id
*remote
= &refs
->old_oid
;
352 const char *remote_hex
;
356 * If that object is complete (i.e. it is an ancestor of a
357 * local ref), we tell them we have it but do not have to
358 * tell them about its ancestors, which they already know
361 * We use lookup_object here because we are only
362 * interested in the case we *know* the object is
363 * reachable and we have already scanned it.
365 if (((o
= lookup_object(the_repository
, remote
->hash
)) != NULL
) &&
366 (o
->flags
& COMPLETE
)) {
370 remote_hex
= oid_to_hex(remote
);
372 struct strbuf c
= STRBUF_INIT
;
373 if (multi_ack
== 2) strbuf_addstr(&c
, " multi_ack_detailed");
374 if (multi_ack
== 1) strbuf_addstr(&c
, " multi_ack");
375 if (no_done
) strbuf_addstr(&c
, " no-done");
376 if (use_sideband
== 2) strbuf_addstr(&c
, " side-band-64k");
377 if (use_sideband
== 1) strbuf_addstr(&c
, " side-band");
378 if (args
->deepen_relative
) strbuf_addstr(&c
, " deepen-relative");
379 if (args
->use_thin_pack
) strbuf_addstr(&c
, " thin-pack");
380 if (args
->no_progress
) strbuf_addstr(&c
, " no-progress");
381 if (args
->include_tag
) strbuf_addstr(&c
, " include-tag");
382 if (prefer_ofs_delta
) strbuf_addstr(&c
, " ofs-delta");
383 if (deepen_since_ok
) strbuf_addstr(&c
, " deepen-since");
384 if (deepen_not_ok
) strbuf_addstr(&c
, " deepen-not");
385 if (agent_supported
) strbuf_addf(&c
, " agent=%s",
386 git_user_agent_sanitized());
387 if (args
->filter_options
.choice
)
388 strbuf_addstr(&c
, " filter");
389 packet_buf_write(&req_buf
, "want %s%s\n", remote_hex
, c
.buf
);
392 packet_buf_write(&req_buf
, "want %s\n", remote_hex
);
397 strbuf_release(&req_buf
);
402 if (is_repository_shallow(the_repository
))
403 write_shallow_commits(&req_buf
, 1, NULL
);
405 packet_buf_write(&req_buf
, "deepen %d", args
->depth
);
406 if (args
->deepen_since
) {
407 timestamp_t max_age
= approxidate(args
->deepen_since
);
408 packet_buf_write(&req_buf
, "deepen-since %"PRItime
, max_age
);
410 if (args
->deepen_not
) {
412 for (i
= 0; i
< args
->deepen_not
->nr
; i
++) {
413 struct string_list_item
*s
= args
->deepen_not
->items
+ i
;
414 packet_buf_write(&req_buf
, "deepen-not %s", s
->string
);
417 if (server_supports_filtering
&& args
->filter_options
.choice
)
418 packet_buf_write(&req_buf
, "filter %s",
419 args
->filter_options
.filter_spec
);
420 packet_buf_flush(&req_buf
);
421 state_len
= req_buf
.len
;
426 struct object_id oid
;
428 send_request(args
, fd
[1], &req_buf
);
429 while ((line
= packet_read_line(fd
[0], NULL
))) {
430 if (skip_prefix(line
, "shallow ", &arg
)) {
431 if (get_oid_hex(arg
, &oid
))
432 die(_("invalid shallow line: %s"), line
);
433 register_shallow(the_repository
, &oid
);
436 if (skip_prefix(line
, "unshallow ", &arg
)) {
437 if (get_oid_hex(arg
, &oid
))
438 die(_("invalid unshallow line: %s"), line
);
439 if (!lookup_object(the_repository
, oid
.hash
))
440 die(_("object not found: %s"), line
);
441 /* make sure that it is parsed as shallow */
442 if (!parse_object(the_repository
, &oid
))
443 die(_("error in object: %s"), line
);
444 if (unregister_shallow(&oid
))
445 die(_("no shallow found: %s"), line
);
448 die(_("expected shallow/unshallow, got %s"), line
);
450 } else if (!args
->stateless_rpc
)
451 send_request(args
, fd
[1], &req_buf
);
453 if (!args
->stateless_rpc
) {
454 /* If we aren't using the stateless-rpc interface
455 * we don't need to retain the headers.
457 strbuf_setlen(&req_buf
, 0);
463 if (args
->no_dependents
)
465 while ((oid
= get_rev())) {
466 packet_buf_write(&req_buf
, "have %s\n", oid_to_hex(oid
));
467 print_verbose(args
, "have %s", oid_to_hex(oid
));
469 if (flush_at
<= ++count
) {
472 packet_buf_flush(&req_buf
);
473 send_request(args
, fd
[1], &req_buf
);
474 strbuf_setlen(&req_buf
, state_len
);
476 flush_at
= next_flush(args
->stateless_rpc
, count
);
479 * We keep one window "ahead" of the other side, and
480 * will wait for an ACK only on the next one
482 if (!args
->stateless_rpc
&& count
== INITIAL_FLUSH
)
485 consume_shallow_list(args
, fd
[0]);
487 ack
= get_ack(fd
[0], result_oid
);
489 print_verbose(args
, _("got %s %d %s"), "ack",
490 ack
, oid_to_hex(result_oid
));
500 struct commit
*commit
=
501 lookup_commit(the_repository
,
504 die(_("invalid commit %s"), oid_to_hex(result_oid
));
505 if (args
->stateless_rpc
507 && !(commit
->object
.flags
& COMMON
)) {
508 /* We need to replay the have for this object
509 * on the next RPC request so the peer knows
510 * it is in common with us.
512 const char *hex
= oid_to_hex(result_oid
);
513 packet_buf_write(&req_buf
, "have %s\n", hex
);
514 state_len
= req_buf
.len
;
516 * Reset in_vain because an ack
517 * for this commit has not been
521 } else if (!args
->stateless_rpc
522 || ack
!= ACK_common
)
524 mark_common(commit
, 0, 1);
527 if (ack
== ACK_ready
) {
528 clear_prio_queue(&rev_list
);
536 if (got_continue
&& MAX_IN_VAIN
< in_vain
) {
537 print_verbose(args
, _("giving up"));
543 if (!got_ready
|| !no_done
) {
544 packet_buf_write(&req_buf
, "done\n");
545 send_request(args
, fd
[1], &req_buf
);
547 print_verbose(args
, _("done"));
552 strbuf_release(&req_buf
);
554 if (!got_ready
|| !no_done
)
555 consume_shallow_list(args
, fd
[0]);
556 while (flushes
|| multi_ack
) {
557 int ack
= get_ack(fd
[0], result_oid
);
559 print_verbose(args
, _("got %s (%d) %s"), "ack",
560 ack
, oid_to_hex(result_oid
));
568 /* it is no error to fetch into a completely empty repo */
569 return count
? retval
: 0;
572 static struct commit_list
*complete
;
574 static int mark_complete(const struct object_id
*oid
)
576 struct object
*o
= parse_object(the_repository
, oid
);
578 while (o
&& o
->type
== OBJ_TAG
) {
579 struct tag
*t
= (struct tag
*) o
;
581 break; /* broken repository */
582 o
->flags
|= COMPLETE
;
583 o
= parse_object(the_repository
, &t
->tagged
->oid
);
585 if (o
&& o
->type
== OBJ_COMMIT
) {
586 struct commit
*commit
= (struct commit
*)o
;
587 if (!(commit
->object
.flags
& COMPLETE
)) {
588 commit
->object
.flags
|= COMPLETE
;
589 commit_list_insert(commit
, &complete
);
595 static int mark_complete_oid(const char *refname
, const struct object_id
*oid
,
596 int flag
, void *cb_data
)
598 return mark_complete(oid
);
601 static void mark_recent_complete_commits(struct fetch_pack_args
*args
,
604 while (complete
&& cutoff
<= complete
->item
->date
) {
605 print_verbose(args
, _("Marking %s as complete"),
606 oid_to_hex(&complete
->item
->object
.oid
));
607 pop_most_recent_commit(&complete
, COMPLETE
);
611 static void add_refs_to_oidset(struct oidset
*oids
, struct ref
*refs
)
613 for (; refs
; refs
= refs
->next
)
614 oidset_insert(oids
, &refs
->old_oid
);
617 static int tip_oids_contain(struct oidset
*tip_oids
,
618 struct ref
*unmatched
, struct ref
*newlist
,
619 const struct object_id
*id
)
622 * Note that this only looks at the ref lists the first time it's
623 * called. This works out in filter_refs() because even though it may
624 * add to "newlist" between calls, the additions will always be for
625 * oids that are already in the set.
627 if (!tip_oids
->map
.map
.tablesize
) {
628 add_refs_to_oidset(tip_oids
, unmatched
);
629 add_refs_to_oidset(tip_oids
, newlist
);
631 return oidset_contains(tip_oids
, id
);
634 static void filter_refs(struct fetch_pack_args
*args
,
636 struct ref
**sought
, int nr_sought
)
638 struct ref
*newlist
= NULL
;
639 struct ref
**newtail
= &newlist
;
640 struct ref
*unmatched
= NULL
;
641 struct ref
*ref
, *next
;
642 struct oidset tip_oids
= OIDSET_INIT
;
646 for (ref
= *refs
; ref
; ref
= next
) {
650 if (starts_with(ref
->name
, "refs/") &&
651 check_refname_format(ref
->name
, 0))
654 while (i
< nr_sought
) {
655 int cmp
= strcmp(ref
->name
, sought
[i
]->name
);
657 break; /* definitely do not have it */
659 keep
= 1; /* definitely have it */
660 sought
[i
]->match_status
= REF_MATCHED
;
665 if (!keep
&& args
->fetch_all
&&
666 (!args
->deepen
|| !starts_with(ref
->name
, "refs/tags/")))
673 newtail
= &ref
->next
;
675 ref
->next
= unmatched
;
680 /* Append unmatched requests to the list */
681 for (i
= 0; i
< nr_sought
; i
++) {
682 struct object_id oid
;
686 if (ref
->match_status
!= REF_NOT_MATCHED
)
688 if (parse_oid_hex(ref
->name
, &oid
, &p
) ||
690 oidcmp(&oid
, &ref
->old_oid
))
693 if ((allow_unadvertised_object_request
&
694 (ALLOW_TIP_SHA1
| ALLOW_REACHABLE_SHA1
)) ||
695 tip_oids_contain(&tip_oids
, unmatched
, newlist
,
697 ref
->match_status
= REF_MATCHED
;
698 *newtail
= copy_ref(ref
);
699 newtail
= &(*newtail
)->next
;
701 ref
->match_status
= REF_UNADVERTISED_NOT_ALLOWED
;
705 oidset_clear(&tip_oids
);
706 for (ref
= unmatched
; ref
; ref
= next
) {
714 static void mark_alternate_complete(struct object
*obj
)
716 mark_complete(&obj
->oid
);
719 struct loose_object_iter
{
720 struct oidset
*loose_object_set
;
725 * If the number of refs is not larger than the number of loose objects,
726 * this function stops inserting.
728 static int add_loose_objects_to_set(const struct object_id
*oid
,
732 struct loose_object_iter
*iter
= data
;
733 oidset_insert(iter
->loose_object_set
, oid
);
734 if (iter
->refs
== NULL
)
737 iter
->refs
= iter
->refs
->next
;
741 static int everything_local(struct fetch_pack_args
*args
,
743 struct ref
**sought
, int nr_sought
)
747 int old_save_commit_buffer
= save_commit_buffer
;
748 timestamp_t cutoff
= 0;
749 struct oidset loose_oid_set
= OIDSET_INIT
;
751 struct loose_object_iter iter
= {&loose_oid_set
, *refs
};
753 /* Enumerate all loose objects or know refs are not so many. */
754 use_oidset
= !for_each_loose_object(add_loose_objects_to_set
,
757 save_commit_buffer
= 0;
759 for (ref
= *refs
; ref
; ref
= ref
->next
) {
761 unsigned int flags
= OBJECT_INFO_QUICK
;
764 !oidset_contains(&loose_oid_set
, &ref
->old_oid
)) {
766 * I know this does not exist in the loose form,
767 * so check if it exists in a non-loose form.
769 flags
|= OBJECT_INFO_IGNORE_LOOSE
;
772 if (!has_object_file_with_flags(&ref
->old_oid
, flags
))
774 o
= parse_object(the_repository
, &ref
->old_oid
);
778 /* We already have it -- which may mean that we were
779 * in sync with the other side at some time after
780 * that (it is OK if we guess wrong here).
782 if (o
->type
== OBJ_COMMIT
) {
783 struct commit
*commit
= (struct commit
*)o
;
784 if (!cutoff
|| cutoff
< commit
->date
)
785 cutoff
= commit
->date
;
789 oidset_clear(&loose_oid_set
);
791 if (!args
->no_dependents
) {
793 for_each_ref(mark_complete_oid
, NULL
);
794 for_each_cached_alternate(mark_alternate_complete
);
795 commit_list_sort_by_date(&complete
);
797 mark_recent_complete_commits(args
, cutoff
);
801 * Mark all complete remote refs as common refs.
802 * Don't mark them common yet; the server has to be told so first.
804 for (ref
= *refs
; ref
; ref
= ref
->next
) {
805 struct object
*o
= deref_tag(lookup_object(the_repository
,
809 if (!o
|| o
->type
!= OBJ_COMMIT
|| !(o
->flags
& COMPLETE
))
812 if (!(o
->flags
& SEEN
)) {
813 rev_list_push((struct commit
*)o
, COMMON_REF
| SEEN
);
815 mark_common((struct commit
*)o
, 1, 1);
820 filter_refs(args
, refs
, sought
, nr_sought
);
822 for (retval
= 1, ref
= *refs
; ref
; ref
= ref
->next
) {
823 const struct object_id
*remote
= &ref
->old_oid
;
826 o
= lookup_object(the_repository
, remote
->hash
);
827 if (!o
|| !(o
->flags
& COMPLETE
)) {
829 print_verbose(args
, "want %s (%s)", oid_to_hex(remote
),
833 print_verbose(args
, _("already have %s (%s)"), oid_to_hex(remote
),
837 save_commit_buffer
= old_save_commit_buffer
;
842 static int sideband_demux(int in
, int out
, void *data
)
847 ret
= recv_sideband("fetch-pack", xd
[0], out
);
852 static int get_pack(struct fetch_pack_args
*args
,
853 int xd
[2], char **pack_lockfile
)
856 int do_keep
= args
->keep_pack
;
857 const char *cmd_name
;
858 struct pack_header header
;
860 struct child_process cmd
= CHILD_PROCESS_INIT
;
863 memset(&demux
, 0, sizeof(demux
));
865 /* xd[] is talking with upload-pack; subprocess reads from
866 * xd[0], spits out band#2 to stderr, and feeds us band#1
867 * through demux->out.
869 demux
.proc
= sideband_demux
;
872 demux
.isolate_sigpipe
= 1;
873 if (start_async(&demux
))
874 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
879 if (!args
->keep_pack
&& unpack_limit
) {
881 if (read_pack_header(demux
.out
, &header
))
882 die(_("protocol error: bad pack header"));
884 if (ntohl(header
.hdr_entries
) < unpack_limit
)
890 if (alternate_shallow_file
) {
891 argv_array_push(&cmd
.args
, "--shallow-file");
892 argv_array_push(&cmd
.args
, alternate_shallow_file
);
895 if (do_keep
|| args
->from_promisor
) {
898 cmd_name
= "index-pack";
899 argv_array_push(&cmd
.args
, cmd_name
);
900 argv_array_push(&cmd
.args
, "--stdin");
901 if (!args
->quiet
&& !args
->no_progress
)
902 argv_array_push(&cmd
.args
, "-v");
903 if (args
->use_thin_pack
)
904 argv_array_push(&cmd
.args
, "--fix-thin");
905 if (do_keep
&& (args
->lock_pack
|| unpack_limit
)) {
906 char hostname
[HOST_NAME_MAX
+ 1];
907 if (xgethostname(hostname
, sizeof(hostname
)))
908 xsnprintf(hostname
, sizeof(hostname
), "localhost");
909 argv_array_pushf(&cmd
.args
,
910 "--keep=fetch-pack %"PRIuMAX
" on %s",
911 (uintmax_t)getpid(), hostname
);
913 if (args
->check_self_contained_and_connected
)
914 argv_array_push(&cmd
.args
, "--check-self-contained-and-connected");
915 if (args
->from_promisor
)
916 argv_array_push(&cmd
.args
, "--promisor");
919 cmd_name
= "unpack-objects";
920 argv_array_push(&cmd
.args
, cmd_name
);
921 if (args
->quiet
|| args
->no_progress
)
922 argv_array_push(&cmd
.args
, "-q");
923 args
->check_self_contained_and_connected
= 0;
927 argv_array_pushf(&cmd
.args
, "--pack_header=%"PRIu32
",%"PRIu32
,
928 ntohl(header
.hdr_version
),
929 ntohl(header
.hdr_entries
));
930 if (fetch_fsck_objects
>= 0
932 : transfer_fsck_objects
>= 0
933 ? transfer_fsck_objects
935 if (args
->from_promisor
)
937 * We cannot use --strict in index-pack because it
938 * checks both broken objects and links, but we only
939 * want to check for broken objects.
941 argv_array_push(&cmd
.args
, "--fsck-objects");
943 argv_array_push(&cmd
.args
, "--strict");
948 if (start_command(&cmd
))
949 die(_("fetch-pack: unable to fork off %s"), cmd_name
);
950 if (do_keep
&& pack_lockfile
) {
951 *pack_lockfile
= index_pack_lockfile(cmd
.out
);
956 /* Closed by start_command() */
959 ret
= finish_command(&cmd
);
960 if (!ret
|| (args
->check_self_contained_and_connected
&& ret
== 1))
961 args
->self_contained_and_connected
=
962 args
->check_self_contained_and_connected
&&
965 die(_("%s failed"), cmd_name
);
966 if (use_sideband
&& finish_async(&demux
))
967 die(_("error in sideband demultiplexer"));
971 static int cmp_ref_by_name(const void *a_
, const void *b_
)
973 const struct ref
*a
= *((const struct ref
**)a_
);
974 const struct ref
*b
= *((const struct ref
**)b_
);
975 return strcmp(a
->name
, b
->name
);
978 static struct ref
*do_fetch_pack(struct fetch_pack_args
*args
,
980 const struct ref
*orig_ref
,
981 struct ref
**sought
, int nr_sought
,
982 struct shallow_info
*si
,
983 char **pack_lockfile
)
985 struct ref
*ref
= copy_ref_list(orig_ref
);
986 struct object_id oid
;
987 const char *agent_feature
;
990 sort_ref_list(&ref
, ref_compare_name
);
991 QSORT(sought
, nr_sought
, cmp_ref_by_name
);
993 if ((args
->depth
> 0 || is_repository_shallow(the_repository
)) && !server_supports("shallow"))
994 die(_("Server does not support shallow clients"));
995 if (args
->depth
> 0 || args
->deepen_since
|| args
->deepen_not
)
997 if (server_supports("multi_ack_detailed")) {
998 print_verbose(args
, _("Server supports multi_ack_detailed"));
1000 if (server_supports("no-done")) {
1001 print_verbose(args
, _("Server supports no-done"));
1002 if (args
->stateless_rpc
)
1006 else if (server_supports("multi_ack")) {
1007 print_verbose(args
, _("Server supports multi_ack"));
1010 if (server_supports("side-band-64k")) {
1011 print_verbose(args
, _("Server supports side-band-64k"));
1014 else if (server_supports("side-band")) {
1015 print_verbose(args
, _("Server supports side-band"));
1018 if (server_supports("allow-tip-sha1-in-want")) {
1019 print_verbose(args
, _("Server supports allow-tip-sha1-in-want"));
1020 allow_unadvertised_object_request
|= ALLOW_TIP_SHA1
;
1022 if (server_supports("allow-reachable-sha1-in-want")) {
1023 print_verbose(args
, _("Server supports allow-reachable-sha1-in-want"));
1024 allow_unadvertised_object_request
|= ALLOW_REACHABLE_SHA1
;
1026 if (!server_supports("thin-pack"))
1027 args
->use_thin_pack
= 0;
1028 if (!server_supports("no-progress"))
1029 args
->no_progress
= 0;
1030 if (!server_supports("include-tag"))
1031 args
->include_tag
= 0;
1032 if (server_supports("ofs-delta"))
1033 print_verbose(args
, _("Server supports ofs-delta"));
1035 prefer_ofs_delta
= 0;
1037 if (server_supports("filter")) {
1038 server_supports_filtering
= 1;
1039 print_verbose(args
, _("Server supports filter"));
1040 } else if (args
->filter_options
.choice
) {
1041 warning("filtering not recognized by server, ignoring");
1044 if ((agent_feature
= server_feature_value("agent", &agent_len
))) {
1045 agent_supported
= 1;
1047 print_verbose(args
, _("Server version is %.*s"),
1048 agent_len
, agent_feature
);
1050 if (server_supports("deepen-since"))
1051 deepen_since_ok
= 1;
1052 else if (args
->deepen_since
)
1053 die(_("Server does not support --shallow-since"));
1054 if (server_supports("deepen-not"))
1056 else if (args
->deepen_not
)
1057 die(_("Server does not support --shallow-exclude"));
1058 if (!server_supports("deepen-relative") && args
->deepen_relative
)
1059 die(_("Server does not support --deepen"));
1061 if (everything_local(args
, &ref
, sought
, nr_sought
)) {
1062 packet_flush(fd
[1]);
1065 if (find_common(args
, fd
, &oid
, ref
) < 0)
1066 if (!args
->keep_pack
)
1067 /* When cloning, it is not unusual to have
1070 warning(_("no common commits"));
1072 if (args
->stateless_rpc
)
1073 packet_flush(fd
[1]);
1075 setup_alternate_shallow(&shallow_lock
, &alternate_shallow_file
,
1077 else if (si
->nr_ours
|| si
->nr_theirs
)
1078 alternate_shallow_file
= setup_temporary_shallow(si
->shallow
);
1080 alternate_shallow_file
= NULL
;
1081 if (get_pack(args
, fd
, pack_lockfile
))
1082 die(_("git fetch-pack: fetch failed."));
1088 static void add_shallow_requests(struct strbuf
*req_buf
,
1089 const struct fetch_pack_args
*args
)
1091 if (is_repository_shallow(the_repository
))
1092 write_shallow_commits(req_buf
, 1, NULL
);
1093 if (args
->depth
> 0)
1094 packet_buf_write(req_buf
, "deepen %d", args
->depth
);
1095 if (args
->deepen_since
) {
1096 timestamp_t max_age
= approxidate(args
->deepen_since
);
1097 packet_buf_write(req_buf
, "deepen-since %"PRItime
, max_age
);
1099 if (args
->deepen_not
) {
1101 for (i
= 0; i
< args
->deepen_not
->nr
; i
++) {
1102 struct string_list_item
*s
= args
->deepen_not
->items
+ i
;
1103 packet_buf_write(req_buf
, "deepen-not %s", s
->string
);
1108 static void add_wants(const struct ref
*wants
, struct strbuf
*req_buf
)
1110 for ( ; wants
; wants
= wants
->next
) {
1111 const struct object_id
*remote
= &wants
->old_oid
;
1112 const char *remote_hex
;
1116 * If that object is complete (i.e. it is an ancestor of a
1117 * local ref), we tell them we have it but do not have to
1118 * tell them about its ancestors, which they already know
1121 * We use lookup_object here because we are only
1122 * interested in the case we *know* the object is
1123 * reachable and we have already scanned it.
1125 if (((o
= lookup_object(the_repository
, remote
->hash
)) != NULL
) &&
1126 (o
->flags
& COMPLETE
)) {
1130 remote_hex
= oid_to_hex(remote
);
1131 packet_buf_write(req_buf
, "want %s\n", remote_hex
);
1135 static void add_common(struct strbuf
*req_buf
, struct oidset
*common
)
1137 struct oidset_iter iter
;
1138 const struct object_id
*oid
;
1139 oidset_iter_init(common
, &iter
);
1141 while ((oid
= oidset_iter_next(&iter
))) {
1142 packet_buf_write(req_buf
, "have %s\n", oid_to_hex(oid
));
1146 static int add_haves(struct strbuf
*req_buf
, int *haves_to_send
, int *in_vain
)
1149 int haves_added
= 0;
1150 const struct object_id
*oid
;
1152 while ((oid
= get_rev())) {
1153 packet_buf_write(req_buf
, "have %s\n", oid_to_hex(oid
));
1154 if (++haves_added
>= *haves_to_send
)
1158 *in_vain
+= haves_added
;
1159 if (!haves_added
|| *in_vain
>= MAX_IN_VAIN
) {
1161 packet_buf_write(req_buf
, "done\n");
1165 /* Increase haves to send on next round */
1166 *haves_to_send
= next_flush(1, *haves_to_send
);
1171 static int send_fetch_request(int fd_out
, const struct fetch_pack_args
*args
,
1172 const struct ref
*wants
, struct oidset
*common
,
1173 int *haves_to_send
, int *in_vain
)
1176 struct strbuf req_buf
= STRBUF_INIT
;
1178 if (server_supports_v2("fetch", 1))
1179 packet_buf_write(&req_buf
, "command=fetch");
1180 if (server_supports_v2("agent", 0))
1181 packet_buf_write(&req_buf
, "agent=%s", git_user_agent_sanitized());
1182 if (args
->server_options
&& args
->server_options
->nr
&&
1183 server_supports_v2("server-option", 1)) {
1185 for (i
= 0; i
< args
->server_options
->nr
; i
++)
1186 packet_write_fmt(fd_out
, "server-option=%s",
1187 args
->server_options
->items
[i
].string
);
1190 packet_buf_delim(&req_buf
);
1191 if (args
->use_thin_pack
)
1192 packet_buf_write(&req_buf
, "thin-pack");
1193 if (args
->no_progress
)
1194 packet_buf_write(&req_buf
, "no-progress");
1195 if (args
->include_tag
)
1196 packet_buf_write(&req_buf
, "include-tag");
1197 if (prefer_ofs_delta
)
1198 packet_buf_write(&req_buf
, "ofs-delta");
1200 /* Add shallow-info and deepen request */
1201 if (server_supports_feature("fetch", "shallow", 0))
1202 add_shallow_requests(&req_buf
, args
);
1203 else if (is_repository_shallow(the_repository
) || args
->deepen
)
1204 die(_("Server does not support shallow requests"));
1207 if (server_supports_feature("fetch", "filter", 0) &&
1208 args
->filter_options
.choice
) {
1209 print_verbose(args
, _("Server supports filter"));
1210 packet_buf_write(&req_buf
, "filter %s",
1211 args
->filter_options
.filter_spec
);
1212 } else if (args
->filter_options
.choice
) {
1213 warning("filtering not recognized by server, ignoring");
1217 add_wants(wants
, &req_buf
);
1219 if (args
->no_dependents
) {
1220 packet_buf_write(&req_buf
, "done");
1223 /* Add all of the common commits we've found in previous rounds */
1224 add_common(&req_buf
, common
);
1226 /* Add initial haves */
1227 ret
= add_haves(&req_buf
, haves_to_send
, in_vain
);
1231 packet_buf_flush(&req_buf
);
1232 write_or_die(fd_out
, req_buf
.buf
, req_buf
.len
);
1234 strbuf_release(&req_buf
);
1239 * Processes a section header in a server's response and checks if it matches
1240 * `section`. If the value of `peek` is 1, the header line will be peeked (and
1241 * not consumed); if 0, the line will be consumed and the function will die if
1242 * the section header doesn't match what was expected.
1244 static int process_section_header(struct packet_reader
*reader
,
1245 const char *section
, int peek
)
1249 if (packet_reader_peek(reader
) != PACKET_READ_NORMAL
)
1250 die("error reading section header '%s'", section
);
1252 ret
= !strcmp(reader
->line
, section
);
1256 die("expected '%s', received '%s'",
1257 section
, reader
->line
);
1258 packet_reader_read(reader
);
1264 static int process_acks(struct packet_reader
*reader
, struct oidset
*common
)
1267 int received_ready
= 0;
1268 int received_ack
= 0;
1270 process_section_header(reader
, "acknowledgments", 0);
1271 while (packet_reader_read(reader
) == PACKET_READ_NORMAL
) {
1274 if (!strcmp(reader
->line
, "NAK"))
1277 if (skip_prefix(reader
->line
, "ACK ", &arg
)) {
1278 struct object_id oid
;
1279 if (!get_oid_hex(arg
, &oid
)) {
1280 struct commit
*commit
;
1281 oidset_insert(common
, &oid
);
1282 commit
= lookup_commit(the_repository
, &oid
);
1283 mark_common(commit
, 0, 1);
1288 if (!strcmp(reader
->line
, "ready")) {
1289 clear_prio_queue(&rev_list
);
1294 die("unexpected acknowledgment line: '%s'", reader
->line
);
1297 if (reader
->status
!= PACKET_READ_FLUSH
&&
1298 reader
->status
!= PACKET_READ_DELIM
)
1299 die("error processing acks: %d", reader
->status
);
1301 /* return 0 if no common, 1 if there are common, or 2 if ready */
1302 return received_ready
? 2 : (received_ack
? 1 : 0);
1305 static void receive_shallow_info(struct fetch_pack_args
*args
,
1306 struct packet_reader
*reader
)
1308 process_section_header(reader
, "shallow-info", 0);
1309 while (packet_reader_read(reader
) == PACKET_READ_NORMAL
) {
1311 struct object_id oid
;
1313 if (skip_prefix(reader
->line
, "shallow ", &arg
)) {
1314 if (get_oid_hex(arg
, &oid
))
1315 die(_("invalid shallow line: %s"), reader
->line
);
1316 register_shallow(the_repository
, &oid
);
1319 if (skip_prefix(reader
->line
, "unshallow ", &arg
)) {
1320 if (get_oid_hex(arg
, &oid
))
1321 die(_("invalid unshallow line: %s"), reader
->line
);
1322 if (!lookup_object(the_repository
, oid
.hash
))
1323 die(_("object not found: %s"), reader
->line
);
1324 /* make sure that it is parsed as shallow */
1325 if (!parse_object(the_repository
, &oid
))
1326 die(_("error in object: %s"), reader
->line
);
1327 if (unregister_shallow(&oid
))
1328 die(_("no shallow found: %s"), reader
->line
);
1331 die(_("expected shallow/unshallow, got %s"), reader
->line
);
1334 if (reader
->status
!= PACKET_READ_FLUSH
&&
1335 reader
->status
!= PACKET_READ_DELIM
)
1336 die("error processing shallow info: %d", reader
->status
);
1338 setup_alternate_shallow(&shallow_lock
, &alternate_shallow_file
, NULL
);
1343 FETCH_CHECK_LOCAL
= 0,
1350 static struct ref
*do_fetch_pack_v2(struct fetch_pack_args
*args
,
1352 const struct ref
*orig_ref
,
1353 struct ref
**sought
, int nr_sought
,
1354 char **pack_lockfile
)
1356 struct ref
*ref
= copy_ref_list(orig_ref
);
1357 enum fetch_state state
= FETCH_CHECK_LOCAL
;
1358 struct oidset common
= OIDSET_INIT
;
1359 struct packet_reader reader
;
1361 int haves_to_send
= INITIAL_FLUSH
;
1362 packet_reader_init(&reader
, fd
[0], NULL
, 0,
1363 PACKET_READ_CHOMP_NEWLINE
);
1365 while (state
!= FETCH_DONE
) {
1367 case FETCH_CHECK_LOCAL
:
1368 sort_ref_list(&ref
, ref_compare_name
);
1369 QSORT(sought
, nr_sought
, cmp_ref_by_name
);
1371 /* v2 supports these by default */
1372 allow_unadvertised_object_request
|= ALLOW_REACHABLE_SHA1
;
1374 if (args
->depth
> 0 || args
->deepen_since
|| args
->deepen_not
)
1378 for_each_ref(clear_marks
, NULL
);
1381 for_each_ref(rev_list_insert_ref_oid
, NULL
);
1382 for_each_cached_alternate(insert_one_alternate_object
);
1384 /* Filter 'ref' by 'sought' and those that aren't local */
1385 if (everything_local(args
, &ref
, sought
, nr_sought
))
1388 state
= FETCH_SEND_REQUEST
;
1390 case FETCH_SEND_REQUEST
:
1391 if (send_fetch_request(fd
[1], args
, ref
, &common
,
1392 &haves_to_send
, &in_vain
))
1393 state
= FETCH_GET_PACK
;
1395 state
= FETCH_PROCESS_ACKS
;
1397 case FETCH_PROCESS_ACKS
:
1398 /* Process ACKs/NAKs */
1399 switch (process_acks(&reader
, &common
)) {
1401 state
= FETCH_GET_PACK
;
1407 state
= FETCH_SEND_REQUEST
;
1411 case FETCH_GET_PACK
:
1412 /* Check for shallow-info section */
1413 if (process_section_header(&reader
, "shallow-info", 1))
1414 receive_shallow_info(args
, &reader
);
1417 process_section_header(&reader
, "packfile", 0);
1418 if (get_pack(args
, fd
, pack_lockfile
))
1419 die(_("git fetch-pack: fetch failed."));
1428 oidset_clear(&common
);
1432 static void fetch_pack_config(void)
1434 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit
);
1435 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit
);
1436 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta
);
1437 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects
);
1438 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects
);
1440 git_config(git_default_config
, NULL
);
1443 static void fetch_pack_setup(void)
1445 static int did_setup
;
1448 fetch_pack_config();
1449 if (0 <= transfer_unpack_limit
)
1450 unpack_limit
= transfer_unpack_limit
;
1451 else if (0 <= fetch_unpack_limit
)
1452 unpack_limit
= fetch_unpack_limit
;
1456 static int remove_duplicates_in_refs(struct ref
**ref
, int nr
)
1458 struct string_list names
= STRING_LIST_INIT_NODUP
;
1461 for (src
= dst
= 0; src
< nr
; src
++) {
1462 struct string_list_item
*item
;
1463 item
= string_list_insert(&names
, ref
[src
]->name
);
1465 continue; /* already have it */
1466 item
->util
= ref
[src
];
1468 ref
[dst
] = ref
[src
];
1471 for (src
= dst
; src
< nr
; src
++)
1473 string_list_clear(&names
, 0);
1477 static void update_shallow(struct fetch_pack_args
*args
,
1478 struct ref
**sought
, int nr_sought
,
1479 struct shallow_info
*si
)
1481 struct oid_array ref
= OID_ARRAY_INIT
;
1485 if (args
->deepen
&& alternate_shallow_file
) {
1486 if (*alternate_shallow_file
== '\0') { /* --unshallow */
1487 unlink_or_warn(git_path_shallow(the_repository
));
1488 rollback_lock_file(&shallow_lock
);
1490 commit_lock_file(&shallow_lock
);
1494 if (!si
->shallow
|| !si
->shallow
->nr
)
1497 if (args
->cloning
) {
1499 * remote is shallow, but this is a clone, there are
1500 * no objects in repo to worry about. Accept any
1501 * shallow points that exist in the pack (iow in repo
1502 * after get_pack() and reprepare_packed_git())
1504 struct oid_array extra
= OID_ARRAY_INIT
;
1505 struct object_id
*oid
= si
->shallow
->oid
;
1506 for (i
= 0; i
< si
->shallow
->nr
; i
++)
1507 if (has_object_file(&oid
[i
]))
1508 oid_array_append(&extra
, &oid
[i
]);
1510 setup_alternate_shallow(&shallow_lock
,
1511 &alternate_shallow_file
,
1513 commit_lock_file(&shallow_lock
);
1515 oid_array_clear(&extra
);
1519 if (!si
->nr_ours
&& !si
->nr_theirs
)
1522 remove_nonexistent_theirs_shallow(si
);
1523 if (!si
->nr_ours
&& !si
->nr_theirs
)
1525 for (i
= 0; i
< nr_sought
; i
++)
1526 oid_array_append(&ref
, &sought
[i
]->old_oid
);
1529 if (args
->update_shallow
) {
1531 * remote is also shallow, .git/shallow may be updated
1532 * so all refs can be accepted. Make sure we only add
1533 * shallow roots that are actually reachable from new
1536 struct oid_array extra
= OID_ARRAY_INIT
;
1537 struct object_id
*oid
= si
->shallow
->oid
;
1538 assign_shallow_commits_to_refs(si
, NULL
, NULL
);
1539 if (!si
->nr_ours
&& !si
->nr_theirs
) {
1540 oid_array_clear(&ref
);
1543 for (i
= 0; i
< si
->nr_ours
; i
++)
1544 oid_array_append(&extra
, &oid
[si
->ours
[i
]]);
1545 for (i
= 0; i
< si
->nr_theirs
; i
++)
1546 oid_array_append(&extra
, &oid
[si
->theirs
[i
]]);
1547 setup_alternate_shallow(&shallow_lock
,
1548 &alternate_shallow_file
,
1550 commit_lock_file(&shallow_lock
);
1551 oid_array_clear(&extra
);
1552 oid_array_clear(&ref
);
1557 * remote is also shallow, check what ref is safe to update
1558 * without updating .git/shallow
1560 status
= xcalloc(nr_sought
, sizeof(*status
));
1561 assign_shallow_commits_to_refs(si
, NULL
, status
);
1562 if (si
->nr_ours
|| si
->nr_theirs
) {
1563 for (i
= 0; i
< nr_sought
; i
++)
1565 sought
[i
]->status
= REF_STATUS_REJECT_SHALLOW
;
1568 oid_array_clear(&ref
);
1571 struct ref
*fetch_pack(struct fetch_pack_args
*args
,
1572 int fd
[], struct child_process
*conn
,
1573 const struct ref
*ref
,
1575 struct ref
**sought
, int nr_sought
,
1576 struct oid_array
*shallow
,
1577 char **pack_lockfile
,
1578 enum protocol_version version
)
1580 struct ref
*ref_cpy
;
1581 struct shallow_info si
;
1585 nr_sought
= remove_duplicates_in_refs(sought
, nr_sought
);
1588 packet_flush(fd
[1]);
1589 die(_("no matching remote head"));
1591 prepare_shallow_info(&si
, shallow
);
1592 if (version
== protocol_v2
)
1593 ref_cpy
= do_fetch_pack_v2(args
, fd
, ref
, sought
, nr_sought
,
1596 ref_cpy
= do_fetch_pack(args
, fd
, ref
, sought
, nr_sought
,
1597 &si
, pack_lockfile
);
1598 reprepare_packed_git(the_repository
);
1599 update_shallow(args
, sought
, nr_sought
, &si
);
1600 clear_shallow_info(&si
);
1604 int report_unmatched_refs(struct ref
**sought
, int nr_sought
)
1608 for (i
= 0; i
< nr_sought
; i
++) {
1611 switch (sought
[i
]->match_status
) {
1614 case REF_NOT_MATCHED
:
1615 error(_("no such remote ref %s"), sought
[i
]->name
);
1617 case REF_UNADVERTISED_NOT_ALLOWED
:
1618 error(_("Server does not allow request for unadvertised object %s"),