2 #include "repository.h"
12 #include "fetch-pack.h"
14 #include "run-command.h"
16 #include "transport.h"
18 #include "oid-array.h"
21 #include "object-store.h"
22 #include "connected.h"
23 #include "fetch-negotiator.h"
26 #include "commit-reach.h"
27 #include "commit-graph.h"
29 #include "mergesort.h"
31 static int transfer_unpack_limit
= -1;
32 static int fetch_unpack_limit
= -1;
33 static int unpack_limit
= 100;
34 static int prefer_ofs_delta
= 1;
36 static int deepen_since_ok
;
37 static int deepen_not_ok
;
38 static int fetch_fsck_objects
= -1;
39 static int transfer_fsck_objects
= -1;
40 static int agent_supported
;
41 static int server_supports_filtering
;
42 static int advertise_sid
;
43 static struct shallow_lock shallow_lock
;
44 static const char *alternate_shallow_file
;
45 static struct fsck_options fsck_options
= FSCK_OPTIONS_MISSING_GITMODULES
;
46 static struct strbuf fsck_msg_types
= STRBUF_INIT
;
47 static struct string_list uri_protocols
= STRING_LIST_INIT_DUP
;
49 /* Remember to update object flag allocation in object.h */
50 #define COMPLETE (1U << 0)
51 #define ALTERNATE (1U << 1)
52 #define COMMON (1U << 6)
53 #define REACH_SCRATCH (1U << 7)
56 * After sending this many "have"s if we do not get any new ACK , we
57 * give up traversing our history.
59 #define MAX_IN_VAIN 256
61 static int multi_ack
, use_sideband
;
62 /* Allow specifying sha1 if it is a ref tip. */
63 #define ALLOW_TIP_SHA1 01
64 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
65 #define ALLOW_REACHABLE_SHA1 02
66 static unsigned int allow_unadvertised_object_request
;
68 __attribute__((format (printf
, 2, 3)))
69 static inline void print_verbose(const struct fetch_pack_args
*args
,
77 va_start(params
, fmt
);
78 vfprintf(stderr
, fmt
, params
);
83 struct alternate_object_cache
{
84 struct object
**items
;
88 static void cache_one_alternate(const struct object_id
*oid
,
91 struct alternate_object_cache
*cache
= vcache
;
92 struct object
*obj
= parse_object(the_repository
, oid
);
94 if (!obj
|| (obj
->flags
& ALTERNATE
))
97 obj
->flags
|= ALTERNATE
;
98 ALLOC_GROW(cache
->items
, cache
->nr
+ 1, cache
->alloc
);
99 cache
->items
[cache
->nr
++] = obj
;
102 static void for_each_cached_alternate(struct fetch_negotiator
*negotiator
,
103 void (*cb
)(struct fetch_negotiator
*,
106 static int initialized
;
107 static struct alternate_object_cache cache
;
111 for_each_alternate_ref(cache_one_alternate
, &cache
);
115 for (i
= 0; i
< cache
.nr
; i
++)
116 cb(negotiator
, cache
.items
[i
]);
119 static struct commit
*deref_without_lazy_fetch(const struct object_id
*oid
,
120 int mark_tags_complete
)
122 enum object_type type
;
123 struct object_info info
= { .typep
= &type
};
124 struct commit
*commit
;
126 commit
= lookup_commit_in_graph(the_repository
, oid
);
131 if (oid_object_info_extended(the_repository
, oid
, &info
,
132 OBJECT_INFO_SKIP_FETCH_OBJECT
| OBJECT_INFO_QUICK
))
134 if (type
== OBJ_TAG
) {
135 struct tag
*tag
= (struct tag
*)
136 parse_object(the_repository
, oid
);
140 if (mark_tags_complete
)
141 tag
->object
.flags
|= COMPLETE
;
142 oid
= &tag
->tagged
->oid
;
148 if (type
== OBJ_COMMIT
) {
149 struct commit
*commit
= lookup_commit(the_repository
, oid
);
150 if (!commit
|| repo_parse_commit(the_repository
, commit
))
158 static int rev_list_insert_ref(struct fetch_negotiator
*negotiator
,
159 const struct object_id
*oid
)
161 struct commit
*c
= deref_without_lazy_fetch(oid
, 0);
164 negotiator
->add_tip(negotiator
, c
);
168 static int rev_list_insert_ref_oid(const char *refname
, const struct object_id
*oid
,
169 int flag
, void *cb_data
)
171 return rev_list_insert_ref(cb_data
, oid
);
182 static void consume_shallow_list(struct fetch_pack_args
*args
,
183 struct packet_reader
*reader
)
185 if (args
->stateless_rpc
&& args
->deepen
) {
186 /* If we sent a depth we will get back "duplicate"
187 * shallow and unshallow commands every time there
188 * is a block of have lines exchanged.
190 while (packet_reader_read(reader
) == PACKET_READ_NORMAL
) {
191 if (starts_with(reader
->line
, "shallow "))
193 if (starts_with(reader
->line
, "unshallow "))
195 die(_("git fetch-pack: expected shallow list"));
197 if (reader
->status
!= PACKET_READ_FLUSH
)
198 die(_("git fetch-pack: expected a flush packet after shallow list"));
202 static enum ack_type
get_ack(struct packet_reader
*reader
,
203 struct object_id
*result_oid
)
208 if (packet_reader_read(reader
) != PACKET_READ_NORMAL
)
209 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
210 len
= reader
->pktlen
;
212 if (!strcmp(reader
->line
, "NAK"))
214 if (skip_prefix(reader
->line
, "ACK ", &arg
)) {
216 if (!parse_oid_hex(arg
, result_oid
, &p
)) {
217 len
-= p
- reader
->line
;
220 if (strstr(p
, "continue"))
222 if (strstr(p
, "common"))
224 if (strstr(p
, "ready"))
229 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), reader
->line
);
232 static void send_request(struct fetch_pack_args
*args
,
233 int fd
, struct strbuf
*buf
)
235 if (args
->stateless_rpc
) {
236 send_sideband(fd
, -1, buf
->buf
, buf
->len
, LARGE_PACKET_MAX
);
239 if (write_in_full(fd
, buf
->buf
, buf
->len
) < 0)
240 die_errno(_("unable to write to remote"));
244 static void insert_one_alternate_object(struct fetch_negotiator
*negotiator
,
247 rev_list_insert_ref(negotiator
, &obj
->oid
);
250 #define INITIAL_FLUSH 16
251 #define PIPESAFE_FLUSH 32
252 #define LARGE_FLUSH 16384
254 static int next_flush(int stateless_rpc
, int count
)
257 if (count
< LARGE_FLUSH
)
260 count
= count
* 11 / 10;
262 if (count
< PIPESAFE_FLUSH
)
265 count
+= PIPESAFE_FLUSH
;
270 static void mark_tips(struct fetch_negotiator
*negotiator
,
271 const struct oid_array
*negotiation_tips
)
275 if (!negotiation_tips
) {
276 for_each_rawref(rev_list_insert_ref_oid
, negotiator
);
280 for (i
= 0; i
< negotiation_tips
->nr
; i
++)
281 rev_list_insert_ref(negotiator
, &negotiation_tips
->oid
[i
]);
285 static int find_common(struct fetch_negotiator
*negotiator
,
286 struct fetch_pack_args
*args
,
287 int fd
[2], struct object_id
*result_oid
,
291 int count
= 0, flushes
= 0, flush_at
= INITIAL_FLUSH
, retval
;
292 const struct object_id
*oid
;
293 unsigned in_vain
= 0;
294 int got_continue
= 0;
296 struct strbuf req_buf
= STRBUF_INIT
;
297 size_t state_len
= 0;
298 struct packet_reader reader
;
300 if (args
->stateless_rpc
&& multi_ack
== 1)
301 die(_("the option '%s' requires '%s'"), "--stateless-rpc", "multi_ack_detailed");
303 packet_reader_init(&reader
, fd
[0], NULL
, 0,
304 PACKET_READ_CHOMP_NEWLINE
|
305 PACKET_READ_DIE_ON_ERR_PACKET
);
307 mark_tips(negotiator
, args
->negotiation_tips
);
308 for_each_cached_alternate(negotiator
, insert_one_alternate_object
);
311 for ( ; refs
; refs
= refs
->next
) {
312 struct object_id
*remote
= &refs
->old_oid
;
313 const char *remote_hex
;
316 if (!args
->refetch
) {
318 * If that object is complete (i.e. it is an ancestor of a
319 * local ref), we tell them we have it but do not have to
320 * tell them about its ancestors, which they already know
323 * We use lookup_object here because we are only
324 * interested in the case we *know* the object is
325 * reachable and we have already scanned it.
327 if (((o
= lookup_object(the_repository
, remote
)) != NULL
) &&
328 (o
->flags
& COMPLETE
)) {
333 remote_hex
= oid_to_hex(remote
);
335 struct strbuf c
= STRBUF_INIT
;
336 if (multi_ack
== 2) strbuf_addstr(&c
, " multi_ack_detailed");
337 if (multi_ack
== 1) strbuf_addstr(&c
, " multi_ack");
338 if (no_done
) strbuf_addstr(&c
, " no-done");
339 if (use_sideband
== 2) strbuf_addstr(&c
, " side-band-64k");
340 if (use_sideband
== 1) strbuf_addstr(&c
, " side-band");
341 if (args
->deepen_relative
) strbuf_addstr(&c
, " deepen-relative");
342 if (args
->use_thin_pack
) strbuf_addstr(&c
, " thin-pack");
343 if (args
->no_progress
) strbuf_addstr(&c
, " no-progress");
344 if (args
->include_tag
) strbuf_addstr(&c
, " include-tag");
345 if (prefer_ofs_delta
) strbuf_addstr(&c
, " ofs-delta");
346 if (deepen_since_ok
) strbuf_addstr(&c
, " deepen-since");
347 if (deepen_not_ok
) strbuf_addstr(&c
, " deepen-not");
348 if (agent_supported
) strbuf_addf(&c
, " agent=%s",
349 git_user_agent_sanitized());
351 strbuf_addf(&c
, " session-id=%s", trace2_session_id());
352 if (args
->filter_options
.choice
)
353 strbuf_addstr(&c
, " filter");
354 packet_buf_write(&req_buf
, "want %s%s\n", remote_hex
, c
.buf
);
357 packet_buf_write(&req_buf
, "want %s\n", remote_hex
);
362 strbuf_release(&req_buf
);
367 if (is_repository_shallow(the_repository
))
368 write_shallow_commits(&req_buf
, 1, NULL
);
370 packet_buf_write(&req_buf
, "deepen %d", args
->depth
);
371 if (args
->deepen_since
) {
372 timestamp_t max_age
= approxidate(args
->deepen_since
);
373 packet_buf_write(&req_buf
, "deepen-since %"PRItime
, max_age
);
375 if (args
->deepen_not
) {
377 for (i
= 0; i
< args
->deepen_not
->nr
; i
++) {
378 struct string_list_item
*s
= args
->deepen_not
->items
+ i
;
379 packet_buf_write(&req_buf
, "deepen-not %s", s
->string
);
382 if (server_supports_filtering
&& args
->filter_options
.choice
) {
384 expand_list_objects_filter_spec(&args
->filter_options
);
385 packet_buf_write(&req_buf
, "filter %s", spec
);
387 packet_buf_flush(&req_buf
);
388 state_len
= req_buf
.len
;
392 struct object_id oid
;
394 send_request(args
, fd
[1], &req_buf
);
395 while (packet_reader_read(&reader
) == PACKET_READ_NORMAL
) {
396 if (skip_prefix(reader
.line
, "shallow ", &arg
)) {
397 if (get_oid_hex(arg
, &oid
))
398 die(_("invalid shallow line: %s"), reader
.line
);
399 register_shallow(the_repository
, &oid
);
402 if (skip_prefix(reader
.line
, "unshallow ", &arg
)) {
403 if (get_oid_hex(arg
, &oid
))
404 die(_("invalid unshallow line: %s"), reader
.line
);
405 if (!lookup_object(the_repository
, &oid
))
406 die(_("object not found: %s"), reader
.line
);
407 /* make sure that it is parsed as shallow */
408 if (!parse_object(the_repository
, &oid
))
409 die(_("error in object: %s"), reader
.line
);
410 if (unregister_shallow(&oid
))
411 die(_("no shallow found: %s"), reader
.line
);
414 die(_("expected shallow/unshallow, got %s"), reader
.line
);
416 } else if (!args
->stateless_rpc
)
417 send_request(args
, fd
[1], &req_buf
);
419 if (!args
->stateless_rpc
) {
420 /* If we aren't using the stateless-rpc interface
421 * we don't need to retain the headers.
423 strbuf_setlen(&req_buf
, 0);
427 trace2_region_enter("fetch-pack", "negotiation_v0_v1", the_repository
);
430 while ((oid
= negotiator
->next(negotiator
))) {
431 packet_buf_write(&req_buf
, "have %s\n", oid_to_hex(oid
));
432 print_verbose(args
, "have %s", oid_to_hex(oid
));
434 if (flush_at
<= ++count
) {
437 packet_buf_flush(&req_buf
);
438 send_request(args
, fd
[1], &req_buf
);
439 strbuf_setlen(&req_buf
, state_len
);
441 flush_at
= next_flush(args
->stateless_rpc
, count
);
444 * We keep one window "ahead" of the other side, and
445 * will wait for an ACK only on the next one
447 if (!args
->stateless_rpc
&& count
== INITIAL_FLUSH
)
450 consume_shallow_list(args
, &reader
);
452 ack
= get_ack(&reader
, result_oid
);
454 print_verbose(args
, _("got %s %d %s"), "ack",
455 ack
, oid_to_hex(result_oid
));
465 struct commit
*commit
=
466 lookup_commit(the_repository
,
471 die(_("invalid commit %s"), oid_to_hex(result_oid
));
472 was_common
= negotiator
->ack(negotiator
, commit
);
473 if (args
->stateless_rpc
476 /* We need to replay the have for this object
477 * on the next RPC request so the peer knows
478 * it is in common with us.
480 const char *hex
= oid_to_hex(result_oid
);
481 packet_buf_write(&req_buf
, "have %s\n", hex
);
482 state_len
= req_buf
.len
;
484 * Reset in_vain because an ack
485 * for this commit has not been
489 } else if (!args
->stateless_rpc
490 || ack
!= ACK_common
)
494 if (ack
== ACK_ready
)
501 if (got_continue
&& MAX_IN_VAIN
< in_vain
) {
502 print_verbose(args
, _("giving up"));
510 trace2_region_leave("fetch-pack", "negotiation_v0_v1", the_repository
);
511 if (!got_ready
|| !no_done
) {
512 packet_buf_write(&req_buf
, "done\n");
513 send_request(args
, fd
[1], &req_buf
);
515 print_verbose(args
, _("done"));
520 strbuf_release(&req_buf
);
522 if (!got_ready
|| !no_done
)
523 consume_shallow_list(args
, &reader
);
524 while (flushes
|| multi_ack
) {
525 int ack
= get_ack(&reader
, result_oid
);
527 print_verbose(args
, _("got %s (%d) %s"), "ack",
528 ack
, oid_to_hex(result_oid
));
536 /* it is no error to fetch into a completely empty repo */
537 return count
? retval
: 0;
540 static struct commit_list
*complete
;
542 static int mark_complete(const struct object_id
*oid
)
544 struct commit
*commit
= deref_without_lazy_fetch(oid
, 1);
546 if (commit
&& !(commit
->object
.flags
& COMPLETE
)) {
547 commit
->object
.flags
|= COMPLETE
;
548 commit_list_insert(commit
, &complete
);
553 static int mark_complete_oid(const char *refname
, const struct object_id
*oid
,
554 int flag
, void *cb_data
)
556 return mark_complete(oid
);
559 static void mark_recent_complete_commits(struct fetch_pack_args
*args
,
562 while (complete
&& cutoff
<= complete
->item
->date
) {
563 print_verbose(args
, _("Marking %s as complete"),
564 oid_to_hex(&complete
->item
->object
.oid
));
565 pop_most_recent_commit(&complete
, COMPLETE
);
569 static void add_refs_to_oidset(struct oidset
*oids
, struct ref
*refs
)
571 for (; refs
; refs
= refs
->next
)
572 oidset_insert(oids
, &refs
->old_oid
);
575 static int is_unmatched_ref(const struct ref
*ref
)
577 struct object_id oid
;
579 return ref
->match_status
== REF_NOT_MATCHED
&&
580 !parse_oid_hex(ref
->name
, &oid
, &p
) &&
582 oideq(&oid
, &ref
->old_oid
);
585 static void filter_refs(struct fetch_pack_args
*args
,
587 struct ref
**sought
, int nr_sought
)
589 struct ref
*newlist
= NULL
;
590 struct ref
**newtail
= &newlist
;
591 struct ref
*unmatched
= NULL
;
592 struct ref
*ref
, *next
;
593 struct oidset tip_oids
= OIDSET_INIT
;
595 int strict
= !(allow_unadvertised_object_request
&
596 (ALLOW_TIP_SHA1
| ALLOW_REACHABLE_SHA1
));
599 for (ref
= *refs
; ref
; ref
= next
) {
603 if (starts_with(ref
->name
, "refs/") &&
604 check_refname_format(ref
->name
, 0)) {
606 * trash or a peeled value; do not even add it to
612 while (i
< nr_sought
) {
613 int cmp
= strcmp(ref
->name
, sought
[i
]->name
);
615 break; /* definitely do not have it */
617 keep
= 1; /* definitely have it */
618 sought
[i
]->match_status
= REF_MATCHED
;
623 if (!keep
&& args
->fetch_all
&&
624 (!args
->deepen
|| !starts_with(ref
->name
, "refs/tags/")))
631 newtail
= &ref
->next
;
633 ref
->next
= unmatched
;
639 for (i
= 0; i
< nr_sought
; i
++) {
641 if (!is_unmatched_ref(ref
))
644 add_refs_to_oidset(&tip_oids
, unmatched
);
645 add_refs_to_oidset(&tip_oids
, newlist
);
650 /* Append unmatched requests to the list */
651 for (i
= 0; i
< nr_sought
; i
++) {
653 if (!is_unmatched_ref(ref
))
656 if (!strict
|| oidset_contains(&tip_oids
, &ref
->old_oid
)) {
657 ref
->match_status
= REF_MATCHED
;
658 *newtail
= copy_ref(ref
);
659 newtail
= &(*newtail
)->next
;
661 ref
->match_status
= REF_UNADVERTISED_NOT_ALLOWED
;
665 oidset_clear(&tip_oids
);
666 free_refs(unmatched
);
671 static void mark_alternate_complete(struct fetch_negotiator
*unused
,
674 mark_complete(&obj
->oid
);
677 struct loose_object_iter
{
678 struct oidset
*loose_object_set
;
683 * Mark recent commits available locally and reachable from a local ref as
686 * The cutoff time for recency is determined by this heuristic: it is the
687 * earliest commit time of the objects in refs that are commits and that we know
688 * the commit time of.
690 static void mark_complete_and_common_ref(struct fetch_negotiator
*negotiator
,
691 struct fetch_pack_args
*args
,
695 int old_save_commit_buffer
= save_commit_buffer
;
696 timestamp_t cutoff
= 0;
701 save_commit_buffer
= 0;
703 trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL
);
704 for (ref
= *refs
; ref
; ref
= ref
->next
) {
705 struct commit
*commit
;
707 commit
= lookup_commit_in_graph(the_repository
, &ref
->old_oid
);
711 if (!has_object_file_with_flags(&ref
->old_oid
,
713 OBJECT_INFO_SKIP_FETCH_OBJECT
))
715 o
= parse_object(the_repository
, &ref
->old_oid
);
716 if (!o
|| o
->type
!= OBJ_COMMIT
)
719 commit
= (struct commit
*)o
;
723 * We already have it -- which may mean that we were
724 * in sync with the other side at some time after
725 * that (it is OK if we guess wrong here).
727 if (!cutoff
|| cutoff
< commit
->date
)
728 cutoff
= commit
->date
;
730 trace2_region_leave("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL
);
733 * This block marks all local refs as COMPLETE, and then recursively marks all
734 * parents of those refs as COMPLETE.
736 trace2_region_enter("fetch-pack", "mark_complete_local_refs", NULL
);
738 for_each_rawref(mark_complete_oid
, NULL
);
739 for_each_cached_alternate(NULL
, mark_alternate_complete
);
740 commit_list_sort_by_date(&complete
);
742 mark_recent_complete_commits(args
, cutoff
);
744 trace2_region_leave("fetch-pack", "mark_complete_local_refs", NULL
);
747 * Mark all complete remote refs as common refs.
748 * Don't mark them common yet; the server has to be told so first.
750 trace2_region_enter("fetch-pack", "mark_common_remote_refs", NULL
);
751 for (ref
= *refs
; ref
; ref
= ref
->next
) {
752 struct commit
*c
= deref_without_lazy_fetch(&ref
->old_oid
, 0);
754 if (!c
|| !(c
->object
.flags
& COMPLETE
))
757 negotiator
->known_common(negotiator
, c
);
759 trace2_region_leave("fetch-pack", "mark_common_remote_refs", NULL
);
761 save_commit_buffer
= old_save_commit_buffer
;
765 * Returns 1 if every object pointed to by the given remote refs is available
766 * locally and reachable from a local ref, and 0 otherwise.
768 static int everything_local(struct fetch_pack_args
*args
,
774 for (retval
= 1, ref
= *refs
; ref
; ref
= ref
->next
) {
775 const struct object_id
*remote
= &ref
->old_oid
;
778 o
= lookup_object(the_repository
, remote
);
779 if (!o
|| !(o
->flags
& COMPLETE
)) {
781 print_verbose(args
, "want %s (%s)", oid_to_hex(remote
),
785 print_verbose(args
, _("already have %s (%s)"), oid_to_hex(remote
),
792 static int sideband_demux(int in
, int out
, void *data
)
797 ret
= recv_sideband("fetch-pack", xd
[0], out
);
802 static void create_promisor_file(const char *keep_name
,
803 struct ref
**sought
, int nr_sought
)
805 struct strbuf promisor_name
= STRBUF_INIT
;
808 strbuf_addstr(&promisor_name
, keep_name
);
809 suffix_stripped
= strbuf_strip_suffix(&promisor_name
, ".keep");
810 if (!suffix_stripped
)
811 BUG("name of pack lockfile should end with .keep (was '%s')",
813 strbuf_addstr(&promisor_name
, ".promisor");
815 write_promisor_file(promisor_name
.buf
, sought
, nr_sought
);
817 strbuf_release(&promisor_name
);
820 static void parse_gitmodules_oids(int fd
, struct oidset
*gitmodules_oids
)
822 int len
= the_hash_algo
->hexsz
+ 1; /* hash + NL */
825 char hex_hash
[GIT_MAX_HEXSZ
+ 1];
826 int read_len
= read_in_full(fd
, hex_hash
, len
);
827 struct object_id oid
;
833 die("invalid length read %d", read_len
);
834 if (parse_oid_hex(hex_hash
, &oid
, &end
) || *end
!= '\n')
836 oidset_insert(gitmodules_oids
, &oid
);
841 * If packfile URIs were provided, pass a non-NULL pointer to index_pack_args.
842 * The strings to pass as the --index-pack-arg arguments to http-fetch will be
843 * stored there. (It must be freed by the caller.)
845 static int get_pack(struct fetch_pack_args
*args
,
846 int xd
[2], struct string_list
*pack_lockfiles
,
847 struct strvec
*index_pack_args
,
848 struct ref
**sought
, int nr_sought
,
849 struct oidset
*gitmodules_oids
)
852 int do_keep
= args
->keep_pack
;
853 const char *cmd_name
;
854 struct pack_header header
;
856 struct child_process cmd
= CHILD_PROCESS_INIT
;
857 int fsck_objects
= 0;
860 memset(&demux
, 0, sizeof(demux
));
862 /* xd[] is talking with upload-pack; subprocess reads from
863 * xd[0], spits out band#2 to stderr, and feeds us band#1
864 * through demux->out.
866 demux
.proc
= sideband_demux
;
869 demux
.isolate_sigpipe
= 1;
870 if (start_async(&demux
))
871 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
876 if (!args
->keep_pack
&& unpack_limit
&& !index_pack_args
) {
878 if (read_pack_header(demux
.out
, &header
))
879 die(_("protocol error: bad pack header"));
881 if (ntohl(header
.hdr_entries
) < unpack_limit
)
887 if (alternate_shallow_file
) {
888 strvec_push(&cmd
.args
, "--shallow-file");
889 strvec_push(&cmd
.args
, alternate_shallow_file
);
892 if (fetch_fsck_objects
>= 0
894 : transfer_fsck_objects
>= 0
895 ? transfer_fsck_objects
899 if (do_keep
|| args
->from_promisor
|| index_pack_args
|| fsck_objects
) {
900 if (pack_lockfiles
|| fsck_objects
)
902 cmd_name
= "index-pack";
903 strvec_push(&cmd
.args
, cmd_name
);
904 strvec_push(&cmd
.args
, "--stdin");
905 if (!args
->quiet
&& !args
->no_progress
)
906 strvec_push(&cmd
.args
, "-v");
907 if (args
->use_thin_pack
)
908 strvec_push(&cmd
.args
, "--fix-thin");
909 if ((do_keep
|| index_pack_args
) && (args
->lock_pack
|| unpack_limit
)) {
910 char hostname
[HOST_NAME_MAX
+ 1];
911 if (xgethostname(hostname
, sizeof(hostname
)))
912 xsnprintf(hostname
, sizeof(hostname
), "localhost");
913 strvec_pushf(&cmd
.args
,
914 "--keep=fetch-pack %"PRIuMAX
" on %s",
915 (uintmax_t)getpid(), hostname
);
917 if (!index_pack_args
&& args
->check_self_contained_and_connected
)
918 strvec_push(&cmd
.args
, "--check-self-contained-and-connected");
921 * We cannot perform any connectivity checks because
922 * not all packs have been downloaded; let the caller
923 * have this responsibility.
925 args
->check_self_contained_and_connected
= 0;
927 if (args
->from_promisor
)
929 * create_promisor_file() may be called afterwards but
930 * we still need index-pack to know that this is a
931 * promisor pack. For example, if transfer.fsckobjects
932 * is true, index-pack needs to know that .gitmodules
933 * is a promisor object (so that it won't complain if
936 strvec_push(&cmd
.args
, "--promisor");
939 cmd_name
= "unpack-objects";
940 strvec_push(&cmd
.args
, cmd_name
);
941 if (args
->quiet
|| args
->no_progress
)
942 strvec_push(&cmd
.args
, "-q");
943 args
->check_self_contained_and_connected
= 0;
947 strvec_pushf(&cmd
.args
, "--pack_header=%"PRIu32
",%"PRIu32
,
948 ntohl(header
.hdr_version
),
949 ntohl(header
.hdr_entries
));
951 if (args
->from_promisor
|| index_pack_args
)
953 * We cannot use --strict in index-pack because it
954 * checks both broken objects and links, but we only
955 * want to check for broken objects.
957 strvec_push(&cmd
.args
, "--fsck-objects");
959 strvec_pushf(&cmd
.args
, "--strict%s",
963 if (index_pack_args
) {
966 for (i
= 0; i
< cmd
.args
.nr
; i
++)
967 strvec_push(index_pack_args
, cmd
.args
.v
[i
]);
970 sigchain_push(SIGPIPE
, SIG_IGN
);
974 if (start_command(&cmd
))
975 die(_("fetch-pack: unable to fork off %s"), cmd_name
);
976 if (do_keep
&& (pack_lockfiles
|| fsck_objects
)) {
978 char *pack_lockfile
= index_pack_lockfile(cmd
.out
, &is_well_formed
);
981 die(_("fetch-pack: invalid index-pack output"));
983 string_list_append_nodup(pack_lockfiles
, pack_lockfile
);
984 parse_gitmodules_oids(cmd
.out
, gitmodules_oids
);
989 /* Closed by start_command() */
992 ret
= finish_command(&cmd
);
993 if (!ret
|| (args
->check_self_contained_and_connected
&& ret
== 1))
994 args
->self_contained_and_connected
=
995 args
->check_self_contained_and_connected
&&
998 die(_("%s failed"), cmd_name
);
999 if (use_sideband
&& finish_async(&demux
))
1000 die(_("error in sideband demultiplexer"));
1002 sigchain_pop(SIGPIPE
);
1005 * Now that index-pack has succeeded, write the promisor file using the
1006 * obtained .keep filename if necessary
1008 if (do_keep
&& pack_lockfiles
&& pack_lockfiles
->nr
&& args
->from_promisor
)
1009 create_promisor_file(pack_lockfiles
->items
[0].string
, sought
, nr_sought
);
1014 static int ref_compare_name(const struct ref
*a
, const struct ref
*b
)
1016 return strcmp(a
->name
, b
->name
);
1019 DEFINE_LIST_SORT(static, sort_ref_list
, struct ref
, next
);
1021 static int cmp_ref_by_name(const void *a_
, const void *b_
)
1023 const struct ref
*a
= *((const struct ref
**)a_
);
1024 const struct ref
*b
= *((const struct ref
**)b_
);
1025 return strcmp(a
->name
, b
->name
);
1028 static struct ref
*do_fetch_pack(struct fetch_pack_args
*args
,
1030 const struct ref
*orig_ref
,
1031 struct ref
**sought
, int nr_sought
,
1032 struct shallow_info
*si
,
1033 struct string_list
*pack_lockfiles
)
1035 struct repository
*r
= the_repository
;
1036 struct ref
*ref
= copy_ref_list(orig_ref
);
1037 struct object_id oid
;
1038 const char *agent_feature
;
1040 struct fetch_negotiator negotiator_alloc
;
1041 struct fetch_negotiator
*negotiator
;
1043 negotiator
= &negotiator_alloc
;
1044 if (args
->refetch
) {
1045 fetch_negotiator_init_noop(negotiator
);
1047 fetch_negotiator_init(r
, negotiator
);
1050 sort_ref_list(&ref
, ref_compare_name
);
1051 QSORT(sought
, nr_sought
, cmp_ref_by_name
);
1053 if ((agent_feature
= server_feature_value("agent", &agent_len
))) {
1054 agent_supported
= 1;
1056 print_verbose(args
, _("Server version is %.*s"),
1057 agent_len
, agent_feature
);
1060 if (!server_supports("session-id"))
1063 if (server_supports("shallow"))
1064 print_verbose(args
, _("Server supports %s"), "shallow");
1065 else if (args
->depth
> 0 || is_repository_shallow(r
))
1066 die(_("Server does not support shallow clients"));
1067 if (args
->depth
> 0 || args
->deepen_since
|| args
->deepen_not
)
1069 if (server_supports("multi_ack_detailed")) {
1070 print_verbose(args
, _("Server supports %s"), "multi_ack_detailed");
1072 if (server_supports("no-done")) {
1073 print_verbose(args
, _("Server supports %s"), "no-done");
1074 if (args
->stateless_rpc
)
1078 else if (server_supports("multi_ack")) {
1079 print_verbose(args
, _("Server supports %s"), "multi_ack");
1082 if (server_supports("side-band-64k")) {
1083 print_verbose(args
, _("Server supports %s"), "side-band-64k");
1086 else if (server_supports("side-band")) {
1087 print_verbose(args
, _("Server supports %s"), "side-band");
1090 if (server_supports("allow-tip-sha1-in-want")) {
1091 print_verbose(args
, _("Server supports %s"), "allow-tip-sha1-in-want");
1092 allow_unadvertised_object_request
|= ALLOW_TIP_SHA1
;
1094 if (server_supports("allow-reachable-sha1-in-want")) {
1095 print_verbose(args
, _("Server supports %s"), "allow-reachable-sha1-in-want");
1096 allow_unadvertised_object_request
|= ALLOW_REACHABLE_SHA1
;
1098 if (server_supports("thin-pack"))
1099 print_verbose(args
, _("Server supports %s"), "thin-pack");
1101 args
->use_thin_pack
= 0;
1102 if (server_supports("no-progress"))
1103 print_verbose(args
, _("Server supports %s"), "no-progress");
1105 args
->no_progress
= 0;
1106 if (server_supports("include-tag"))
1107 print_verbose(args
, _("Server supports %s"), "include-tag");
1109 args
->include_tag
= 0;
1110 if (server_supports("ofs-delta"))
1111 print_verbose(args
, _("Server supports %s"), "ofs-delta");
1113 prefer_ofs_delta
= 0;
1115 if (server_supports("filter")) {
1116 server_supports_filtering
= 1;
1117 print_verbose(args
, _("Server supports %s"), "filter");
1118 } else if (args
->filter_options
.choice
) {
1119 warning("filtering not recognized by server, ignoring");
1122 if (server_supports("deepen-since")) {
1123 print_verbose(args
, _("Server supports %s"), "deepen-since");
1124 deepen_since_ok
= 1;
1125 } else if (args
->deepen_since
)
1126 die(_("Server does not support --shallow-since"));
1127 if (server_supports("deepen-not")) {
1128 print_verbose(args
, _("Server supports %s"), "deepen-not");
1130 } else if (args
->deepen_not
)
1131 die(_("Server does not support --shallow-exclude"));
1132 if (server_supports("deepen-relative"))
1133 print_verbose(args
, _("Server supports %s"), "deepen-relative");
1134 else if (args
->deepen_relative
)
1135 die(_("Server does not support --deepen"));
1136 if (!server_supports_hash(the_hash_algo
->name
, NULL
))
1137 die(_("Server does not support this repository's object format"));
1139 mark_complete_and_common_ref(negotiator
, args
, &ref
);
1140 filter_refs(args
, &ref
, sought
, nr_sought
);
1141 if (!args
->refetch
&& everything_local(args
, &ref
)) {
1142 packet_flush(fd
[1]);
1145 if (find_common(negotiator
, args
, fd
, &oid
, ref
) < 0)
1146 if (!args
->keep_pack
)
1147 /* When cloning, it is not unusual to have
1150 warning(_("no common commits"));
1152 if (args
->stateless_rpc
)
1153 packet_flush(fd
[1]);
1155 setup_alternate_shallow(&shallow_lock
, &alternate_shallow_file
,
1157 else if (si
->nr_ours
|| si
->nr_theirs
) {
1158 if (args
->reject_shallow_remote
)
1159 die(_("source repository is shallow, reject to clone."));
1160 alternate_shallow_file
= setup_temporary_shallow(si
->shallow
);
1162 alternate_shallow_file
= NULL
;
1163 if (get_pack(args
, fd
, pack_lockfiles
, NULL
, sought
, nr_sought
,
1164 &fsck_options
.gitmodules_found
))
1165 die(_("git fetch-pack: fetch failed."));
1166 if (fsck_finish(&fsck_options
))
1171 negotiator
->release(negotiator
);
1175 static void add_shallow_requests(struct strbuf
*req_buf
,
1176 const struct fetch_pack_args
*args
)
1178 if (is_repository_shallow(the_repository
))
1179 write_shallow_commits(req_buf
, 1, NULL
);
1180 if (args
->depth
> 0)
1181 packet_buf_write(req_buf
, "deepen %d", args
->depth
);
1182 if (args
->deepen_since
) {
1183 timestamp_t max_age
= approxidate(args
->deepen_since
);
1184 packet_buf_write(req_buf
, "deepen-since %"PRItime
, max_age
);
1186 if (args
->deepen_not
) {
1188 for (i
= 0; i
< args
->deepen_not
->nr
; i
++) {
1189 struct string_list_item
*s
= args
->deepen_not
->items
+ i
;
1190 packet_buf_write(req_buf
, "deepen-not %s", s
->string
);
1193 if (args
->deepen_relative
)
1194 packet_buf_write(req_buf
, "deepen-relative\n");
1197 static void add_wants(const struct ref
*wants
, struct strbuf
*req_buf
)
1199 int use_ref_in_want
= server_supports_feature("fetch", "ref-in-want", 0);
1201 for ( ; wants
; wants
= wants
->next
) {
1202 const struct object_id
*remote
= &wants
->old_oid
;
1206 * If that object is complete (i.e. it is an ancestor of a
1207 * local ref), we tell them we have it but do not have to
1208 * tell them about its ancestors, which they already know
1211 * We use lookup_object here because we are only
1212 * interested in the case we *know* the object is
1213 * reachable and we have already scanned it.
1215 if (((o
= lookup_object(the_repository
, remote
)) != NULL
) &&
1216 (o
->flags
& COMPLETE
)) {
1220 if (!use_ref_in_want
|| wants
->exact_oid
)
1221 packet_buf_write(req_buf
, "want %s\n", oid_to_hex(remote
));
1223 packet_buf_write(req_buf
, "want-ref %s\n", wants
->name
);
1227 static void add_common(struct strbuf
*req_buf
, struct oidset
*common
)
1229 struct oidset_iter iter
;
1230 const struct object_id
*oid
;
1231 oidset_iter_init(common
, &iter
);
1233 while ((oid
= oidset_iter_next(&iter
))) {
1234 packet_buf_write(req_buf
, "have %s\n", oid_to_hex(oid
));
1238 static int add_haves(struct fetch_negotiator
*negotiator
,
1239 struct strbuf
*req_buf
,
1242 int haves_added
= 0;
1243 const struct object_id
*oid
;
1245 while ((oid
= negotiator
->next(negotiator
))) {
1246 packet_buf_write(req_buf
, "have %s\n", oid_to_hex(oid
));
1247 if (++haves_added
>= *haves_to_send
)
1251 /* Increase haves to send on next round */
1252 *haves_to_send
= next_flush(1, *haves_to_send
);
1257 static void write_fetch_command_and_capabilities(struct strbuf
*req_buf
,
1258 const struct string_list
*server_options
)
1260 const char *hash_name
;
1262 if (server_supports_v2("fetch", 1))
1263 packet_buf_write(req_buf
, "command=fetch");
1264 if (server_supports_v2("agent", 0))
1265 packet_buf_write(req_buf
, "agent=%s", git_user_agent_sanitized());
1266 if (advertise_sid
&& server_supports_v2("session-id", 0))
1267 packet_buf_write(req_buf
, "session-id=%s", trace2_session_id());
1268 if (server_options
&& server_options
->nr
&&
1269 server_supports_v2("server-option", 1)) {
1271 for (i
= 0; i
< server_options
->nr
; i
++)
1272 packet_buf_write(req_buf
, "server-option=%s",
1273 server_options
->items
[i
].string
);
1276 if (server_feature_v2("object-format", &hash_name
)) {
1277 int hash_algo
= hash_algo_by_name(hash_name
);
1278 if (hash_algo_by_ptr(the_hash_algo
) != hash_algo
)
1279 die(_("mismatched algorithms: client %s; server %s"),
1280 the_hash_algo
->name
, hash_name
);
1281 packet_buf_write(req_buf
, "object-format=%s", the_hash_algo
->name
);
1282 } else if (hash_algo_by_ptr(the_hash_algo
) != GIT_HASH_SHA1
) {
1283 die(_("the server does not support algorithm '%s'"),
1284 the_hash_algo
->name
);
1286 packet_buf_delim(req_buf
);
1289 static int send_fetch_request(struct fetch_negotiator
*negotiator
, int fd_out
,
1290 struct fetch_pack_args
*args
,
1291 const struct ref
*wants
, struct oidset
*common
,
1292 int *haves_to_send
, int *in_vain
,
1293 int sideband_all
, int seen_ack
)
1297 struct strbuf req_buf
= STRBUF_INIT
;
1299 write_fetch_command_and_capabilities(&req_buf
, args
->server_options
);
1301 if (args
->use_thin_pack
)
1302 packet_buf_write(&req_buf
, "thin-pack");
1303 if (args
->no_progress
)
1304 packet_buf_write(&req_buf
, "no-progress");
1305 if (args
->include_tag
)
1306 packet_buf_write(&req_buf
, "include-tag");
1307 if (prefer_ofs_delta
)
1308 packet_buf_write(&req_buf
, "ofs-delta");
1310 packet_buf_write(&req_buf
, "sideband-all");
1312 /* Add shallow-info and deepen request */
1313 if (server_supports_feature("fetch", "shallow", 0))
1314 add_shallow_requests(&req_buf
, args
);
1315 else if (is_repository_shallow(the_repository
) || args
->deepen
)
1316 die(_("Server does not support shallow requests"));
1319 if (server_supports_feature("fetch", "filter", 0) &&
1320 args
->filter_options
.choice
) {
1322 expand_list_objects_filter_spec(&args
->filter_options
);
1323 print_verbose(args
, _("Server supports filter"));
1324 packet_buf_write(&req_buf
, "filter %s", spec
);
1325 } else if (args
->filter_options
.choice
) {
1326 warning("filtering not recognized by server, ignoring");
1329 if (server_supports_feature("fetch", "packfile-uris", 0)) {
1331 struct strbuf to_send
= STRBUF_INIT
;
1333 for (i
= 0; i
< uri_protocols
.nr
; i
++) {
1334 const char *s
= uri_protocols
.items
[i
].string
;
1336 if (!strcmp(s
, "https") || !strcmp(s
, "http")) {
1338 strbuf_addch(&to_send
, ',');
1339 strbuf_addstr(&to_send
, s
);
1343 packet_buf_write(&req_buf
, "packfile-uris %s",
1345 strbuf_release(&to_send
);
1350 add_wants(wants
, &req_buf
);
1352 /* Add all of the common commits we've found in previous rounds */
1353 add_common(&req_buf
, common
);
1355 haves_added
= add_haves(negotiator
, &req_buf
, haves_to_send
);
1356 *in_vain
+= haves_added
;
1357 if (!haves_added
|| (seen_ack
&& *in_vain
>= MAX_IN_VAIN
)) {
1359 packet_buf_write(&req_buf
, "done\n");
1364 packet_buf_flush(&req_buf
);
1365 if (write_in_full(fd_out
, req_buf
.buf
, req_buf
.len
) < 0)
1366 die_errno(_("unable to write request to remote"));
1368 strbuf_release(&req_buf
);
1373 * Processes a section header in a server's response and checks if it matches
1374 * `section`. If the value of `peek` is 1, the header line will be peeked (and
1375 * not consumed); if 0, the line will be consumed and the function will die if
1376 * the section header doesn't match what was expected.
1378 static int process_section_header(struct packet_reader
*reader
,
1379 const char *section
, int peek
)
1383 if (packet_reader_peek(reader
) != PACKET_READ_NORMAL
)
1384 die(_("error reading section header '%s'"), section
);
1386 ret
= !strcmp(reader
->line
, section
);
1390 die(_("expected '%s', received '%s'"),
1391 section
, reader
->line
);
1392 packet_reader_read(reader
);
1398 static int process_ack(struct fetch_negotiator
*negotiator
,
1399 struct packet_reader
*reader
,
1400 struct object_id
*common_oid
,
1401 int *received_ready
)
1403 while (packet_reader_read(reader
) == PACKET_READ_NORMAL
) {
1406 if (!strcmp(reader
->line
, "NAK"))
1409 if (skip_prefix(reader
->line
, "ACK ", &arg
)) {
1410 if (!get_oid_hex(arg
, common_oid
)) {
1411 struct commit
*commit
;
1412 commit
= lookup_commit(the_repository
, common_oid
);
1414 negotiator
->ack(negotiator
, commit
);
1419 if (!strcmp(reader
->line
, "ready")) {
1420 *received_ready
= 1;
1424 die(_("unexpected acknowledgment line: '%s'"), reader
->line
);
1427 if (reader
->status
!= PACKET_READ_FLUSH
&&
1428 reader
->status
!= PACKET_READ_DELIM
)
1429 die(_("error processing acks: %d"), reader
->status
);
1432 * If an "acknowledgments" section is sent, a packfile is sent if and
1433 * only if "ready" was sent in this section. The other sections
1434 * ("shallow-info" and "wanted-refs") are sent only if a packfile is
1435 * sent. Therefore, a DELIM is expected if "ready" is sent, and a FLUSH
1438 if (*received_ready
&& reader
->status
!= PACKET_READ_DELIM
)
1440 * TRANSLATORS: The parameter will be 'ready', a protocol
1443 die(_("expected packfile to be sent after '%s'"), "ready");
1444 if (!*received_ready
&& reader
->status
!= PACKET_READ_FLUSH
)
1446 * TRANSLATORS: The parameter will be 'ready', a protocol
1449 die(_("expected no other sections to be sent after no '%s'"), "ready");
1454 static void receive_shallow_info(struct fetch_pack_args
*args
,
1455 struct packet_reader
*reader
,
1456 struct oid_array
*shallows
,
1457 struct shallow_info
*si
)
1459 int unshallow_received
= 0;
1461 process_section_header(reader
, "shallow-info", 0);
1462 while (packet_reader_read(reader
) == PACKET_READ_NORMAL
) {
1464 struct object_id oid
;
1466 if (skip_prefix(reader
->line
, "shallow ", &arg
)) {
1467 if (get_oid_hex(arg
, &oid
))
1468 die(_("invalid shallow line: %s"), reader
->line
);
1469 oid_array_append(shallows
, &oid
);
1472 if (skip_prefix(reader
->line
, "unshallow ", &arg
)) {
1473 if (get_oid_hex(arg
, &oid
))
1474 die(_("invalid unshallow line: %s"), reader
->line
);
1475 if (!lookup_object(the_repository
, &oid
))
1476 die(_("object not found: %s"), reader
->line
);
1477 /* make sure that it is parsed as shallow */
1478 if (!parse_object(the_repository
, &oid
))
1479 die(_("error in object: %s"), reader
->line
);
1480 if (unregister_shallow(&oid
))
1481 die(_("no shallow found: %s"), reader
->line
);
1482 unshallow_received
= 1;
1485 die(_("expected shallow/unshallow, got %s"), reader
->line
);
1488 if (reader
->status
!= PACKET_READ_FLUSH
&&
1489 reader
->status
!= PACKET_READ_DELIM
)
1490 die(_("error processing shallow info: %d"), reader
->status
);
1492 if (args
->deepen
|| unshallow_received
) {
1494 * Treat these as shallow lines caused by our depth settings.
1495 * In v0, these lines cannot cause refs to be rejected; do the
1500 for (i
= 0; i
< shallows
->nr
; i
++)
1501 register_shallow(the_repository
, &shallows
->oid
[i
]);
1502 setup_alternate_shallow(&shallow_lock
, &alternate_shallow_file
,
1505 } else if (shallows
->nr
) {
1507 * Treat these as shallow lines caused by the remote being
1508 * shallow. In v0, remote refs that reach these objects are
1509 * rejected (unless --update-shallow is set); do the same.
1511 prepare_shallow_info(si
, shallows
);
1512 if (si
->nr_ours
|| si
->nr_theirs
) {
1513 if (args
->reject_shallow_remote
)
1514 die(_("source repository is shallow, reject to clone."));
1515 alternate_shallow_file
=
1516 setup_temporary_shallow(si
->shallow
);
1518 alternate_shallow_file
= NULL
;
1520 alternate_shallow_file
= NULL
;
1524 static int cmp_name_ref(const void *name
, const void *ref
)
1526 return strcmp(name
, (*(struct ref
**)ref
)->name
);
1529 static void receive_wanted_refs(struct packet_reader
*reader
,
1530 struct ref
**sought
, int nr_sought
)
1532 process_section_header(reader
, "wanted-refs", 0);
1533 while (packet_reader_read(reader
) == PACKET_READ_NORMAL
) {
1534 struct object_id oid
;
1538 if (parse_oid_hex(reader
->line
, &oid
, &end
) || *end
++ != ' ')
1539 die(_("expected wanted-ref, got '%s'"), reader
->line
);
1541 found
= bsearch(end
, sought
, nr_sought
, sizeof(*sought
),
1544 die(_("unexpected wanted-ref: '%s'"), reader
->line
);
1545 oidcpy(&(*found
)->old_oid
, &oid
);
1548 if (reader
->status
!= PACKET_READ_DELIM
)
1549 die(_("error processing wanted refs: %d"), reader
->status
);
1552 static void receive_packfile_uris(struct packet_reader
*reader
,
1553 struct string_list
*uris
)
1555 process_section_header(reader
, "packfile-uris", 0);
1556 while (packet_reader_read(reader
) == PACKET_READ_NORMAL
) {
1557 if (reader
->pktlen
< the_hash_algo
->hexsz
||
1558 reader
->line
[the_hash_algo
->hexsz
] != ' ')
1559 die("expected '<hash> <uri>', got: %s\n", reader
->line
);
1561 string_list_append(uris
, reader
->line
);
1563 if (reader
->status
!= PACKET_READ_DELIM
)
1564 die("expected DELIM");
1568 FETCH_CHECK_LOCAL
= 0,
1575 static void do_check_stateless_delimiter(int stateless_rpc
,
1576 struct packet_reader
*reader
)
1578 check_stateless_delimiter(stateless_rpc
, reader
,
1579 _("git fetch-pack: expected response end packet"));
1582 static struct ref
*do_fetch_pack_v2(struct fetch_pack_args
*args
,
1584 const struct ref
*orig_ref
,
1585 struct ref
**sought
, int nr_sought
,
1586 struct oid_array
*shallows
,
1587 struct shallow_info
*si
,
1588 struct string_list
*pack_lockfiles
)
1590 struct repository
*r
= the_repository
;
1591 struct ref
*ref
= copy_ref_list(orig_ref
);
1592 enum fetch_state state
= FETCH_CHECK_LOCAL
;
1593 struct oidset common
= OIDSET_INIT
;
1594 struct packet_reader reader
;
1595 int in_vain
= 0, negotiation_started
= 0;
1596 int haves_to_send
= INITIAL_FLUSH
;
1597 struct fetch_negotiator negotiator_alloc
;
1598 struct fetch_negotiator
*negotiator
;
1600 struct object_id common_oid
;
1601 int received_ready
= 0;
1602 struct string_list packfile_uris
= STRING_LIST_INIT_DUP
;
1604 struct strvec index_pack_args
= STRVEC_INIT
;
1606 negotiator
= &negotiator_alloc
;
1608 fetch_negotiator_init_noop(negotiator
);
1610 fetch_negotiator_init(r
, negotiator
);
1612 packet_reader_init(&reader
, fd
[0], NULL
, 0,
1613 PACKET_READ_CHOMP_NEWLINE
|
1614 PACKET_READ_DIE_ON_ERR_PACKET
);
1615 if (git_env_bool("GIT_TEST_SIDEBAND_ALL", 1) &&
1616 server_supports_feature("fetch", "sideband-all", 0)) {
1617 reader
.use_sideband
= 1;
1618 reader
.me
= "fetch-pack";
1621 while (state
!= FETCH_DONE
) {
1623 case FETCH_CHECK_LOCAL
:
1624 sort_ref_list(&ref
, ref_compare_name
);
1625 QSORT(sought
, nr_sought
, cmp_ref_by_name
);
1627 /* v2 supports these by default */
1628 allow_unadvertised_object_request
|= ALLOW_REACHABLE_SHA1
;
1630 if (args
->depth
> 0 || args
->deepen_since
|| args
->deepen_not
)
1633 /* Filter 'ref' by 'sought' and those that aren't local */
1634 mark_complete_and_common_ref(negotiator
, args
, &ref
);
1635 filter_refs(args
, &ref
, sought
, nr_sought
);
1636 if (!args
->refetch
&& everything_local(args
, &ref
))
1639 state
= FETCH_SEND_REQUEST
;
1641 mark_tips(negotiator
, args
->negotiation_tips
);
1642 for_each_cached_alternate(negotiator
,
1643 insert_one_alternate_object
);
1645 case FETCH_SEND_REQUEST
:
1646 if (!negotiation_started
) {
1647 negotiation_started
= 1;
1648 trace2_region_enter("fetch-pack",
1652 if (send_fetch_request(negotiator
, fd
[1], args
, ref
,
1654 &haves_to_send
, &in_vain
,
1655 reader
.use_sideband
,
1657 state
= FETCH_GET_PACK
;
1659 state
= FETCH_PROCESS_ACKS
;
1661 case FETCH_PROCESS_ACKS
:
1662 /* Process ACKs/NAKs */
1663 process_section_header(&reader
, "acknowledgments", 0);
1664 while (process_ack(negotiator
, &reader
, &common_oid
,
1668 oidset_insert(&common
, &common_oid
);
1670 if (received_ready
) {
1672 * Don't check for response delimiter; get_pack() will
1673 * read the rest of this response.
1675 state
= FETCH_GET_PACK
;
1677 do_check_stateless_delimiter(args
->stateless_rpc
, &reader
);
1678 state
= FETCH_SEND_REQUEST
;
1681 case FETCH_GET_PACK
:
1682 trace2_region_leave("fetch-pack",
1685 /* Check for shallow-info section */
1686 if (process_section_header(&reader
, "shallow-info", 1))
1687 receive_shallow_info(args
, &reader
, shallows
, si
);
1689 if (process_section_header(&reader
, "wanted-refs", 1))
1690 receive_wanted_refs(&reader
, sought
, nr_sought
);
1692 /* get the pack(s) */
1693 if (git_env_bool("GIT_TRACE_REDACT", 1))
1694 reader
.options
|= PACKET_READ_REDACT_URI_PATH
;
1695 if (process_section_header(&reader
, "packfile-uris", 1))
1696 receive_packfile_uris(&reader
, &packfile_uris
);
1697 /* We don't expect more URIs. Reset to avoid expensive URI check. */
1698 reader
.options
&= ~PACKET_READ_REDACT_URI_PATH
;
1700 process_section_header(&reader
, "packfile", 0);
1703 * this is the final request we'll make of the server;
1704 * do a half-duplex shutdown to indicate that they can
1705 * hang up as soon as the pack is sent.
1710 if (get_pack(args
, fd
, pack_lockfiles
,
1711 packfile_uris
.nr
? &index_pack_args
: NULL
,
1712 sought
, nr_sought
, &fsck_options
.gitmodules_found
))
1713 die(_("git fetch-pack: fetch failed."));
1714 do_check_stateless_delimiter(args
->stateless_rpc
, &reader
);
1723 for (i
= 0; i
< packfile_uris
.nr
; i
++) {
1725 struct child_process cmd
= CHILD_PROCESS_INIT
;
1726 char packname
[GIT_MAX_HEXSZ
+ 1];
1727 const char *uri
= packfile_uris
.items
[i
].string
+
1728 the_hash_algo
->hexsz
+ 1;
1730 strvec_push(&cmd
.args
, "http-fetch");
1731 strvec_pushf(&cmd
.args
, "--packfile=%.*s",
1732 (int) the_hash_algo
->hexsz
,
1733 packfile_uris
.items
[i
].string
);
1734 for (j
= 0; j
< index_pack_args
.nr
; j
++)
1735 strvec_pushf(&cmd
.args
, "--index-pack-arg=%s",
1736 index_pack_args
.v
[j
]);
1737 strvec_push(&cmd
.args
, uri
);
1741 if (start_command(&cmd
))
1742 die("fetch-pack: unable to spawn http-fetch");
1744 if (read_in_full(cmd
.out
, packname
, 5) < 0 ||
1745 memcmp(packname
, "keep\t", 5))
1746 die("fetch-pack: expected keep then TAB at start of http-fetch output");
1748 if (read_in_full(cmd
.out
, packname
,
1749 the_hash_algo
->hexsz
+ 1) < 0 ||
1750 packname
[the_hash_algo
->hexsz
] != '\n')
1751 die("fetch-pack: expected hash then LF at end of http-fetch output");
1753 packname
[the_hash_algo
->hexsz
] = '\0';
1755 parse_gitmodules_oids(cmd
.out
, &fsck_options
.gitmodules_found
);
1759 if (finish_command(&cmd
))
1760 die("fetch-pack: unable to finish http-fetch");
1762 if (memcmp(packfile_uris
.items
[i
].string
, packname
,
1763 the_hash_algo
->hexsz
))
1764 die("fetch-pack: pack downloaded from %s does not match expected hash %.*s",
1765 uri
, (int) the_hash_algo
->hexsz
,
1766 packfile_uris
.items
[i
].string
);
1768 string_list_append_nodup(pack_lockfiles
,
1769 xstrfmt("%s/pack/pack-%s.keep",
1770 get_object_directory(),
1773 string_list_clear(&packfile_uris
, 0);
1774 strvec_clear(&index_pack_args
);
1776 if (fsck_finish(&fsck_options
))
1780 negotiator
->release(negotiator
);
1782 oidset_clear(&common
);
1786 static int fetch_pack_config_cb(const char *var
, const char *value
, void *cb
)
1788 if (strcmp(var
, "fetch.fsck.skiplist") == 0) {
1791 if (git_config_pathname(&path
, var
, value
))
1793 strbuf_addf(&fsck_msg_types
, "%cskiplist=%s",
1794 fsck_msg_types
.len
? ',' : '=', path
);
1799 if (skip_prefix(var
, "fetch.fsck.", &var
)) {
1800 if (is_valid_msg_type(var
, value
))
1801 strbuf_addf(&fsck_msg_types
, "%c%s=%s",
1802 fsck_msg_types
.len
? ',' : '=', var
, value
);
1804 warning("Skipping unknown msg id '%s'", var
);
1808 return git_default_config(var
, value
, cb
);
1811 static void fetch_pack_config(void)
1813 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit
);
1814 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit
);
1815 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta
);
1816 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects
);
1817 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects
);
1818 git_config_get_bool("transfer.advertisesid", &advertise_sid
);
1819 if (!uri_protocols
.nr
) {
1822 if (!git_config_get_string("fetch.uriprotocols", &str
) && str
) {
1823 string_list_split(&uri_protocols
, str
, ',', -1);
1828 git_config(fetch_pack_config_cb
, NULL
);
1831 static void fetch_pack_setup(void)
1833 static int did_setup
;
1836 fetch_pack_config();
1837 if (0 <= transfer_unpack_limit
)
1838 unpack_limit
= transfer_unpack_limit
;
1839 else if (0 <= fetch_unpack_limit
)
1840 unpack_limit
= fetch_unpack_limit
;
1844 static int remove_duplicates_in_refs(struct ref
**ref
, int nr
)
1846 struct string_list names
= STRING_LIST_INIT_NODUP
;
1849 for (src
= dst
= 0; src
< nr
; src
++) {
1850 struct string_list_item
*item
;
1851 item
= string_list_insert(&names
, ref
[src
]->name
);
1853 continue; /* already have it */
1854 item
->util
= ref
[src
];
1856 ref
[dst
] = ref
[src
];
1859 for (src
= dst
; src
< nr
; src
++)
1861 string_list_clear(&names
, 0);
1865 static void update_shallow(struct fetch_pack_args
*args
,
1866 struct ref
**sought
, int nr_sought
,
1867 struct shallow_info
*si
)
1869 struct oid_array ref
= OID_ARRAY_INIT
;
1873 if (args
->deepen
&& alternate_shallow_file
) {
1874 if (*alternate_shallow_file
== '\0') { /* --unshallow */
1875 unlink_or_warn(git_path_shallow(the_repository
));
1876 rollback_shallow_file(the_repository
, &shallow_lock
);
1878 commit_shallow_file(the_repository
, &shallow_lock
);
1879 alternate_shallow_file
= NULL
;
1883 if (!si
->shallow
|| !si
->shallow
->nr
)
1886 if (args
->cloning
) {
1888 * remote is shallow, but this is a clone, there are
1889 * no objects in repo to worry about. Accept any
1890 * shallow points that exist in the pack (iow in repo
1891 * after get_pack() and reprepare_packed_git())
1893 struct oid_array extra
= OID_ARRAY_INIT
;
1894 struct object_id
*oid
= si
->shallow
->oid
;
1895 for (i
= 0; i
< si
->shallow
->nr
; i
++)
1896 if (has_object_file(&oid
[i
]))
1897 oid_array_append(&extra
, &oid
[i
]);
1899 setup_alternate_shallow(&shallow_lock
,
1900 &alternate_shallow_file
,
1902 commit_shallow_file(the_repository
, &shallow_lock
);
1903 alternate_shallow_file
= NULL
;
1905 oid_array_clear(&extra
);
1909 if (!si
->nr_ours
&& !si
->nr_theirs
)
1912 remove_nonexistent_theirs_shallow(si
);
1913 if (!si
->nr_ours
&& !si
->nr_theirs
)
1915 for (i
= 0; i
< nr_sought
; i
++)
1916 oid_array_append(&ref
, &sought
[i
]->old_oid
);
1919 if (args
->update_shallow
) {
1921 * remote is also shallow, .git/shallow may be updated
1922 * so all refs can be accepted. Make sure we only add
1923 * shallow roots that are actually reachable from new
1926 struct oid_array extra
= OID_ARRAY_INIT
;
1927 struct object_id
*oid
= si
->shallow
->oid
;
1928 assign_shallow_commits_to_refs(si
, NULL
, NULL
);
1929 if (!si
->nr_ours
&& !si
->nr_theirs
) {
1930 oid_array_clear(&ref
);
1933 for (i
= 0; i
< si
->nr_ours
; i
++)
1934 oid_array_append(&extra
, &oid
[si
->ours
[i
]]);
1935 for (i
= 0; i
< si
->nr_theirs
; i
++)
1936 oid_array_append(&extra
, &oid
[si
->theirs
[i
]]);
1937 setup_alternate_shallow(&shallow_lock
,
1938 &alternate_shallow_file
,
1940 commit_shallow_file(the_repository
, &shallow_lock
);
1941 oid_array_clear(&extra
);
1942 oid_array_clear(&ref
);
1943 alternate_shallow_file
= NULL
;
1948 * remote is also shallow, check what ref is safe to update
1949 * without updating .git/shallow
1951 CALLOC_ARRAY(status
, nr_sought
);
1952 assign_shallow_commits_to_refs(si
, NULL
, status
);
1953 if (si
->nr_ours
|| si
->nr_theirs
) {
1954 for (i
= 0; i
< nr_sought
; i
++)
1956 sought
[i
]->status
= REF_STATUS_REJECT_SHALLOW
;
1959 oid_array_clear(&ref
);
1962 static const struct object_id
*iterate_ref_map(void *cb_data
)
1964 struct ref
**rm
= cb_data
;
1965 struct ref
*ref
= *rm
;
1970 return &ref
->old_oid
;
1973 struct ref
*fetch_pack(struct fetch_pack_args
*args
,
1975 const struct ref
*ref
,
1976 struct ref
**sought
, int nr_sought
,
1977 struct oid_array
*shallow
,
1978 struct string_list
*pack_lockfiles
,
1979 enum protocol_version version
)
1981 struct ref
*ref_cpy
;
1982 struct shallow_info si
;
1983 struct oid_array shallows_scratch
= OID_ARRAY_INIT
;
1987 nr_sought
= remove_duplicates_in_refs(sought
, nr_sought
);
1989 if (version
!= protocol_v2
&& !ref
) {
1990 packet_flush(fd
[1]);
1991 die(_("no matching remote head"));
1993 if (version
== protocol_v2
) {
1995 BUG("Protocol V2 does not provide shallows at this point in the fetch");
1996 memset(&si
, 0, sizeof(si
));
1997 ref_cpy
= do_fetch_pack_v2(args
, fd
, ref
, sought
, nr_sought
,
1998 &shallows_scratch
, &si
,
2001 prepare_shallow_info(&si
, shallow
);
2002 ref_cpy
= do_fetch_pack(args
, fd
, ref
, sought
, nr_sought
,
2003 &si
, pack_lockfiles
);
2005 reprepare_packed_git(the_repository
);
2007 if (!args
->cloning
&& args
->deepen
) {
2008 struct check_connected_options opt
= CHECK_CONNECTED_INIT
;
2009 struct ref
*iterator
= ref_cpy
;
2010 opt
.shallow_file
= alternate_shallow_file
;
2012 opt
.is_deepening_fetch
= 1;
2013 if (check_connected(iterate_ref_map
, &iterator
, &opt
)) {
2014 error(_("remote did not send all necessary objects"));
2017 rollback_shallow_file(the_repository
, &shallow_lock
);
2020 args
->connectivity_checked
= 1;
2023 update_shallow(args
, sought
, nr_sought
, &si
);
2025 clear_shallow_info(&si
);
2026 oid_array_clear(&shallows_scratch
);
2030 static int add_to_object_array(const struct object_id
*oid
, void *data
)
2032 struct object_array
*a
= data
;
2034 add_object_array(lookup_object(the_repository
, oid
), "", a
);
2038 static void clear_common_flag(struct oidset
*s
)
2040 struct oidset_iter iter
;
2041 const struct object_id
*oid
;
2042 oidset_iter_init(s
, &iter
);
2044 while ((oid
= oidset_iter_next(&iter
))) {
2045 struct object
*obj
= lookup_object(the_repository
, oid
);
2046 obj
->flags
&= ~COMMON
;
2050 void negotiate_using_fetch(const struct oid_array
*negotiation_tips
,
2051 const struct string_list
*server_options
,
2054 struct oidset
*acked_commits
)
2056 struct fetch_negotiator negotiator
;
2057 struct packet_reader reader
;
2058 struct object_array nt_object_array
= OBJECT_ARRAY_INIT
;
2059 struct strbuf req_buf
= STRBUF_INIT
;
2060 int haves_to_send
= INITIAL_FLUSH
;
2063 int last_iteration
= 0;
2064 timestamp_t min_generation
= GENERATION_NUMBER_INFINITY
;
2066 fetch_negotiator_init(the_repository
, &negotiator
);
2067 mark_tips(&negotiator
, negotiation_tips
);
2069 packet_reader_init(&reader
, fd
[0], NULL
, 0,
2070 PACKET_READ_CHOMP_NEWLINE
|
2071 PACKET_READ_DIE_ON_ERR_PACKET
);
2073 oid_array_for_each((struct oid_array
*) negotiation_tips
,
2074 add_to_object_array
,
2077 while (!last_iteration
) {
2079 struct object_id common_oid
;
2080 int received_ready
= 0;
2082 strbuf_reset(&req_buf
);
2083 write_fetch_command_and_capabilities(&req_buf
, server_options
);
2085 packet_buf_write(&req_buf
, "wait-for-done");
2087 haves_added
= add_haves(&negotiator
, &req_buf
, &haves_to_send
);
2088 in_vain
+= haves_added
;
2089 if (!haves_added
|| (seen_ack
&& in_vain
>= MAX_IN_VAIN
))
2093 packet_buf_flush(&req_buf
);
2094 if (write_in_full(fd
[1], req_buf
.buf
, req_buf
.len
) < 0)
2095 die_errno(_("unable to write request to remote"));
2097 /* Process ACKs/NAKs */
2098 process_section_header(&reader
, "acknowledgments", 0);
2099 while (process_ack(&negotiator
, &reader
, &common_oid
,
2101 struct commit
*commit
= lookup_commit(the_repository
,
2104 timestamp_t generation
;
2106 parse_commit_or_die(commit
);
2107 commit
->object
.flags
|= COMMON
;
2108 generation
= commit_graph_generation(commit
);
2109 if (generation
< min_generation
)
2110 min_generation
= generation
;
2114 oidset_insert(acked_commits
, &common_oid
);
2117 die(_("unexpected 'ready' from remote"));
2119 do_check_stateless_delimiter(stateless_rpc
, &reader
);
2120 if (can_all_from_reach_with_flag(&nt_object_array
, COMMON
,
2125 clear_common_flag(acked_commits
);
2126 strbuf_release(&req_buf
);
2129 int report_unmatched_refs(struct ref
**sought
, int nr_sought
)
2133 for (i
= 0; i
< nr_sought
; i
++) {
2136 switch (sought
[i
]->match_status
) {
2139 case REF_NOT_MATCHED
:
2140 error(_("no such remote ref %s"), sought
[i
]->name
);
2142 case REF_UNADVERTISED_NOT_ALLOWED
:
2143 error(_("Server does not allow request for unadvertised object %s"),