9 #include "fetch-pack.h"
11 #include "run-command.h"
13 static int transfer_unpack_limit
= -1;
14 static int fetch_unpack_limit
= -1;
15 static int unpack_limit
= 100;
16 static int prefer_ofs_delta
= 1;
17 static struct fetch_pack_args args
= {
18 /* .uploadpack = */ "git-upload-pack",
21 static const char fetch_pack_usage
[] =
22 "git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
24 #define COMPLETE (1U << 0)
25 #define COMMON (1U << 1)
26 #define COMMON_REF (1U << 2)
27 #define SEEN (1U << 3)
28 #define POPPED (1U << 4)
33 * After sending this many "have"s if we do not get any new ACK , we
34 * give up traversing our history.
36 #define MAX_IN_VAIN 256
38 static struct commit_list
*rev_list
;
39 static int non_common_revs
, multi_ack
, use_sideband
;
41 static void rev_list_push(struct commit
*commit
, int mark
)
43 if (!(commit
->object
.flags
& mark
)) {
44 commit
->object
.flags
|= mark
;
46 if (!(commit
->object
.parsed
))
47 if (parse_commit(commit
))
50 commit_list_insert_by_date(commit
, &rev_list
);
52 if (!(commit
->object
.flags
& COMMON
))
57 static int rev_list_insert_ref(const char *path
, const unsigned char *sha1
, int flag
, void *cb_data
)
59 struct object
*o
= deref_tag(parse_object(sha1
), path
, 0);
61 if (o
&& o
->type
== OBJ_COMMIT
)
62 rev_list_push((struct commit
*)o
, SEEN
);
67 static int clear_marks(const char *path
, const unsigned char *sha1
, int flag
, void *cb_data
)
69 struct object
*o
= deref_tag(parse_object(sha1
), path
, 0);
71 if (o
&& o
->type
== OBJ_COMMIT
)
72 clear_commit_marks((struct commit
*)o
,
73 COMMON
| COMMON_REF
| SEEN
| POPPED
);
78 This function marks a rev and its ancestors as common.
79 In some cases, it is desirable to mark only the ancestors (for example
80 when only the server does not yet know that they are common).
83 static void mark_common(struct commit
*commit
,
84 int ancestors_only
, int dont_parse
)
86 if (commit
!= NULL
&& !(commit
->object
.flags
& COMMON
)) {
87 struct object
*o
= (struct object
*)commit
;
92 if (!(o
->flags
& SEEN
))
93 rev_list_push(commit
, SEEN
);
95 struct commit_list
*parents
;
97 if (!ancestors_only
&& !(o
->flags
& POPPED
))
99 if (!o
->parsed
&& !dont_parse
)
100 if (parse_commit(commit
))
103 for (parents
= commit
->parents
;
105 parents
= parents
->next
)
106 mark_common(parents
->item
, 0, dont_parse
);
112 Get the next rev to send, ignoring the common.
115 static const unsigned char *get_rev(void)
117 struct commit
*commit
= NULL
;
119 while (commit
== NULL
) {
121 struct commit_list
*parents
;
123 if (rev_list
== NULL
|| non_common_revs
== 0)
126 commit
= rev_list
->item
;
127 if (!commit
->object
.parsed
)
128 parse_commit(commit
);
129 parents
= commit
->parents
;
131 commit
->object
.flags
|= POPPED
;
132 if (!(commit
->object
.flags
& COMMON
))
135 if (commit
->object
.flags
& COMMON
) {
136 /* do not send "have", and ignore ancestors */
138 mark
= COMMON
| SEEN
;
139 } else if (commit
->object
.flags
& COMMON_REF
)
140 /* send "have", and ignore ancestors */
141 mark
= COMMON
| SEEN
;
143 /* send "have", also for its ancestors */
147 if (!(parents
->item
->object
.flags
& SEEN
))
148 rev_list_push(parents
->item
, mark
);
150 mark_common(parents
->item
, 1, 0);
151 parents
= parents
->next
;
154 rev_list
= rev_list
->next
;
157 return commit
->object
.sha1
;
168 static void consume_shallow_list(int fd
)
170 if (args
.stateless_rpc
&& args
.depth
> 0) {
171 /* If we sent a depth we will get back "duplicate"
172 * shallow and unshallow commands every time there
173 * is a block of have lines exchanged.
176 while (packet_read_line(fd
, line
, sizeof(line
))) {
177 if (!prefixcmp(line
, "shallow "))
179 if (!prefixcmp(line
, "unshallow "))
181 die("git fetch-pack: expected shallow list");
186 static enum ack_type
get_ack(int fd
, unsigned char *result_sha1
)
188 static char line
[1000];
189 int len
= packet_read_line(fd
, line
, sizeof(line
));
192 die("git fetch-pack: expected ACK/NAK, got EOF");
193 if (line
[len
-1] == '\n')
195 if (!strcmp(line
, "NAK"))
197 if (!prefixcmp(line
, "ACK ")) {
198 if (!get_sha1_hex(line
+4, result_sha1
)) {
199 if (strstr(line
+45, "continue"))
201 if (strstr(line
+45, "common"))
203 if (strstr(line
+45, "ready"))
208 die("git fetch_pack: expected ACK/NAK, got '%s'", line
);
211 static void send_request(int fd
, struct strbuf
*buf
)
213 if (args
.stateless_rpc
) {
214 send_sideband(fd
, -1, buf
->buf
, buf
->len
, LARGE_PACKET_MAX
);
217 safe_write(fd
, buf
->buf
, buf
->len
);
220 static int find_common(int fd
[2], unsigned char *result_sha1
,
224 int count
= 0, flushes
= 0, retval
;
225 const unsigned char *sha1
;
226 unsigned in_vain
= 0;
227 int got_continue
= 0;
228 struct strbuf req_buf
= STRBUF_INIT
;
229 size_t state_len
= 0;
231 if (args
.stateless_rpc
&& multi_ack
== 1)
232 die("--stateless-rpc requires multi_ack_detailed");
234 for_each_ref(clear_marks
, NULL
);
237 for_each_ref(rev_list_insert_ref
, NULL
);
240 for ( ; refs
; refs
= refs
->next
) {
241 unsigned char *remote
= refs
->old_sha1
;
242 const char *remote_hex
;
246 * If that object is complete (i.e. it is an ancestor of a
247 * local ref), we tell them we have it but do not have to
248 * tell them about its ancestors, which they already know
251 * We use lookup_object here because we are only
252 * interested in the case we *know* the object is
253 * reachable and we have already scanned it.
255 if (((o
= lookup_object(remote
)) != NULL
) &&
256 (o
->flags
& COMPLETE
)) {
260 remote_hex
= sha1_to_hex(remote
);
262 struct strbuf c
= STRBUF_INIT
;
263 if (multi_ack
== 2) strbuf_addstr(&c
, " multi_ack_detailed");
264 if (multi_ack
== 1) strbuf_addstr(&c
, " multi_ack");
265 if (use_sideband
== 2) strbuf_addstr(&c
, " side-band-64k");
266 if (use_sideband
== 1) strbuf_addstr(&c
, " side-band");
267 if (args
.use_thin_pack
) strbuf_addstr(&c
, " thin-pack");
268 if (args
.no_progress
) strbuf_addstr(&c
, " no-progress");
269 if (args
.include_tag
) strbuf_addstr(&c
, " include-tag");
270 if (prefer_ofs_delta
) strbuf_addstr(&c
, " ofs-delta");
271 packet_buf_write(&req_buf
, "want %s%s\n", remote_hex
, c
.buf
);
274 packet_buf_write(&req_buf
, "want %s\n", remote_hex
);
279 strbuf_release(&req_buf
);
284 if (is_repository_shallow())
285 write_shallow_commits(&req_buf
, 1);
287 packet_buf_write(&req_buf
, "deepen %d", args
.depth
);
288 packet_buf_flush(&req_buf
);
289 state_len
= req_buf
.len
;
291 if (args
.depth
> 0) {
293 unsigned char sha1
[20];
295 send_request(fd
[1], &req_buf
);
296 while (packet_read_line(fd
[0], line
, sizeof(line
))) {
297 if (!prefixcmp(line
, "shallow ")) {
298 if (get_sha1_hex(line
+ 8, sha1
))
299 die("invalid shallow line: %s", line
);
300 register_shallow(sha1
);
303 if (!prefixcmp(line
, "unshallow ")) {
304 if (get_sha1_hex(line
+ 10, sha1
))
305 die("invalid unshallow line: %s", line
);
306 if (!lookup_object(sha1
))
307 die("object not found: %s", line
);
308 /* make sure that it is parsed as shallow */
309 if (!parse_object(sha1
))
310 die("error in object: %s", line
);
311 if (unregister_shallow(sha1
))
312 die("no shallow found: %s", line
);
315 die("expected shallow/unshallow, got %s", line
);
317 } else if (!args
.stateless_rpc
)
318 send_request(fd
[1], &req_buf
);
320 if (!args
.stateless_rpc
) {
321 /* If we aren't using the stateless-rpc interface
322 * we don't need to retain the headers.
324 strbuf_setlen(&req_buf
, 0);
330 while ((sha1
= get_rev())) {
331 packet_buf_write(&req_buf
, "have %s\n", sha1_to_hex(sha1
));
333 fprintf(stderr
, "have %s\n", sha1_to_hex(sha1
));
335 if (!(31 & ++count
)) {
338 packet_buf_flush(&req_buf
);
339 send_request(fd
[1], &req_buf
);
340 strbuf_setlen(&req_buf
, state_len
);
344 * We keep one window "ahead" of the other side, and
345 * will wait for an ACK only on the next one
347 if (!args
.stateless_rpc
&& count
== 32)
350 consume_shallow_list(fd
[0]);
352 ack
= get_ack(fd
[0], result_sha1
);
353 if (args
.verbose
&& ack
)
354 fprintf(stderr
, "got ack %d %s\n", ack
,
355 sha1_to_hex(result_sha1
));
365 struct commit
*commit
=
366 lookup_commit(result_sha1
);
367 if (args
.stateless_rpc
369 && !(commit
->object
.flags
& COMMON
)) {
370 /* We need to replay the have for this object
371 * on the next RPC request so the peer knows
372 * it is in common with us.
374 const char *hex
= sha1_to_hex(result_sha1
);
375 packet_buf_write(&req_buf
, "have %s\n", hex
);
376 state_len
= req_buf
.len
;
378 mark_common(commit
, 0, 1);
387 if (got_continue
&& MAX_IN_VAIN
< in_vain
) {
389 fprintf(stderr
, "giving up\n");
395 packet_buf_write(&req_buf
, "done\n");
396 send_request(fd
[1], &req_buf
);
398 fprintf(stderr
, "done\n");
403 strbuf_release(&req_buf
);
405 consume_shallow_list(fd
[0]);
406 while (flushes
|| multi_ack
) {
407 int ack
= get_ack(fd
[0], result_sha1
);
410 fprintf(stderr
, "got ack (%d) %s\n", ack
,
411 sha1_to_hex(result_sha1
));
419 /* it is no error to fetch into a completely empty repo */
420 return count
? retval
: 0;
423 static struct commit_list
*complete
;
425 static int mark_complete(const char *path
, const unsigned char *sha1
, int flag
, void *cb_data
)
427 struct object
*o
= parse_object(sha1
);
429 while (o
&& o
->type
== OBJ_TAG
) {
430 struct tag
*t
= (struct tag
*) o
;
432 break; /* broken repository */
433 o
->flags
|= COMPLETE
;
434 o
= parse_object(t
->tagged
->sha1
);
436 if (o
&& o
->type
== OBJ_COMMIT
) {
437 struct commit
*commit
= (struct commit
*)o
;
438 commit
->object
.flags
|= COMPLETE
;
439 commit_list_insert_by_date(commit
, &complete
);
444 static void mark_recent_complete_commits(unsigned long cutoff
)
446 while (complete
&& cutoff
<= complete
->item
->date
) {
448 fprintf(stderr
, "Marking %s as complete\n",
449 sha1_to_hex(complete
->item
->object
.sha1
));
450 pop_most_recent_commit(&complete
, COMPLETE
);
454 static void filter_refs(struct ref
**refs
, int nr_match
, char **match
)
456 struct ref
**return_refs
;
457 struct ref
*newlist
= NULL
;
458 struct ref
**newtail
= &newlist
;
459 struct ref
*ref
, *next
;
460 struct ref
*fastarray
[32];
462 if (nr_match
&& !args
.fetch_all
) {
463 if (ARRAY_SIZE(fastarray
) < nr_match
)
464 return_refs
= xcalloc(nr_match
, sizeof(struct ref
*));
466 return_refs
= fastarray
;
467 memset(return_refs
, 0, sizeof(struct ref
*) * nr_match
);
473 for (ref
= *refs
; ref
; ref
= next
) {
475 if (!memcmp(ref
->name
, "refs/", 5) &&
476 check_ref_format(ref
->name
+ 5))
478 else if (args
.fetch_all
&&
479 (!args
.depth
|| prefixcmp(ref
->name
, "refs/tags/") )) {
482 newtail
= &ref
->next
;
486 int order
= path_match(ref
->name
, nr_match
, match
);
488 return_refs
[order
-1] = ref
;
489 continue; /* we will link it later */
495 if (!args
.fetch_all
) {
497 for (i
= 0; i
< nr_match
; i
++) {
498 ref
= return_refs
[i
];
502 newtail
= &ref
->next
;
505 if (return_refs
!= fastarray
)
511 static int everything_local(struct ref
**refs
, int nr_match
, char **match
)
515 unsigned long cutoff
= 0;
517 save_commit_buffer
= 0;
519 for (ref
= *refs
; ref
; ref
= ref
->next
) {
522 o
= parse_object(ref
->old_sha1
);
526 /* We already have it -- which may mean that we were
527 * in sync with the other side at some time after
528 * that (it is OK if we guess wrong here).
530 if (o
->type
== OBJ_COMMIT
) {
531 struct commit
*commit
= (struct commit
*)o
;
532 if (!cutoff
|| cutoff
< commit
->date
)
533 cutoff
= commit
->date
;
538 for_each_ref(mark_complete
, NULL
);
540 mark_recent_complete_commits(cutoff
);
544 * Mark all complete remote refs as common refs.
545 * Don't mark them common yet; the server has to be told so first.
547 for (ref
= *refs
; ref
; ref
= ref
->next
) {
548 struct object
*o
= deref_tag(lookup_object(ref
->old_sha1
),
551 if (!o
|| o
->type
!= OBJ_COMMIT
|| !(o
->flags
& COMPLETE
))
554 if (!(o
->flags
& SEEN
)) {
555 rev_list_push((struct commit
*)o
, COMMON_REF
| SEEN
);
557 mark_common((struct commit
*)o
, 1, 1);
561 filter_refs(refs
, nr_match
, match
);
563 for (retval
= 1, ref
= *refs
; ref
; ref
= ref
->next
) {
564 const unsigned char *remote
= ref
->old_sha1
;
565 unsigned char local
[20];
568 o
= lookup_object(remote
);
569 if (!o
|| !(o
->flags
& COMPLETE
)) {
574 "want %s (%s)\n", sha1_to_hex(remote
),
579 hashcpy(ref
->new_sha1
, local
);
583 "already have %s (%s)\n", sha1_to_hex(remote
),
589 static int sideband_demux(int in
, int out
, void *data
)
593 int ret
= recv_sideband("fetch-pack", xd
[0], out
);
598 static int get_pack(int xd
[2], char **pack_lockfile
)
601 const char *argv
[20];
605 int do_keep
= args
.keep_pack
;
606 struct child_process cmd
;
608 memset(&demux
, 0, sizeof(demux
));
610 /* xd[] is talking with upload-pack; subprocess reads from
611 * xd[0], spits out band#2 to stderr, and feeds us band#1
612 * through demux->out.
614 demux
.proc
= sideband_demux
;
617 if (start_async(&demux
))
618 die("fetch-pack: unable to fork off sideband"
624 memset(&cmd
, 0, sizeof(cmd
));
628 if (!args
.keep_pack
&& unpack_limit
) {
629 struct pack_header header
;
631 if (read_pack_header(demux
.out
, &header
))
632 die("protocol error: bad pack header");
633 snprintf(hdr_arg
, sizeof(hdr_arg
),
634 "--pack_header=%"PRIu32
",%"PRIu32
,
635 ntohl(header
.hdr_version
), ntohl(header
.hdr_entries
));
636 if (ntohl(header
.hdr_entries
) < unpack_limit
)
645 *av
++ = "index-pack";
647 if (!args
.quiet
&& !args
.no_progress
)
649 if (args
.use_thin_pack
)
650 *av
++ = "--fix-thin";
651 if (args
.lock_pack
|| unpack_limit
) {
652 int s
= sprintf(keep_arg
,
653 "--keep=fetch-pack %"PRIuMAX
" on ", (uintmax_t) getpid());
654 if (gethostname(keep_arg
+ s
, sizeof(keep_arg
) - s
))
655 strcpy(keep_arg
+ s
, "localhost");
660 *av
++ = "unpack-objects";
670 if (start_command(&cmd
))
671 die("fetch-pack: unable to fork off %s", argv
[0]);
672 if (do_keep
&& pack_lockfile
) {
673 *pack_lockfile
= index_pack_lockfile(cmd
.out
);
677 if (finish_command(&cmd
))
678 die("%s failed", argv
[0]);
679 if (use_sideband
&& finish_async(&demux
))
680 die("error in sideband demultiplexer");
684 static struct ref
*do_fetch_pack(int fd
[2],
685 const struct ref
*orig_ref
,
688 char **pack_lockfile
)
690 struct ref
*ref
= copy_ref_list(orig_ref
);
691 unsigned char sha1
[20];
693 if (is_repository_shallow() && !server_supports("shallow"))
694 die("Server does not support shallow clients");
695 if (server_supports("multi_ack_detailed")) {
697 fprintf(stderr
, "Server supports multi_ack_detailed\n");
700 else if (server_supports("multi_ack")) {
702 fprintf(stderr
, "Server supports multi_ack\n");
705 if (server_supports("side-band-64k")) {
707 fprintf(stderr
, "Server supports side-band-64k\n");
710 else if (server_supports("side-band")) {
712 fprintf(stderr
, "Server supports side-band\n");
715 if (server_supports("ofs-delta")) {
717 fprintf(stderr
, "Server supports ofs-delta\n");
719 prefer_ofs_delta
= 0;
720 if (everything_local(&ref
, nr_match
, match
)) {
724 if (find_common(fd
, sha1
, ref
) < 0)
726 /* When cloning, it is not unusual to have
729 warning("no common commits");
731 if (args
.stateless_rpc
)
733 if (get_pack(fd
, pack_lockfile
))
734 die("git fetch-pack: fetch failed.");
740 static int remove_duplicates(int nr_heads
, char **heads
)
744 for (src
= dst
= 0; src
< nr_heads
; src
++) {
745 /* If heads[src] is different from any of
746 * heads[0..dst], push it in.
749 for (i
= 0; i
< dst
; i
++) {
750 if (!strcmp(heads
[i
], heads
[src
]))
756 heads
[dst
] = heads
[src
];
762 static int fetch_pack_config(const char *var
, const char *value
, void *cb
)
764 if (strcmp(var
, "fetch.unpacklimit") == 0) {
765 fetch_unpack_limit
= git_config_int(var
, value
);
769 if (strcmp(var
, "transfer.unpacklimit") == 0) {
770 transfer_unpack_limit
= git_config_int(var
, value
);
774 if (strcmp(var
, "repack.usedeltabaseoffset") == 0) {
775 prefer_ofs_delta
= git_config_bool(var
, value
);
779 return git_default_config(var
, value
, cb
);
782 static struct lock_file lock
;
784 static void fetch_pack_setup(void)
786 static int did_setup
;
789 git_config(fetch_pack_config
, NULL
);
790 if (0 <= transfer_unpack_limit
)
791 unpack_limit
= transfer_unpack_limit
;
792 else if (0 <= fetch_unpack_limit
)
793 unpack_limit
= fetch_unpack_limit
;
797 int cmd_fetch_pack(int argc
, const char **argv
, const char *prefix
)
799 int i
, ret
, nr_heads
;
800 struct ref
*ref
= NULL
;
801 char *dest
= NULL
, **heads
;
803 char *pack_lockfile
= NULL
;
804 char **pack_lockfile_ptr
= NULL
;
805 struct child_process
*conn
;
807 packet_trace_identity("fetch-pack");
811 for (i
= 1; i
< argc
; i
++) {
812 const char *arg
= argv
[i
];
815 if (!prefixcmp(arg
, "--upload-pack=")) {
816 args
.uploadpack
= arg
+ 14;
819 if (!prefixcmp(arg
, "--exec=")) {
820 args
.uploadpack
= arg
+ 7;
823 if (!strcmp("--quiet", arg
) || !strcmp("-q", arg
)) {
827 if (!strcmp("--keep", arg
) || !strcmp("-k", arg
)) {
828 args
.lock_pack
= args
.keep_pack
;
832 if (!strcmp("--thin", arg
)) {
833 args
.use_thin_pack
= 1;
836 if (!strcmp("--include-tag", arg
)) {
837 args
.include_tag
= 1;
840 if (!strcmp("--all", arg
)) {
844 if (!strcmp("-v", arg
)) {
848 if (!prefixcmp(arg
, "--depth=")) {
849 args
.depth
= strtol(arg
+ 8, NULL
, 0);
852 if (!strcmp("--no-progress", arg
)) {
853 args
.no_progress
= 1;
856 if (!strcmp("--stateless-rpc", arg
)) {
857 args
.stateless_rpc
= 1;
860 if (!strcmp("--lock-pack", arg
)) {
862 pack_lockfile_ptr
= &pack_lockfile
;
865 usage(fetch_pack_usage
);
868 heads
= (char **)(argv
+ i
+ 1);
869 nr_heads
= argc
- i
- 1;
873 usage(fetch_pack_usage
);
875 if (args
.stateless_rpc
) {
880 conn
= git_connect(fd
, (char *)dest
, args
.uploadpack
,
881 args
.verbose
? CONNECT_VERBOSE
: 0);
884 get_remote_heads(fd
[0], &ref
, 0, NULL
, 0, NULL
);
886 ref
= fetch_pack(&args
, fd
, conn
, ref
, dest
,
887 nr_heads
, heads
, pack_lockfile_ptr
);
889 printf("lock %s\n", pack_lockfile
);
894 if (finish_connect(conn
))
898 if (!ret
&& nr_heads
) {
899 /* If the heads to pull were given, we should have
900 * consumed all of them by matching the remote.
901 * Otherwise, 'git fetch remote no-such-ref' would
902 * silently succeed without issuing an error.
904 for (i
= 0; i
< nr_heads
; i
++)
905 if (heads
[i
] && heads
[i
][0]) {
906 error("no such remote ref %s", heads
[i
]);
912 sha1_to_hex(ref
->old_sha1
), ref
->name
);
919 struct ref
*fetch_pack(struct fetch_pack_args
*my_args
,
920 int fd
[], struct child_process
*conn
,
921 const struct ref
*ref
,
925 char **pack_lockfile
)
931 if (&args
!= my_args
)
932 memcpy(&args
, my_args
, sizeof(args
));
933 if (args
.depth
> 0) {
934 if (stat(git_path("shallow"), &st
))
938 if (heads
&& nr_heads
)
939 nr_heads
= remove_duplicates(nr_heads
, heads
);
942 die("no matching remote head");
944 ref_cpy
= do_fetch_pack(fd
, ref
, nr_heads
, heads
, pack_lockfile
);
946 if (args
.depth
> 0) {
947 struct cache_time mtime
;
948 struct strbuf sb
= STRBUF_INIT
;
949 char *shallow
= git_path("shallow");
952 mtime
.sec
= st
.st_mtime
;
953 mtime
.nsec
= ST_MTIME_NSEC(st
);
954 if (stat(shallow
, &st
)) {
956 die("shallow file was removed during fetch");
957 } else if (st
.st_mtime
!= mtime
.sec
959 || ST_MTIME_NSEC(st
) != mtime
.nsec
962 die("shallow file was changed during fetch");
964 fd
= hold_lock_file_for_update(&lock
, shallow
,
966 if (!write_shallow_commits(&sb
, 0)
967 || write_in_full(fd
, sb
.buf
, sb
.len
) != sb
.len
) {
968 unlink_or_warn(shallow
);
969 rollback_lock_file(&lock
);
971 commit_lock_file(&lock
);
976 reprepare_packed_git();