9 #include "fetch-pack.h"
11 #include "run-command.h"
13 static int transfer_unpack_limit
= -1;
14 static int fetch_unpack_limit
= -1;
15 static int unpack_limit
= 100;
16 static int prefer_ofs_delta
= 1;
17 static struct fetch_pack_args args
= {
18 /* .uploadpack = */ "git-upload-pack",
21 static const char fetch_pack_usage
[] =
22 "git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
24 #define COMPLETE (1U << 0)
25 #define COMMON (1U << 1)
26 #define COMMON_REF (1U << 2)
27 #define SEEN (1U << 3)
28 #define POPPED (1U << 4)
33 * After sending this many "have"s if we do not get any new ACK , we
34 * give up traversing our history.
36 #define MAX_IN_VAIN 256
38 static struct commit_list
*rev_list
;
39 static int non_common_revs
, multi_ack
, use_sideband
;
41 static void rev_list_push(struct commit
*commit
, int mark
)
43 if (!(commit
->object
.flags
& mark
)) {
44 commit
->object
.flags
|= mark
;
46 if (!(commit
->object
.parsed
))
47 if (parse_commit(commit
))
50 insert_by_date(commit
, &rev_list
);
52 if (!(commit
->object
.flags
& COMMON
))
57 static int rev_list_insert_ref(const char *path
, const unsigned char *sha1
, int flag
, void *cb_data
)
59 struct object
*o
= deref_tag(parse_object(sha1
), path
, 0);
61 if (o
&& o
->type
== OBJ_COMMIT
)
62 rev_list_push((struct commit
*)o
, SEEN
);
67 static int clear_marks(const char *path
, const unsigned char *sha1
, int flag
, void *cb_data
)
69 struct object
*o
= deref_tag(parse_object(sha1
), path
, 0);
71 if (o
&& o
->type
== OBJ_COMMIT
)
72 clear_commit_marks((struct commit
*)o
,
73 COMMON
| COMMON_REF
| SEEN
| POPPED
);
78 This function marks a rev and its ancestors as common.
79 In some cases, it is desirable to mark only the ancestors (for example
80 when only the server does not yet know that they are common).
83 static void mark_common(struct commit
*commit
,
84 int ancestors_only
, int dont_parse
)
86 if (commit
!= NULL
&& !(commit
->object
.flags
& COMMON
)) {
87 struct object
*o
= (struct object
*)commit
;
92 if (!(o
->flags
& SEEN
))
93 rev_list_push(commit
, SEEN
);
95 struct commit_list
*parents
;
97 if (!ancestors_only
&& !(o
->flags
& POPPED
))
99 if (!o
->parsed
&& !dont_parse
)
100 if (parse_commit(commit
))
103 for (parents
= commit
->parents
;
105 parents
= parents
->next
)
106 mark_common(parents
->item
, 0, dont_parse
);
112 Get the next rev to send, ignoring the common.
115 static const unsigned char *get_rev(void)
117 struct commit
*commit
= NULL
;
119 while (commit
== NULL
) {
121 struct commit_list
*parents
;
123 if (rev_list
== NULL
|| non_common_revs
== 0)
126 commit
= rev_list
->item
;
127 if (!commit
->object
.parsed
)
128 parse_commit(commit
);
129 parents
= commit
->parents
;
131 commit
->object
.flags
|= POPPED
;
132 if (!(commit
->object
.flags
& COMMON
))
135 if (commit
->object
.flags
& COMMON
) {
136 /* do not send "have", and ignore ancestors */
138 mark
= COMMON
| SEEN
;
139 } else if (commit
->object
.flags
& COMMON_REF
)
140 /* send "have", and ignore ancestors */
141 mark
= COMMON
| SEEN
;
143 /* send "have", also for its ancestors */
147 if (!(parents
->item
->object
.flags
& SEEN
))
148 rev_list_push(parents
->item
, mark
);
150 mark_common(parents
->item
, 1, 0);
151 parents
= parents
->next
;
154 rev_list
= rev_list
->next
;
157 return commit
->object
.sha1
;
160 static int find_common(int fd
[2], unsigned char *result_sha1
,
164 int count
= 0, flushes
= 0, retval
;
165 const unsigned char *sha1
;
166 unsigned in_vain
= 0;
167 int got_continue
= 0;
168 struct strbuf req_buf
= STRBUF_INIT
;
171 for_each_ref(clear_marks
, NULL
);
174 for_each_ref(rev_list_insert_ref
, NULL
);
177 for ( ; refs
; refs
= refs
->next
) {
178 unsigned char *remote
= refs
->old_sha1
;
179 const char *remote_hex
;
183 * If that object is complete (i.e. it is an ancestor of a
184 * local ref), we tell them we have it but do not have to
185 * tell them about its ancestors, which they already know
188 * We use lookup_object here because we are only
189 * interested in the case we *know* the object is
190 * reachable and we have already scanned it.
192 if (((o
= lookup_object(remote
)) != NULL
) &&
193 (o
->flags
& COMPLETE
)) {
197 remote_hex
= sha1_to_hex(remote
);
199 struct strbuf c
= STRBUF_INIT
;
200 if (multi_ack
) strbuf_addstr(&c
, " multi_ack");
201 if (use_sideband
== 2) strbuf_addstr(&c
, " side-band-64k");
202 if (use_sideband
== 1) strbuf_addstr(&c
, " side-band");
203 if (args
.use_thin_pack
) strbuf_addstr(&c
, " thin-pack");
204 if (args
.no_progress
) strbuf_addstr(&c
, " no-progress");
205 if (args
.include_tag
) strbuf_addstr(&c
, " include-tag");
206 if (prefer_ofs_delta
) strbuf_addstr(&c
, " ofs-delta");
207 packet_buf_write(&req_buf
, "want %s%s\n", remote_hex
, c
.buf
);
210 packet_buf_write(&req_buf
, "want %s\n", remote_hex
);
215 strbuf_release(&req_buf
);
220 if (is_repository_shallow())
221 write_shallow_commits(&req_buf
, 1);
223 packet_buf_write(&req_buf
, "deepen %d", args
.depth
);
224 packet_buf_flush(&req_buf
);
226 safe_write(fd
[1], req_buf
.buf
, req_buf
.len
);
228 if (args
.depth
> 0) {
230 unsigned char sha1
[20];
232 while (packet_read_line(fd
[0], line
, sizeof(line
))) {
233 if (!prefixcmp(line
, "shallow ")) {
234 if (get_sha1_hex(line
+ 8, sha1
))
235 die("invalid shallow line: %s", line
);
236 register_shallow(sha1
);
239 if (!prefixcmp(line
, "unshallow ")) {
240 if (get_sha1_hex(line
+ 10, sha1
))
241 die("invalid unshallow line: %s", line
);
242 if (!lookup_object(sha1
))
243 die("object not found: %s", line
);
244 /* make sure that it is parsed as shallow */
245 if (!parse_object(sha1
))
246 die("error in object: %s", line
);
247 if (unregister_shallow(sha1
))
248 die("no shallow found: %s", line
);
251 die("expected shallow/unshallow, got %s", line
);
257 while ((sha1
= get_rev())) {
258 packet_write(fd
[1], "have %s\n", sha1_to_hex(sha1
));
260 fprintf(stderr
, "have %s\n", sha1_to_hex(sha1
));
262 if (!(31 & ++count
)) {
269 * We keep one window "ahead" of the other side, and
270 * will wait for an ACK only on the next one
276 ack
= get_ack(fd
[0], result_sha1
);
277 if (args
.verbose
&& ack
)
278 fprintf(stderr
, "got ack %d %s\n", ack
,
279 sha1_to_hex(result_sha1
));
285 } else if (ack
== 2) {
286 struct commit
*commit
=
287 lookup_commit(result_sha1
);
288 mark_common(commit
, 0, 1);
295 if (got_continue
&& MAX_IN_VAIN
< in_vain
) {
297 fprintf(stderr
, "giving up\n");
303 packet_write(fd
[1], "done\n");
305 fprintf(stderr
, "done\n");
310 strbuf_release(&req_buf
);
312 while (flushes
|| multi_ack
) {
313 int ack
= get_ack(fd
[0], result_sha1
);
316 fprintf(stderr
, "got ack (%d) %s\n", ack
,
317 sha1_to_hex(result_sha1
));
325 /* it is no error to fetch into a completely empty repo */
326 return count
? retval
: 0;
329 static struct commit_list
*complete
;
331 static int mark_complete(const char *path
, const unsigned char *sha1
, int flag
, void *cb_data
)
333 struct object
*o
= parse_object(sha1
);
335 while (o
&& o
->type
== OBJ_TAG
) {
336 struct tag
*t
= (struct tag
*) o
;
338 break; /* broken repository */
339 o
->flags
|= COMPLETE
;
340 o
= parse_object(t
->tagged
->sha1
);
342 if (o
&& o
->type
== OBJ_COMMIT
) {
343 struct commit
*commit
= (struct commit
*)o
;
344 commit
->object
.flags
|= COMPLETE
;
345 insert_by_date(commit
, &complete
);
350 static void mark_recent_complete_commits(unsigned long cutoff
)
352 while (complete
&& cutoff
<= complete
->item
->date
) {
354 fprintf(stderr
, "Marking %s as complete\n",
355 sha1_to_hex(complete
->item
->object
.sha1
));
356 pop_most_recent_commit(&complete
, COMPLETE
);
360 static void filter_refs(struct ref
**refs
, int nr_match
, char **match
)
362 struct ref
**return_refs
;
363 struct ref
*newlist
= NULL
;
364 struct ref
**newtail
= &newlist
;
365 struct ref
*ref
, *next
;
366 struct ref
*fastarray
[32];
368 if (nr_match
&& !args
.fetch_all
) {
369 if (ARRAY_SIZE(fastarray
) < nr_match
)
370 return_refs
= xcalloc(nr_match
, sizeof(struct ref
*));
372 return_refs
= fastarray
;
373 memset(return_refs
, 0, sizeof(struct ref
*) * nr_match
);
379 for (ref
= *refs
; ref
; ref
= next
) {
381 if (!memcmp(ref
->name
, "refs/", 5) &&
382 check_ref_format(ref
->name
+ 5))
384 else if (args
.fetch_all
&&
385 (!args
.depth
|| prefixcmp(ref
->name
, "refs/tags/") )) {
388 newtail
= &ref
->next
;
392 int order
= path_match(ref
->name
, nr_match
, match
);
394 return_refs
[order
-1] = ref
;
395 continue; /* we will link it later */
401 if (!args
.fetch_all
) {
403 for (i
= 0; i
< nr_match
; i
++) {
404 ref
= return_refs
[i
];
408 newtail
= &ref
->next
;
411 if (return_refs
!= fastarray
)
417 static int everything_local(struct ref
**refs
, int nr_match
, char **match
)
421 unsigned long cutoff
= 0;
423 save_commit_buffer
= 0;
425 for (ref
= *refs
; ref
; ref
= ref
->next
) {
428 o
= parse_object(ref
->old_sha1
);
432 /* We already have it -- which may mean that we were
433 * in sync with the other side at some time after
434 * that (it is OK if we guess wrong here).
436 if (o
->type
== OBJ_COMMIT
) {
437 struct commit
*commit
= (struct commit
*)o
;
438 if (!cutoff
|| cutoff
< commit
->date
)
439 cutoff
= commit
->date
;
444 for_each_ref(mark_complete
, NULL
);
446 mark_recent_complete_commits(cutoff
);
450 * Mark all complete remote refs as common refs.
451 * Don't mark them common yet; the server has to be told so first.
453 for (ref
= *refs
; ref
; ref
= ref
->next
) {
454 struct object
*o
= deref_tag(lookup_object(ref
->old_sha1
),
457 if (!o
|| o
->type
!= OBJ_COMMIT
|| !(o
->flags
& COMPLETE
))
460 if (!(o
->flags
& SEEN
)) {
461 rev_list_push((struct commit
*)o
, COMMON_REF
| SEEN
);
463 mark_common((struct commit
*)o
, 1, 1);
467 filter_refs(refs
, nr_match
, match
);
469 for (retval
= 1, ref
= *refs
; ref
; ref
= ref
->next
) {
470 const unsigned char *remote
= ref
->old_sha1
;
471 unsigned char local
[20];
474 o
= lookup_object(remote
);
475 if (!o
|| !(o
->flags
& COMPLETE
)) {
480 "want %s (%s)\n", sha1_to_hex(remote
),
485 hashcpy(ref
->new_sha1
, local
);
489 "already have %s (%s)\n", sha1_to_hex(remote
),
495 static int sideband_demux(int fd
, void *data
)
499 int ret
= recv_sideband("fetch-pack", xd
[0], fd
);
504 static int get_pack(int xd
[2], char **pack_lockfile
)
507 const char *argv
[20];
511 int do_keep
= args
.keep_pack
;
512 struct child_process cmd
;
514 memset(&demux
, 0, sizeof(demux
));
516 /* xd[] is talking with upload-pack; subprocess reads from
517 * xd[0], spits out band#2 to stderr, and feeds us band#1
518 * through demux->out.
520 demux
.proc
= sideband_demux
;
522 if (start_async(&demux
))
523 die("fetch-pack: unable to fork off sideband"
529 memset(&cmd
, 0, sizeof(cmd
));
533 if (!args
.keep_pack
&& unpack_limit
) {
534 struct pack_header header
;
536 if (read_pack_header(demux
.out
, &header
))
537 die("protocol error: bad pack header");
538 snprintf(hdr_arg
, sizeof(hdr_arg
),
539 "--pack_header=%"PRIu32
",%"PRIu32
,
540 ntohl(header
.hdr_version
), ntohl(header
.hdr_entries
));
541 if (ntohl(header
.hdr_entries
) < unpack_limit
)
550 *av
++ = "index-pack";
552 if (!args
.quiet
&& !args
.no_progress
)
554 if (args
.use_thin_pack
)
555 *av
++ = "--fix-thin";
556 if (args
.lock_pack
|| unpack_limit
) {
557 int s
= sprintf(keep_arg
,
558 "--keep=fetch-pack %"PRIuMAX
" on ", (uintmax_t) getpid());
559 if (gethostname(keep_arg
+ s
, sizeof(keep_arg
) - s
))
560 strcpy(keep_arg
+ s
, "localhost");
565 *av
++ = "unpack-objects";
575 if (start_command(&cmd
))
576 die("fetch-pack: unable to fork off %s", argv
[0]);
577 if (do_keep
&& pack_lockfile
) {
578 *pack_lockfile
= index_pack_lockfile(cmd
.out
);
582 if (finish_command(&cmd
))
583 die("%s failed", argv
[0]);
584 if (use_sideband
&& finish_async(&demux
))
585 die("error in sideband demultiplexer");
589 static struct ref
*do_fetch_pack(int fd
[2],
590 const struct ref
*orig_ref
,
593 char **pack_lockfile
)
595 struct ref
*ref
= copy_ref_list(orig_ref
);
596 unsigned char sha1
[20];
598 if (is_repository_shallow() && !server_supports("shallow"))
599 die("Server does not support shallow clients");
600 if (server_supports("multi_ack")) {
602 fprintf(stderr
, "Server supports multi_ack\n");
605 if (server_supports("side-band-64k")) {
607 fprintf(stderr
, "Server supports side-band-64k\n");
610 else if (server_supports("side-band")) {
612 fprintf(stderr
, "Server supports side-band\n");
615 if (server_supports("ofs-delta")) {
617 fprintf(stderr
, "Server supports ofs-delta\n");
619 prefer_ofs_delta
= 0;
620 if (everything_local(&ref
, nr_match
, match
)) {
624 if (find_common(fd
, sha1
, ref
) < 0)
626 /* When cloning, it is not unusual to have
629 warning("no common commits");
631 if (get_pack(fd
, pack_lockfile
))
632 die("git fetch-pack: fetch failed.");
638 static int remove_duplicates(int nr_heads
, char **heads
)
642 for (src
= dst
= 0; src
< nr_heads
; src
++) {
643 /* If heads[src] is different from any of
644 * heads[0..dst], push it in.
647 for (i
= 0; i
< dst
; i
++) {
648 if (!strcmp(heads
[i
], heads
[src
]))
654 heads
[dst
] = heads
[src
];
660 static int fetch_pack_config(const char *var
, const char *value
, void *cb
)
662 if (strcmp(var
, "fetch.unpacklimit") == 0) {
663 fetch_unpack_limit
= git_config_int(var
, value
);
667 if (strcmp(var
, "transfer.unpacklimit") == 0) {
668 transfer_unpack_limit
= git_config_int(var
, value
);
672 if (strcmp(var
, "repack.usedeltabaseoffset") == 0) {
673 prefer_ofs_delta
= git_config_bool(var
, value
);
677 return git_default_config(var
, value
, cb
);
680 static struct lock_file lock
;
682 static void fetch_pack_setup(void)
684 static int did_setup
;
687 git_config(fetch_pack_config
, NULL
);
688 if (0 <= transfer_unpack_limit
)
689 unpack_limit
= transfer_unpack_limit
;
690 else if (0 <= fetch_unpack_limit
)
691 unpack_limit
= fetch_unpack_limit
;
695 int cmd_fetch_pack(int argc
, const char **argv
, const char *prefix
)
697 int i
, ret
, nr_heads
;
698 struct ref
*ref
= NULL
;
699 char *dest
= NULL
, **heads
;
701 struct child_process
*conn
;
705 for (i
= 1; i
< argc
; i
++) {
706 const char *arg
= argv
[i
];
709 if (!prefixcmp(arg
, "--upload-pack=")) {
710 args
.uploadpack
= arg
+ 14;
713 if (!prefixcmp(arg
, "--exec=")) {
714 args
.uploadpack
= arg
+ 7;
717 if (!strcmp("--quiet", arg
) || !strcmp("-q", arg
)) {
721 if (!strcmp("--keep", arg
) || !strcmp("-k", arg
)) {
722 args
.lock_pack
= args
.keep_pack
;
726 if (!strcmp("--thin", arg
)) {
727 args
.use_thin_pack
= 1;
730 if (!strcmp("--include-tag", arg
)) {
731 args
.include_tag
= 1;
734 if (!strcmp("--all", arg
)) {
738 if (!strcmp("-v", arg
)) {
742 if (!prefixcmp(arg
, "--depth=")) {
743 args
.depth
= strtol(arg
+ 8, NULL
, 0);
746 if (!strcmp("--no-progress", arg
)) {
747 args
.no_progress
= 1;
750 usage(fetch_pack_usage
);
753 heads
= (char **)(argv
+ i
+ 1);
754 nr_heads
= argc
- i
- 1;
758 usage(fetch_pack_usage
);
760 conn
= git_connect(fd
, (char *)dest
, args
.uploadpack
,
761 args
.verbose
? CONNECT_VERBOSE
: 0);
763 get_remote_heads(fd
[0], &ref
, 0, NULL
, 0, NULL
);
765 ref
= fetch_pack(&args
, fd
, conn
, ref
, dest
, nr_heads
, heads
, NULL
);
768 if (finish_connect(conn
))
775 if (!ret
&& nr_heads
) {
776 /* If the heads to pull were given, we should have
777 * consumed all of them by matching the remote.
778 * Otherwise, 'git fetch remote no-such-ref' would
779 * silently succeed without issuing an error.
781 for (i
= 0; i
< nr_heads
; i
++)
782 if (heads
[i
] && heads
[i
][0]) {
783 error("no such remote ref %s", heads
[i
]);
789 sha1_to_hex(ref
->old_sha1
), ref
->name
);
796 struct ref
*fetch_pack(struct fetch_pack_args
*my_args
,
797 int fd
[], struct child_process
*conn
,
798 const struct ref
*ref
,
802 char **pack_lockfile
)
808 if (&args
!= my_args
)
809 memcpy(&args
, my_args
, sizeof(args
));
810 if (args
.depth
> 0) {
811 if (stat(git_path("shallow"), &st
))
815 if (heads
&& nr_heads
)
816 nr_heads
= remove_duplicates(nr_heads
, heads
);
819 die("no matching remote head");
821 ref_cpy
= do_fetch_pack(fd
, ref
, nr_heads
, heads
, pack_lockfile
);
823 if (args
.depth
> 0) {
824 struct cache_time mtime
;
825 struct strbuf sb
= STRBUF_INIT
;
826 char *shallow
= git_path("shallow");
829 mtime
.sec
= st
.st_mtime
;
830 mtime
.nsec
= ST_MTIME_NSEC(st
);
831 if (stat(shallow
, &st
)) {
833 die("shallow file was removed during fetch");
834 } else if (st
.st_mtime
!= mtime
.sec
836 || ST_MTIME_NSEC(st
) != mtime
.nsec
839 die("shallow file was changed during fetch");
841 fd
= hold_lock_file_for_update(&lock
, shallow
,
843 if (!write_shallow_commits(&sb
, 0)
844 || write_in_full(fd
, sb
.buf
, sb
.len
) != sb
.len
) {
845 unlink_or_warn(shallow
);
846 rollback_lock_file(&lock
);
848 commit_lock_file(&lock
);
853 reprepare_packed_git();