9 #include "fetch-pack.h"
11 static int transfer_unpack_limit
= -1;
12 static int fetch_unpack_limit
= -1;
13 static int unpack_limit
= 100;
14 static struct fetch_pack_args args
;
16 static const char fetch_pack_usage
[] =
17 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
18 static const char *uploadpack
= "git-upload-pack";
20 #define COMPLETE (1U << 0)
21 #define COMMON (1U << 1)
22 #define COMMON_REF (1U << 2)
23 #define SEEN (1U << 3)
24 #define POPPED (1U << 4)
27 * After sending this many "have"s if we do not get any new ACK , we
28 * give up traversing our history.
30 #define MAX_IN_VAIN 256
32 static struct commit_list
*rev_list
;
33 static int non_common_revs
, multi_ack
, use_thin_pack
, use_sideband
;
35 static void rev_list_push(struct commit
*commit
, int mark
)
37 if (!(commit
->object
.flags
& mark
)) {
38 commit
->object
.flags
|= mark
;
40 if (!(commit
->object
.parsed
))
43 insert_by_date(commit
, &rev_list
);
45 if (!(commit
->object
.flags
& COMMON
))
50 static int rev_list_insert_ref(const char *path
, const unsigned char *sha1
, int flag
, void *cb_data
)
52 struct object
*o
= deref_tag(parse_object(sha1
), path
, 0);
54 if (o
&& o
->type
== OBJ_COMMIT
)
55 rev_list_push((struct commit
*)o
, SEEN
);
61 This function marks a rev and its ancestors as common.
62 In some cases, it is desirable to mark only the ancestors (for example
63 when only the server does not yet know that they are common).
66 static void mark_common(struct commit
*commit
,
67 int ancestors_only
, int dont_parse
)
69 if (commit
!= NULL
&& !(commit
->object
.flags
& COMMON
)) {
70 struct object
*o
= (struct object
*)commit
;
75 if (!(o
->flags
& SEEN
))
76 rev_list_push(commit
, SEEN
);
78 struct commit_list
*parents
;
80 if (!ancestors_only
&& !(o
->flags
& POPPED
))
82 if (!o
->parsed
&& !dont_parse
)
85 for (parents
= commit
->parents
;
87 parents
= parents
->next
)
88 mark_common(parents
->item
, 0, dont_parse
);
94 Get the next rev to send, ignoring the common.
97 static const unsigned char* get_rev(void)
99 struct commit
*commit
= NULL
;
101 while (commit
== NULL
) {
103 struct commit_list
* parents
;
105 if (rev_list
== NULL
|| non_common_revs
== 0)
108 commit
= rev_list
->item
;
109 if (!(commit
->object
.parsed
))
110 parse_commit(commit
);
111 commit
->object
.flags
|= POPPED
;
112 if (!(commit
->object
.flags
& COMMON
))
115 parents
= commit
->parents
;
117 if (commit
->object
.flags
& COMMON
) {
118 /* do not send "have", and ignore ancestors */
120 mark
= COMMON
| SEEN
;
121 } else if (commit
->object
.flags
& COMMON_REF
)
122 /* send "have", and ignore ancestors */
123 mark
= COMMON
| SEEN
;
125 /* send "have", also for its ancestors */
129 if (!(parents
->item
->object
.flags
& SEEN
))
130 rev_list_push(parents
->item
, mark
);
132 mark_common(parents
->item
, 1, 0);
133 parents
= parents
->next
;
136 rev_list
= rev_list
->next
;
139 return commit
->object
.sha1
;
142 static int find_common(int fd
[2], unsigned char *result_sha1
,
146 int count
= 0, flushes
= 0, retval
;
147 const unsigned char *sha1
;
148 unsigned in_vain
= 0;
149 int got_continue
= 0;
151 for_each_ref(rev_list_insert_ref
, NULL
);
154 for ( ; refs
; refs
= refs
->next
) {
155 unsigned char *remote
= refs
->old_sha1
;
159 * If that object is complete (i.e. it is an ancestor of a
160 * local ref), we tell them we have it but do not have to
161 * tell them about its ancestors, which they already know
164 * We use lookup_object here because we are only
165 * interested in the case we *know* the object is
166 * reachable and we have already scanned it.
168 if (((o
= lookup_object(remote
)) != NULL
) &&
169 (o
->flags
& COMPLETE
)) {
174 packet_write(fd
[1], "want %s%s%s%s%s%s%s\n",
176 (multi_ack
? " multi_ack" : ""),
177 (use_sideband
== 2 ? " side-band-64k" : ""),
178 (use_sideband
== 1 ? " side-band" : ""),
179 (use_thin_pack
? " thin-pack" : ""),
180 (args
.no_progress
? " no-progress" : ""),
183 packet_write(fd
[1], "want %s\n", sha1_to_hex(remote
));
186 if (is_repository_shallow())
187 write_shallow_commits(fd
[1], 1);
189 packet_write(fd
[1], "deepen %d", args
.depth
);
194 if (args
.depth
> 0) {
196 unsigned char sha1
[20];
199 while ((len
= packet_read_line(fd
[0], line
, sizeof(line
)))) {
200 if (!prefixcmp(line
, "shallow ")) {
201 if (get_sha1_hex(line
+ 8, sha1
))
202 die("invalid shallow line: %s", line
);
203 register_shallow(sha1
);
206 if (!prefixcmp(line
, "unshallow ")) {
207 if (get_sha1_hex(line
+ 10, sha1
))
208 die("invalid unshallow line: %s", line
);
209 if (!lookup_object(sha1
))
210 die("object not found: %s", line
);
211 /* make sure that it is parsed as shallow */
213 if (unregister_shallow(sha1
))
214 die("no shallow found: %s", line
);
217 die("expected shallow/unshallow, got %s", line
);
223 while ((sha1
= get_rev())) {
224 packet_write(fd
[1], "have %s\n", sha1_to_hex(sha1
));
226 fprintf(stderr
, "have %s\n", sha1_to_hex(sha1
));
228 if (!(31 & ++count
)) {
235 * We keep one window "ahead" of the other side, and
236 * will wait for an ACK only on the next one
242 ack
= get_ack(fd
[0], result_sha1
);
243 if (args
.verbose
&& ack
)
244 fprintf(stderr
, "got ack %d %s\n", ack
,
245 sha1_to_hex(result_sha1
));
251 } else if (ack
== 2) {
252 struct commit
*commit
=
253 lookup_commit(result_sha1
);
254 mark_common(commit
, 0, 1);
261 if (got_continue
&& MAX_IN_VAIN
< in_vain
) {
263 fprintf(stderr
, "giving up\n");
269 packet_write(fd
[1], "done\n");
271 fprintf(stderr
, "done\n");
276 while (flushes
|| multi_ack
) {
277 int ack
= get_ack(fd
[0], result_sha1
);
280 fprintf(stderr
, "got ack (%d) %s\n", ack
,
281 sha1_to_hex(result_sha1
));
292 static struct commit_list
*complete
;
294 static int mark_complete(const char *path
, const unsigned char *sha1
, int flag
, void *cb_data
)
296 struct object
*o
= parse_object(sha1
);
298 while (o
&& o
->type
== OBJ_TAG
) {
299 struct tag
*t
= (struct tag
*) o
;
301 break; /* broken repository */
302 o
->flags
|= COMPLETE
;
303 o
= parse_object(t
->tagged
->sha1
);
305 if (o
&& o
->type
== OBJ_COMMIT
) {
306 struct commit
*commit
= (struct commit
*)o
;
307 commit
->object
.flags
|= COMPLETE
;
308 insert_by_date(commit
, &complete
);
313 static void mark_recent_complete_commits(unsigned long cutoff
)
315 while (complete
&& cutoff
<= complete
->item
->date
) {
317 fprintf(stderr
, "Marking %s as complete\n",
318 sha1_to_hex(complete
->item
->object
.sha1
));
319 pop_most_recent_commit(&complete
, COMPLETE
);
323 static void filter_refs(struct ref
**refs
, int nr_match
, char **match
)
325 struct ref
**return_refs
;
326 struct ref
*newlist
= NULL
;
327 struct ref
**newtail
= &newlist
;
328 struct ref
*ref
, *next
;
329 struct ref
*fastarray
[32];
331 if (nr_match
&& !args
.fetch_all
) {
332 if (ARRAY_SIZE(fastarray
) < nr_match
)
333 return_refs
= xcalloc(nr_match
, sizeof(struct ref
*));
335 return_refs
= fastarray
;
336 memset(return_refs
, 0, sizeof(struct ref
*) * nr_match
);
342 for (ref
= *refs
; ref
; ref
= next
) {
344 if (!memcmp(ref
->name
, "refs/", 5) &&
345 check_ref_format(ref
->name
+ 5))
347 else if (args
.fetch_all
&&
348 (!args
.depth
|| prefixcmp(ref
->name
, "refs/tags/") )) {
351 newtail
= &ref
->next
;
355 int order
= path_match(ref
->name
, nr_match
, match
);
357 return_refs
[order
-1] = ref
;
358 continue; /* we will link it later */
364 if (!args
.fetch_all
) {
366 for (i
= 0; i
< nr_match
; i
++) {
367 ref
= return_refs
[i
];
371 newtail
= &ref
->next
;
374 if (return_refs
!= fastarray
)
380 static int everything_local(struct ref
**refs
, int nr_match
, char **match
)
384 unsigned long cutoff
= 0;
386 track_object_refs
= 0;
387 save_commit_buffer
= 0;
389 for (ref
= *refs
; ref
; ref
= ref
->next
) {
392 o
= parse_object(ref
->old_sha1
);
396 /* We already have it -- which may mean that we were
397 * in sync with the other side at some time after
398 * that (it is OK if we guess wrong here).
400 if (o
->type
== OBJ_COMMIT
) {
401 struct commit
*commit
= (struct commit
*)o
;
402 if (!cutoff
|| cutoff
< commit
->date
)
403 cutoff
= commit
->date
;
408 for_each_ref(mark_complete
, NULL
);
410 mark_recent_complete_commits(cutoff
);
414 * Mark all complete remote refs as common refs.
415 * Don't mark them common yet; the server has to be told so first.
417 for (ref
= *refs
; ref
; ref
= ref
->next
) {
418 struct object
*o
= deref_tag(lookup_object(ref
->old_sha1
),
421 if (!o
|| o
->type
!= OBJ_COMMIT
|| !(o
->flags
& COMPLETE
))
424 if (!(o
->flags
& SEEN
)) {
425 rev_list_push((struct commit
*)o
, COMMON_REF
| SEEN
);
427 mark_common((struct commit
*)o
, 1, 1);
431 filter_refs(refs
, nr_match
, match
);
433 for (retval
= 1, ref
= *refs
; ref
; ref
= ref
->next
) {
434 const unsigned char *remote
= ref
->old_sha1
;
435 unsigned char local
[20];
438 o
= lookup_object(remote
);
439 if (!o
|| !(o
->flags
& COMPLETE
)) {
444 "want %s (%s)\n", sha1_to_hex(remote
),
449 hashcpy(ref
->new_sha1
, local
);
453 "already have %s (%s)\n", sha1_to_hex(remote
),
459 static pid_t
setup_sideband(int fd
[2], int xd
[2])
468 /* xd[] is talking with upload-pack; subprocess reads from
469 * xd[0], spits out band#2 to stderr, and feeds us band#1
473 die("fetch-pack: unable to set up pipe");
476 die("fetch-pack: unable to fork off sideband demultiplexer");
482 if (recv_sideband("fetch-pack", xd
[0], fd
[1], 2))
492 static int get_pack(int xd
[2], char **pack_lockfile
)
497 const char *argv
[20];
501 int do_keep
= args
.keep_pack
;
504 side_pid
= setup_sideband(fd
, xd
);
508 if (!args
.keep_pack
&& unpack_limit
) {
509 struct pack_header header
;
511 if (read_pack_header(fd
[0], &header
))
512 die("protocol error: bad pack header");
513 snprintf(hdr_arg
, sizeof(hdr_arg
), "--pack_header=%u,%u",
514 ntohl(header
.hdr_version
), ntohl(header
.hdr_entries
));
515 if (ntohl(header
.hdr_entries
) < unpack_limit
)
522 if (pack_lockfile
&& pipe(keep_pipe
))
523 die("fetch-pack: pipe setup failure: %s", strerror(errno
));
524 *av
++ = "index-pack";
526 if (!args
.quiet
&& !args
.no_progress
)
528 if (args
.use_thin_pack
)
529 *av
++ = "--fix-thin";
530 if (args
.lock_pack
|| unpack_limit
) {
531 int s
= sprintf(keep_arg
,
532 "--keep=fetch-pack %d on ", getpid());
533 if (gethostname(keep_arg
+ s
, sizeof(keep_arg
) - s
))
534 strcpy(keep_arg
+ s
, "localhost");
539 *av
++ = "unpack-objects";
549 die("fetch-pack: unable to fork off %s", argv
[0]);
552 if (do_keep
&& pack_lockfile
) {
553 dup2(keep_pipe
[1], 1);
560 die("%s exec failed", argv
[0]);
564 if (do_keep
&& pack_lockfile
) {
566 *pack_lockfile
= index_pack_lockfile(keep_pipe
[0]);
569 while (waitpid(pid
, &status
, 0) < 0) {
571 die("waiting for %s: %s", argv
[0], strerror(errno
));
573 if (WIFEXITED(status
)) {
574 int code
= WEXITSTATUS(status
);
576 die("%s died with error code %d", argv
[0], code
);
579 if (WIFSIGNALED(status
)) {
580 int sig
= WTERMSIG(status
);
581 die("%s died of signal %d", argv
[0], sig
);
583 die("%s died of unnatural causes %d", argv
[0], status
);
586 static struct ref
*do_fetch_pack(int fd
[2],
589 char **pack_lockfile
)
592 unsigned char sha1
[20];
594 get_remote_heads(fd
[0], &ref
, 0, NULL
, 0);
595 if (is_repository_shallow() && !server_supports("shallow"))
596 die("Server does not support shallow clients");
597 if (server_supports("multi_ack")) {
599 fprintf(stderr
, "Server supports multi_ack\n");
602 if (server_supports("side-band-64k")) {
604 fprintf(stderr
, "Server supports side-band-64k\n");
607 else if (server_supports("side-band")) {
609 fprintf(stderr
, "Server supports side-band\n");
614 die("no matching remote head");
616 if (everything_local(&ref
, nr_match
, match
)) {
620 if (find_common(fd
, sha1
, ref
) < 0)
622 /* When cloning, it is not unusual to have
625 fprintf(stderr
, "warning: no common commits\n");
627 if (get_pack(fd
, pack_lockfile
))
628 die("git-fetch-pack: fetch failed.");
634 static int remove_duplicates(int nr_heads
, char **heads
)
638 for (src
= dst
= 0; src
< nr_heads
; src
++) {
639 /* If heads[src] is different from any of
640 * heads[0..dst], push it in.
643 for (i
= 0; i
< dst
; i
++) {
644 if (!strcmp(heads
[i
], heads
[src
]))
650 heads
[dst
] = heads
[src
];
656 static int fetch_pack_config(const char *var
, const char *value
)
658 if (strcmp(var
, "fetch.unpacklimit") == 0) {
659 fetch_unpack_limit
= git_config_int(var
, value
);
663 if (strcmp(var
, "transfer.unpacklimit") == 0) {
664 transfer_unpack_limit
= git_config_int(var
, value
);
668 return git_default_config(var
, value
);
671 static struct lock_file lock
;
673 int cmd_fetch_pack(int argc
, const char **argv
, const char *prefix
)
675 int i
, ret
, nr_heads
;
677 char *dest
= NULL
, **heads
;
679 git_config(fetch_pack_config
);
681 if (0 <= transfer_unpack_limit
)
682 unpack_limit
= transfer_unpack_limit
;
683 else if (0 <= fetch_unpack_limit
)
684 unpack_limit
= fetch_unpack_limit
;
688 for (i
= 1; i
< argc
; i
++) {
689 const char *arg
= argv
[i
];
692 if (!prefixcmp(arg
, "--upload-pack=")) {
693 args
.uploadpack
= arg
+ 14;
696 if (!prefixcmp(arg
, "--exec=")) {
697 args
.uploadpack
= arg
+ 7;
700 if (!strcmp("--quiet", arg
) || !strcmp("-q", arg
)) {
704 if (!strcmp("--keep", arg
) || !strcmp("-k", arg
)) {
705 args
.lock_pack
= args
.keep_pack
;
709 if (!strcmp("--thin", arg
)) {
710 args
.use_thin_pack
= 1;
713 if (!strcmp("--all", arg
)) {
717 if (!strcmp("-v", arg
)) {
721 if (!prefixcmp(arg
, "--depth=")) {
722 args
.depth
= strtol(arg
+ 8, NULL
, 0);
725 if (!strcmp("--no-progress", arg
)) {
726 args
.no_progress
= 1;
729 usage(fetch_pack_usage
);
732 heads
= (char **)(argv
+ i
+ 1);
733 nr_heads
= argc
- i
- 1;
737 usage(fetch_pack_usage
);
739 ref
= fetch_pack(&args
, dest
, nr_heads
, heads
, NULL
);
744 sha1_to_hex(ref
->old_sha1
), ref
->name
);
751 struct ref
*fetch_pack(struct fetch_pack_args
*my_args
,
755 char **pack_lockfile
)
763 memcpy(&args
, my_args
, sizeof(args
));
764 if (args
.depth
> 0) {
765 if (stat(git_path("shallow"), &st
))
769 pid
= git_connect(fd
, (char *)dest
, uploadpack
,
770 args
.verbose
? CONNECT_VERBOSE
: 0);
773 if (heads
&& nr_heads
)
774 nr_heads
= remove_duplicates(nr_heads
, heads
);
775 ref
= do_fetch_pack(fd
, nr_heads
, heads
, pack_lockfile
);
778 ret
= finish_connect(pid
);
780 if (!ret
&& nr_heads
) {
781 /* If the heads to pull were given, we should have
782 * consumed all of them by matching the remote.
783 * Otherwise, 'git-fetch remote no-such-ref' would
784 * silently succeed without issuing an error.
786 for (i
= 0; i
< nr_heads
; i
++)
787 if (heads
[i
] && heads
[i
][0]) {
788 error("no such remote ref %s", heads
[i
]);
793 if (!ret
&& args
.depth
> 0) {
794 struct cache_time mtime
;
795 char *shallow
= git_path("shallow");
798 mtime
.sec
= st
.st_mtime
;
800 mtime
.usec
= st
.st_mtim
.usec
;
802 if (stat(shallow
, &st
)) {
804 die("shallow file was removed during fetch");
805 } else if (st
.st_mtime
!= mtime
.sec
807 || st
.st_mtim
.usec
!= mtime
.usec
810 die("shallow file was changed during fetch");
812 fd
= hold_lock_file_for_update(&lock
, shallow
, 1);
813 if (!write_shallow_commits(fd
, 0)) {
815 rollback_lock_file(&lock
);
818 commit_lock_file(&lock
);