9 #include "fetch-pack.h"
10 #include "run-command.h"
12 static int transfer_unpack_limit
= -1;
13 static int fetch_unpack_limit
= -1;
14 static int unpack_limit
= 100;
15 static struct fetch_pack_args args
= {
16 /* .uploadpack = */ "git-upload-pack",
19 static const char fetch_pack_usage
[] =
20 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
22 #define COMPLETE (1U << 0)
23 #define COMMON (1U << 1)
24 #define COMMON_REF (1U << 2)
25 #define SEEN (1U << 3)
26 #define POPPED (1U << 4)
29 * After sending this many "have"s if we do not get any new ACK , we
30 * give up traversing our history.
32 #define MAX_IN_VAIN 256
34 static struct commit_list
*rev_list
;
35 static int non_common_revs
, multi_ack
, use_sideband
;
37 static void rev_list_push(struct commit
*commit
, int mark
)
39 if (!(commit
->object
.flags
& mark
)) {
40 commit
->object
.flags
|= mark
;
42 if (!(commit
->object
.parsed
))
45 insert_by_date(commit
, &rev_list
);
47 if (!(commit
->object
.flags
& COMMON
))
52 static int rev_list_insert_ref(const char *path
, const unsigned char *sha1
, int flag
, void *cb_data
)
54 struct object
*o
= deref_tag(parse_object(sha1
), path
, 0);
56 if (o
&& o
->type
== OBJ_COMMIT
)
57 rev_list_push((struct commit
*)o
, SEEN
);
63 This function marks a rev and its ancestors as common.
64 In some cases, it is desirable to mark only the ancestors (for example
65 when only the server does not yet know that they are common).
68 static void mark_common(struct commit
*commit
,
69 int ancestors_only
, int dont_parse
)
71 if (commit
!= NULL
&& !(commit
->object
.flags
& COMMON
)) {
72 struct object
*o
= (struct object
*)commit
;
77 if (!(o
->flags
& SEEN
))
78 rev_list_push(commit
, SEEN
);
80 struct commit_list
*parents
;
82 if (!ancestors_only
&& !(o
->flags
& POPPED
))
84 if (!o
->parsed
&& !dont_parse
)
87 for (parents
= commit
->parents
;
89 parents
= parents
->next
)
90 mark_common(parents
->item
, 0, dont_parse
);
96 Get the next rev to send, ignoring the common.
99 static const unsigned char* get_rev(void)
101 struct commit
*commit
= NULL
;
103 while (commit
== NULL
) {
105 struct commit_list
* parents
;
107 if (rev_list
== NULL
|| non_common_revs
== 0)
110 commit
= rev_list
->item
;
111 if (!(commit
->object
.parsed
))
112 parse_commit(commit
);
113 commit
->object
.flags
|= POPPED
;
114 if (!(commit
->object
.flags
& COMMON
))
117 parents
= commit
->parents
;
119 if (commit
->object
.flags
& COMMON
) {
120 /* do not send "have", and ignore ancestors */
122 mark
= COMMON
| SEEN
;
123 } else if (commit
->object
.flags
& COMMON_REF
)
124 /* send "have", and ignore ancestors */
125 mark
= COMMON
| SEEN
;
127 /* send "have", also for its ancestors */
131 if (!(parents
->item
->object
.flags
& SEEN
))
132 rev_list_push(parents
->item
, mark
);
134 mark_common(parents
->item
, 1, 0);
135 parents
= parents
->next
;
138 rev_list
= rev_list
->next
;
141 return commit
->object
.sha1
;
144 static int find_common(int fd
[2], unsigned char *result_sha1
,
148 int count
= 0, flushes
= 0, retval
;
149 const unsigned char *sha1
;
150 unsigned in_vain
= 0;
151 int got_continue
= 0;
153 for_each_ref(rev_list_insert_ref
, NULL
);
156 for ( ; refs
; refs
= refs
->next
) {
157 unsigned char *remote
= refs
->old_sha1
;
161 * If that object is complete (i.e. it is an ancestor of a
162 * local ref), we tell them we have it but do not have to
163 * tell them about its ancestors, which they already know
166 * We use lookup_object here because we are only
167 * interested in the case we *know* the object is
168 * reachable and we have already scanned it.
170 if (((o
= lookup_object(remote
)) != NULL
) &&
171 (o
->flags
& COMPLETE
)) {
176 packet_write(fd
[1], "want %s%s%s%s%s%s%s\n",
178 (multi_ack
? " multi_ack" : ""),
179 (use_sideband
== 2 ? " side-band-64k" : ""),
180 (use_sideband
== 1 ? " side-band" : ""),
181 (args
.use_thin_pack
? " thin-pack" : ""),
182 (args
.no_progress
? " no-progress" : ""),
185 packet_write(fd
[1], "want %s\n", sha1_to_hex(remote
));
188 if (is_repository_shallow())
189 write_shallow_commits(fd
[1], 1);
191 packet_write(fd
[1], "deepen %d", args
.depth
);
196 if (args
.depth
> 0) {
198 unsigned char sha1
[20];
201 while ((len
= packet_read_line(fd
[0], line
, sizeof(line
)))) {
202 if (!prefixcmp(line
, "shallow ")) {
203 if (get_sha1_hex(line
+ 8, sha1
))
204 die("invalid shallow line: %s", line
);
205 register_shallow(sha1
);
208 if (!prefixcmp(line
, "unshallow ")) {
209 if (get_sha1_hex(line
+ 10, sha1
))
210 die("invalid unshallow line: %s", line
);
211 if (!lookup_object(sha1
))
212 die("object not found: %s", line
);
213 /* make sure that it is parsed as shallow */
215 if (unregister_shallow(sha1
))
216 die("no shallow found: %s", line
);
219 die("expected shallow/unshallow, got %s", line
);
225 while ((sha1
= get_rev())) {
226 packet_write(fd
[1], "have %s\n", sha1_to_hex(sha1
));
228 fprintf(stderr
, "have %s\n", sha1_to_hex(sha1
));
230 if (!(31 & ++count
)) {
237 * We keep one window "ahead" of the other side, and
238 * will wait for an ACK only on the next one
244 ack
= get_ack(fd
[0], result_sha1
);
245 if (args
.verbose
&& ack
)
246 fprintf(stderr
, "got ack %d %s\n", ack
,
247 sha1_to_hex(result_sha1
));
253 } else if (ack
== 2) {
254 struct commit
*commit
=
255 lookup_commit(result_sha1
);
256 mark_common(commit
, 0, 1);
263 if (got_continue
&& MAX_IN_VAIN
< in_vain
) {
265 fprintf(stderr
, "giving up\n");
271 packet_write(fd
[1], "done\n");
273 fprintf(stderr
, "done\n");
278 while (flushes
|| multi_ack
) {
279 int ack
= get_ack(fd
[0], result_sha1
);
282 fprintf(stderr
, "got ack (%d) %s\n", ack
,
283 sha1_to_hex(result_sha1
));
294 static struct commit_list
*complete
;
296 static int mark_complete(const char *path
, const unsigned char *sha1
, int flag
, void *cb_data
)
298 struct object
*o
= parse_object(sha1
);
300 while (o
&& o
->type
== OBJ_TAG
) {
301 struct tag
*t
= (struct tag
*) o
;
303 break; /* broken repository */
304 o
->flags
|= COMPLETE
;
305 o
= parse_object(t
->tagged
->sha1
);
307 if (o
&& o
->type
== OBJ_COMMIT
) {
308 struct commit
*commit
= (struct commit
*)o
;
309 commit
->object
.flags
|= COMPLETE
;
310 insert_by_date(commit
, &complete
);
315 static void mark_recent_complete_commits(unsigned long cutoff
)
317 while (complete
&& cutoff
<= complete
->item
->date
) {
319 fprintf(stderr
, "Marking %s as complete\n",
320 sha1_to_hex(complete
->item
->object
.sha1
));
321 pop_most_recent_commit(&complete
, COMPLETE
);
325 static void filter_refs(struct ref
**refs
, int nr_match
, char **match
)
327 struct ref
**return_refs
;
328 struct ref
*newlist
= NULL
;
329 struct ref
**newtail
= &newlist
;
330 struct ref
*ref
, *next
;
331 struct ref
*fastarray
[32];
333 if (nr_match
&& !args
.fetch_all
) {
334 if (ARRAY_SIZE(fastarray
) < nr_match
)
335 return_refs
= xcalloc(nr_match
, sizeof(struct ref
*));
337 return_refs
= fastarray
;
338 memset(return_refs
, 0, sizeof(struct ref
*) * nr_match
);
344 for (ref
= *refs
; ref
; ref
= next
) {
346 if (!memcmp(ref
->name
, "refs/", 5) &&
347 check_ref_format(ref
->name
+ 5))
349 else if (args
.fetch_all
&&
350 (!args
.depth
|| prefixcmp(ref
->name
, "refs/tags/") )) {
353 newtail
= &ref
->next
;
357 int order
= path_match(ref
->name
, nr_match
, match
);
359 return_refs
[order
-1] = ref
;
360 continue; /* we will link it later */
366 if (!args
.fetch_all
) {
368 for (i
= 0; i
< nr_match
; i
++) {
369 ref
= return_refs
[i
];
373 newtail
= &ref
->next
;
376 if (return_refs
!= fastarray
)
382 static int everything_local(struct ref
**refs
, int nr_match
, char **match
)
386 unsigned long cutoff
= 0;
388 track_object_refs
= 0;
389 save_commit_buffer
= 0;
391 for (ref
= *refs
; ref
; ref
= ref
->next
) {
394 o
= parse_object(ref
->old_sha1
);
398 /* We already have it -- which may mean that we were
399 * in sync with the other side at some time after
400 * that (it is OK if we guess wrong here).
402 if (o
->type
== OBJ_COMMIT
) {
403 struct commit
*commit
= (struct commit
*)o
;
404 if (!cutoff
|| cutoff
< commit
->date
)
405 cutoff
= commit
->date
;
410 for_each_ref(mark_complete
, NULL
);
412 mark_recent_complete_commits(cutoff
);
416 * Mark all complete remote refs as common refs.
417 * Don't mark them common yet; the server has to be told so first.
419 for (ref
= *refs
; ref
; ref
= ref
->next
) {
420 struct object
*o
= deref_tag(lookup_object(ref
->old_sha1
),
423 if (!o
|| o
->type
!= OBJ_COMMIT
|| !(o
->flags
& COMPLETE
))
426 if (!(o
->flags
& SEEN
)) {
427 rev_list_push((struct commit
*)o
, COMMON_REF
| SEEN
);
429 mark_common((struct commit
*)o
, 1, 1);
433 filter_refs(refs
, nr_match
, match
);
435 for (retval
= 1, ref
= *refs
; ref
; ref
= ref
->next
) {
436 const unsigned char *remote
= ref
->old_sha1
;
437 unsigned char local
[20];
440 o
= lookup_object(remote
);
441 if (!o
|| !(o
->flags
& COMPLETE
)) {
446 "want %s (%s)\n", sha1_to_hex(remote
),
451 hashcpy(ref
->new_sha1
, local
);
455 "already have %s (%s)\n", sha1_to_hex(remote
),
461 static int sideband_demux(int fd
, void *data
)
466 return recv_sideband("fetch-pack", xd
[0], fd
, 2);
469 static void setup_sideband(int fd
[2], int xd
[2], struct async
*demux
)
476 /* xd[] is talking with upload-pack; subprocess reads from
477 * xd[0], spits out band#2 to stderr, and feeds us band#1
478 * through demux->out.
480 demux
->proc
= sideband_demux
;
482 if (start_async(demux
))
483 die("fetch-pack: unable to fork off sideband demultiplexer");
489 static int get_pack(int xd
[2], char **pack_lockfile
)
493 const char *argv
[20];
497 int do_keep
= args
.keep_pack
;
498 struct child_process cmd
;
500 setup_sideband(fd
, xd
, &demux
);
502 memset(&cmd
, 0, sizeof(cmd
));
506 if (!args
.keep_pack
&& unpack_limit
) {
507 struct pack_header header
;
509 if (read_pack_header(fd
[0], &header
))
510 die("protocol error: bad pack header");
511 snprintf(hdr_arg
, sizeof(hdr_arg
), "--pack_header=%u,%u",
512 ntohl(header
.hdr_version
), ntohl(header
.hdr_entries
));
513 if (ntohl(header
.hdr_entries
) < unpack_limit
)
522 *av
++ = "index-pack";
524 if (!args
.quiet
&& !args
.no_progress
)
526 if (args
.use_thin_pack
)
527 *av
++ = "--fix-thin";
528 if (args
.lock_pack
|| unpack_limit
) {
529 int s
= sprintf(keep_arg
,
530 "--keep=fetch-pack %d on ", getpid());
531 if (gethostname(keep_arg
+ s
, sizeof(keep_arg
) - s
))
532 strcpy(keep_arg
+ s
, "localhost");
537 *av
++ = "unpack-objects";
547 if (start_command(&cmd
))
548 die("fetch-pack: unable to fork off %s", argv
[0]);
550 if (do_keep
&& pack_lockfile
)
551 *pack_lockfile
= index_pack_lockfile(cmd
.out
);
553 if (finish_command(&cmd
))
554 die("%s failed", argv
[0]);
555 if (use_sideband
&& finish_async(&demux
))
556 die("error in sideband demultiplexer");
560 static struct ref
*do_fetch_pack(int fd
[2],
563 char **pack_lockfile
)
566 unsigned char sha1
[20];
568 get_remote_heads(fd
[0], &ref
, 0, NULL
, 0);
569 if (is_repository_shallow() && !server_supports("shallow"))
570 die("Server does not support shallow clients");
571 if (server_supports("multi_ack")) {
573 fprintf(stderr
, "Server supports multi_ack\n");
577 if (server_supports("side-band-64k")) {
579 fprintf(stderr
, "Server supports side-band-64k\n");
582 else if (server_supports("side-band")) {
584 fprintf(stderr
, "Server supports side-band\n");
590 die("no matching remote head");
592 if (everything_local(&ref
, nr_match
, match
)) {
596 if (find_common(fd
, sha1
, ref
) < 0)
598 /* When cloning, it is not unusual to have
601 fprintf(stderr
, "warning: no common commits\n");
603 if (get_pack(fd
, pack_lockfile
))
604 die("git-fetch-pack: fetch failed.");
610 static int remove_duplicates(int nr_heads
, char **heads
)
614 for (src
= dst
= 0; src
< nr_heads
; src
++) {
615 /* If heads[src] is different from any of
616 * heads[0..dst], push it in.
619 for (i
= 0; i
< dst
; i
++) {
620 if (!strcmp(heads
[i
], heads
[src
]))
626 heads
[dst
] = heads
[src
];
632 static int fetch_pack_config(const char *var
, const char *value
)
634 if (strcmp(var
, "fetch.unpacklimit") == 0) {
635 fetch_unpack_limit
= git_config_int(var
, value
);
639 if (strcmp(var
, "transfer.unpacklimit") == 0) {
640 transfer_unpack_limit
= git_config_int(var
, value
);
644 return git_default_config(var
, value
);
647 static struct lock_file lock
;
649 static void fetch_pack_setup(void)
651 static int did_setup
;
654 git_config(fetch_pack_config
);
655 if (0 <= transfer_unpack_limit
)
656 unpack_limit
= transfer_unpack_limit
;
657 else if (0 <= fetch_unpack_limit
)
658 unpack_limit
= fetch_unpack_limit
;
662 int cmd_fetch_pack(int argc
, const char **argv
, const char *prefix
)
664 int i
, ret
, nr_heads
;
666 char *dest
= NULL
, **heads
;
670 for (i
= 1; i
< argc
; i
++) {
671 const char *arg
= argv
[i
];
674 if (!prefixcmp(arg
, "--upload-pack=")) {
675 args
.uploadpack
= arg
+ 14;
678 if (!prefixcmp(arg
, "--exec=")) {
679 args
.uploadpack
= arg
+ 7;
682 if (!strcmp("--quiet", arg
) || !strcmp("-q", arg
)) {
686 if (!strcmp("--keep", arg
) || !strcmp("-k", arg
)) {
687 args
.lock_pack
= args
.keep_pack
;
691 if (!strcmp("--thin", arg
)) {
692 args
.use_thin_pack
= 1;
695 if (!strcmp("--all", arg
)) {
699 if (!strcmp("-v", arg
)) {
703 if (!prefixcmp(arg
, "--depth=")) {
704 args
.depth
= strtol(arg
+ 8, NULL
, 0);
707 if (!strcmp("--no-progress", arg
)) {
708 args
.no_progress
= 1;
711 usage(fetch_pack_usage
);
714 heads
= (char **)(argv
+ i
+ 1);
715 nr_heads
= argc
- i
- 1;
719 usage(fetch_pack_usage
);
721 ref
= fetch_pack(&args
, dest
, nr_heads
, heads
, NULL
);
726 sha1_to_hex(ref
->old_sha1
), ref
->name
);
733 struct ref
*fetch_pack(struct fetch_pack_args
*my_args
,
737 char **pack_lockfile
)
741 struct child_process
*conn
;
746 memcpy(&args
, my_args
, sizeof(args
));
747 if (args
.depth
> 0) {
748 if (stat(git_path("shallow"), &st
))
752 conn
= git_connect(fd
, (char *)dest
, args
.uploadpack
,
753 args
.verbose
? CONNECT_VERBOSE
: 0);
754 if (heads
&& nr_heads
)
755 nr_heads
= remove_duplicates(nr_heads
, heads
);
756 ref
= do_fetch_pack(fd
, nr_heads
, heads
, pack_lockfile
);
759 ret
= finish_connect(conn
);
761 if (!ret
&& nr_heads
) {
762 /* If the heads to pull were given, we should have
763 * consumed all of them by matching the remote.
764 * Otherwise, 'git-fetch remote no-such-ref' would
765 * silently succeed without issuing an error.
767 for (i
= 0; i
< nr_heads
; i
++)
768 if (heads
[i
] && heads
[i
][0]) {
769 error("no such remote ref %s", heads
[i
]);
774 if (!ret
&& args
.depth
> 0) {
775 struct cache_time mtime
;
776 char *shallow
= git_path("shallow");
779 mtime
.sec
= st
.st_mtime
;
781 mtime
.usec
= st
.st_mtim
.usec
;
783 if (stat(shallow
, &st
)) {
785 die("shallow file was removed during fetch");
786 } else if (st
.st_mtime
!= mtime
.sec
788 || st
.st_mtim
.usec
!= mtime
.usec
791 die("shallow file was changed during fetch");
793 fd
= hold_lock_file_for_update(&lock
, shallow
, 1);
794 if (!write_shallow_commits(fd
, 0)) {
796 rollback_lock_file(&lock
);
799 commit_lock_file(&lock
);