9 #include "fetch-pack.h"
11 #include "run-command.h"
13 static int transfer_unpack_limit
= -1;
14 static int fetch_unpack_limit
= -1;
15 static int unpack_limit
= 100;
16 static struct fetch_pack_args args
= {
17 /* .uploadpack = */ "git-upload-pack",
20 static const char fetch_pack_usage
[] =
21 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
23 #define COMPLETE (1U << 0)
24 #define COMMON (1U << 1)
25 #define COMMON_REF (1U << 2)
26 #define SEEN (1U << 3)
27 #define POPPED (1U << 4)
30 * After sending this many "have"s if we do not get any new ACK , we
31 * give up traversing our history.
33 #define MAX_IN_VAIN 256
35 static struct commit_list
*rev_list
;
36 static int non_common_revs
, multi_ack
, use_sideband
;
38 static void rev_list_push(struct commit
*commit
, int mark
)
40 if (!(commit
->object
.flags
& mark
)) {
41 commit
->object
.flags
|= mark
;
43 if (!(commit
->object
.parsed
))
46 insert_by_date(commit
, &rev_list
);
48 if (!(commit
->object
.flags
& COMMON
))
53 static int rev_list_insert_ref(const char *path
, const unsigned char *sha1
, int flag
, void *cb_data
)
55 struct object
*o
= deref_tag(parse_object(sha1
), path
, 0);
57 if (o
&& o
->type
== OBJ_COMMIT
)
58 rev_list_push((struct commit
*)o
, SEEN
);
64 This function marks a rev and its ancestors as common.
65 In some cases, it is desirable to mark only the ancestors (for example
66 when only the server does not yet know that they are common).
69 static void mark_common(struct commit
*commit
,
70 int ancestors_only
, int dont_parse
)
72 if (commit
!= NULL
&& !(commit
->object
.flags
& COMMON
)) {
73 struct object
*o
= (struct object
*)commit
;
78 if (!(o
->flags
& SEEN
))
79 rev_list_push(commit
, SEEN
);
81 struct commit_list
*parents
;
83 if (!ancestors_only
&& !(o
->flags
& POPPED
))
85 if (!o
->parsed
&& !dont_parse
)
88 for (parents
= commit
->parents
;
90 parents
= parents
->next
)
91 mark_common(parents
->item
, 0, dont_parse
);
97 Get the next rev to send, ignoring the common.
100 static const unsigned char* get_rev(void)
102 struct commit
*commit
= NULL
;
104 while (commit
== NULL
) {
106 struct commit_list
* parents
;
108 if (rev_list
== NULL
|| non_common_revs
== 0)
111 commit
= rev_list
->item
;
112 if (!(commit
->object
.parsed
))
113 parse_commit(commit
);
114 commit
->object
.flags
|= POPPED
;
115 if (!(commit
->object
.flags
& COMMON
))
118 parents
= commit
->parents
;
120 if (commit
->object
.flags
& COMMON
) {
121 /* do not send "have", and ignore ancestors */
123 mark
= COMMON
| SEEN
;
124 } else if (commit
->object
.flags
& COMMON_REF
)
125 /* send "have", and ignore ancestors */
126 mark
= COMMON
| SEEN
;
128 /* send "have", also for its ancestors */
132 if (!(parents
->item
->object
.flags
& SEEN
))
133 rev_list_push(parents
->item
, mark
);
135 mark_common(parents
->item
, 1, 0);
136 parents
= parents
->next
;
139 rev_list
= rev_list
->next
;
142 return commit
->object
.sha1
;
145 static int find_common(int fd
[2], unsigned char *result_sha1
,
149 int count
= 0, flushes
= 0, retval
;
150 const unsigned char *sha1
;
151 unsigned in_vain
= 0;
152 int got_continue
= 0;
154 for_each_ref(rev_list_insert_ref
, NULL
);
157 for ( ; refs
; refs
= refs
->next
) {
158 unsigned char *remote
= refs
->old_sha1
;
162 * If that object is complete (i.e. it is an ancestor of a
163 * local ref), we tell them we have it but do not have to
164 * tell them about its ancestors, which they already know
167 * We use lookup_object here because we are only
168 * interested in the case we *know* the object is
169 * reachable and we have already scanned it.
171 if (((o
= lookup_object(remote
)) != NULL
) &&
172 (o
->flags
& COMPLETE
)) {
177 packet_write(fd
[1], "want %s%s%s%s%s%s%s%s\n",
179 (multi_ack
? " multi_ack" : ""),
180 (use_sideband
== 2 ? " side-band-64k" : ""),
181 (use_sideband
== 1 ? " side-band" : ""),
182 (args
.use_thin_pack
? " thin-pack" : ""),
183 (args
.no_progress
? " no-progress" : ""),
184 (args
.include_tag
? " include-tag" : ""),
187 packet_write(fd
[1], "want %s\n", sha1_to_hex(remote
));
190 if (is_repository_shallow())
191 write_shallow_commits(fd
[1], 1);
193 packet_write(fd
[1], "deepen %d", args
.depth
);
198 if (args
.depth
> 0) {
200 unsigned char sha1
[20];
203 while ((len
= packet_read_line(fd
[0], line
, sizeof(line
)))) {
204 if (!prefixcmp(line
, "shallow ")) {
205 if (get_sha1_hex(line
+ 8, sha1
))
206 die("invalid shallow line: %s", line
);
207 register_shallow(sha1
);
210 if (!prefixcmp(line
, "unshallow ")) {
211 if (get_sha1_hex(line
+ 10, sha1
))
212 die("invalid unshallow line: %s", line
);
213 if (!lookup_object(sha1
))
214 die("object not found: %s", line
);
215 /* make sure that it is parsed as shallow */
217 if (unregister_shallow(sha1
))
218 die("no shallow found: %s", line
);
221 die("expected shallow/unshallow, got %s", line
);
227 while ((sha1
= get_rev())) {
228 packet_write(fd
[1], "have %s\n", sha1_to_hex(sha1
));
230 fprintf(stderr
, "have %s\n", sha1_to_hex(sha1
));
232 if (!(31 & ++count
)) {
239 * We keep one window "ahead" of the other side, and
240 * will wait for an ACK only on the next one
246 ack
= get_ack(fd
[0], result_sha1
);
247 if (args
.verbose
&& ack
)
248 fprintf(stderr
, "got ack %d %s\n", ack
,
249 sha1_to_hex(result_sha1
));
255 } else if (ack
== 2) {
256 struct commit
*commit
=
257 lookup_commit(result_sha1
);
258 mark_common(commit
, 0, 1);
265 if (got_continue
&& MAX_IN_VAIN
< in_vain
) {
267 fprintf(stderr
, "giving up\n");
273 packet_write(fd
[1], "done\n");
275 fprintf(stderr
, "done\n");
280 while (flushes
|| multi_ack
) {
281 int ack
= get_ack(fd
[0], result_sha1
);
284 fprintf(stderr
, "got ack (%d) %s\n", ack
,
285 sha1_to_hex(result_sha1
));
296 static struct commit_list
*complete
;
298 static int mark_complete(const char *path
, const unsigned char *sha1
, int flag
, void *cb_data
)
300 struct object
*o
= parse_object(sha1
);
302 while (o
&& o
->type
== OBJ_TAG
) {
303 struct tag
*t
= (struct tag
*) o
;
305 break; /* broken repository */
306 o
->flags
|= COMPLETE
;
307 o
= parse_object(t
->tagged
->sha1
);
309 if (o
&& o
->type
== OBJ_COMMIT
) {
310 struct commit
*commit
= (struct commit
*)o
;
311 commit
->object
.flags
|= COMPLETE
;
312 insert_by_date(commit
, &complete
);
317 static void mark_recent_complete_commits(unsigned long cutoff
)
319 while (complete
&& cutoff
<= complete
->item
->date
) {
321 fprintf(stderr
, "Marking %s as complete\n",
322 sha1_to_hex(complete
->item
->object
.sha1
));
323 pop_most_recent_commit(&complete
, COMPLETE
);
327 static void filter_refs(struct ref
**refs
, int nr_match
, char **match
)
329 struct ref
**return_refs
;
330 struct ref
*newlist
= NULL
;
331 struct ref
**newtail
= &newlist
;
332 struct ref
*ref
, *next
;
333 struct ref
*fastarray
[32];
335 if (nr_match
&& !args
.fetch_all
) {
336 if (ARRAY_SIZE(fastarray
) < nr_match
)
337 return_refs
= xcalloc(nr_match
, sizeof(struct ref
*));
339 return_refs
= fastarray
;
340 memset(return_refs
, 0, sizeof(struct ref
*) * nr_match
);
346 for (ref
= *refs
; ref
; ref
= next
) {
348 if (!memcmp(ref
->name
, "refs/", 5) &&
349 check_ref_format(ref
->name
+ 5))
351 else if (args
.fetch_all
&&
352 (!args
.depth
|| prefixcmp(ref
->name
, "refs/tags/") )) {
355 newtail
= &ref
->next
;
359 int order
= path_match(ref
->name
, nr_match
, match
);
361 return_refs
[order
-1] = ref
;
362 continue; /* we will link it later */
368 if (!args
.fetch_all
) {
370 for (i
= 0; i
< nr_match
; i
++) {
371 ref
= return_refs
[i
];
375 newtail
= &ref
->next
;
378 if (return_refs
!= fastarray
)
384 static int everything_local(struct ref
**refs
, int nr_match
, char **match
)
388 unsigned long cutoff
= 0;
390 save_commit_buffer
= 0;
392 for (ref
= *refs
; ref
; ref
= ref
->next
) {
395 o
= parse_object(ref
->old_sha1
);
399 /* We already have it -- which may mean that we were
400 * in sync with the other side at some time after
401 * that (it is OK if we guess wrong here).
403 if (o
->type
== OBJ_COMMIT
) {
404 struct commit
*commit
= (struct commit
*)o
;
405 if (!cutoff
|| cutoff
< commit
->date
)
406 cutoff
= commit
->date
;
411 for_each_ref(mark_complete
, NULL
);
413 mark_recent_complete_commits(cutoff
);
417 * Mark all complete remote refs as common refs.
418 * Don't mark them common yet; the server has to be told so first.
420 for (ref
= *refs
; ref
; ref
= ref
->next
) {
421 struct object
*o
= deref_tag(lookup_object(ref
->old_sha1
),
424 if (!o
|| o
->type
!= OBJ_COMMIT
|| !(o
->flags
& COMPLETE
))
427 if (!(o
->flags
& SEEN
)) {
428 rev_list_push((struct commit
*)o
, COMMON_REF
| SEEN
);
430 mark_common((struct commit
*)o
, 1, 1);
434 filter_refs(refs
, nr_match
, match
);
436 for (retval
= 1, ref
= *refs
; ref
; ref
= ref
->next
) {
437 const unsigned char *remote
= ref
->old_sha1
;
438 unsigned char local
[20];
441 o
= lookup_object(remote
);
442 if (!o
|| !(o
->flags
& COMPLETE
)) {
447 "want %s (%s)\n", sha1_to_hex(remote
),
452 hashcpy(ref
->new_sha1
, local
);
456 "already have %s (%s)\n", sha1_to_hex(remote
),
462 static int sideband_demux(int fd
, void *data
)
466 return recv_sideband("fetch-pack", xd
[0], fd
, 2);
469 static int get_pack(int xd
[2], char **pack_lockfile
)
472 const char *argv
[20];
476 int do_keep
= args
.keep_pack
;
477 struct child_process cmd
;
479 memset(&demux
, 0, sizeof(demux
));
481 /* xd[] is talking with upload-pack; subprocess reads from
482 * xd[0], spits out band#2 to stderr, and feeds us band#1
483 * through demux->out.
485 demux
.proc
= sideband_demux
;
487 if (start_async(&demux
))
488 die("fetch-pack: unable to fork off sideband"
494 memset(&cmd
, 0, sizeof(cmd
));
498 if (!args
.keep_pack
&& unpack_limit
) {
499 struct pack_header header
;
501 if (read_pack_header(demux
.out
, &header
))
502 die("protocol error: bad pack header");
503 snprintf(hdr_arg
, sizeof(hdr_arg
), "--pack_header=%u,%u",
504 ntohl(header
.hdr_version
), ntohl(header
.hdr_entries
));
505 if (ntohl(header
.hdr_entries
) < unpack_limit
)
514 *av
++ = "index-pack";
516 if (!args
.quiet
&& !args
.no_progress
)
518 if (args
.use_thin_pack
)
519 *av
++ = "--fix-thin";
520 if (args
.lock_pack
|| unpack_limit
) {
521 int s
= sprintf(keep_arg
,
522 "--keep=fetch-pack %d on ", getpid());
523 if (gethostname(keep_arg
+ s
, sizeof(keep_arg
) - s
))
524 strcpy(keep_arg
+ s
, "localhost");
529 *av
++ = "unpack-objects";
539 if (start_command(&cmd
))
540 die("fetch-pack: unable to fork off %s", argv
[0]);
541 if (do_keep
&& pack_lockfile
) {
542 *pack_lockfile
= index_pack_lockfile(cmd
.out
);
546 if (finish_command(&cmd
))
547 die("%s failed", argv
[0]);
548 if (use_sideband
&& finish_async(&demux
))
549 die("error in sideband demultiplexer");
553 static struct ref
*do_fetch_pack(int fd
[2],
554 const struct ref
*orig_ref
,
557 char **pack_lockfile
)
559 struct ref
*ref
= copy_ref_list(orig_ref
);
560 unsigned char sha1
[20];
562 if (is_repository_shallow() && !server_supports("shallow"))
563 die("Server does not support shallow clients");
564 if (server_supports("multi_ack")) {
566 fprintf(stderr
, "Server supports multi_ack\n");
569 if (server_supports("side-band-64k")) {
571 fprintf(stderr
, "Server supports side-band-64k\n");
574 else if (server_supports("side-band")) {
576 fprintf(stderr
, "Server supports side-band\n");
579 if (everything_local(&ref
, nr_match
, match
)) {
583 if (find_common(fd
, sha1
, ref
) < 0)
585 /* When cloning, it is not unusual to have
588 fprintf(stderr
, "warning: no common commits\n");
590 if (get_pack(fd
, pack_lockfile
))
591 die("git-fetch-pack: fetch failed.");
597 static int remove_duplicates(int nr_heads
, char **heads
)
601 for (src
= dst
= 0; src
< nr_heads
; src
++) {
602 /* If heads[src] is different from any of
603 * heads[0..dst], push it in.
606 for (i
= 0; i
< dst
; i
++) {
607 if (!strcmp(heads
[i
], heads
[src
]))
613 heads
[dst
] = heads
[src
];
619 static int fetch_pack_config(const char *var
, const char *value
)
621 if (strcmp(var
, "fetch.unpacklimit") == 0) {
622 fetch_unpack_limit
= git_config_int(var
, value
);
626 if (strcmp(var
, "transfer.unpacklimit") == 0) {
627 transfer_unpack_limit
= git_config_int(var
, value
);
631 return git_default_config(var
, value
);
634 static struct lock_file lock
;
636 static void fetch_pack_setup(void)
638 static int did_setup
;
641 git_config(fetch_pack_config
);
642 if (0 <= transfer_unpack_limit
)
643 unpack_limit
= transfer_unpack_limit
;
644 else if (0 <= fetch_unpack_limit
)
645 unpack_limit
= fetch_unpack_limit
;
649 int cmd_fetch_pack(int argc
, const char **argv
, const char *prefix
)
651 int i
, ret
, nr_heads
;
652 struct ref
*ref
= NULL
;
653 char *dest
= NULL
, **heads
;
655 struct child_process
*conn
;
659 for (i
= 1; i
< argc
; i
++) {
660 const char *arg
= argv
[i
];
663 if (!prefixcmp(arg
, "--upload-pack=")) {
664 args
.uploadpack
= arg
+ 14;
667 if (!prefixcmp(arg
, "--exec=")) {
668 args
.uploadpack
= arg
+ 7;
671 if (!strcmp("--quiet", arg
) || !strcmp("-q", arg
)) {
675 if (!strcmp("--keep", arg
) || !strcmp("-k", arg
)) {
676 args
.lock_pack
= args
.keep_pack
;
680 if (!strcmp("--thin", arg
)) {
681 args
.use_thin_pack
= 1;
684 if (!strcmp("--include-tag", arg
)) {
685 args
.include_tag
= 1;
688 if (!strcmp("--all", arg
)) {
692 if (!strcmp("-v", arg
)) {
696 if (!prefixcmp(arg
, "--depth=")) {
697 args
.depth
= strtol(arg
+ 8, NULL
, 0);
700 if (!strcmp("--no-progress", arg
)) {
701 args
.no_progress
= 1;
704 usage(fetch_pack_usage
);
707 heads
= (char **)(argv
+ i
+ 1);
708 nr_heads
= argc
- i
- 1;
712 usage(fetch_pack_usage
);
714 conn
= git_connect(fd
, (char *)dest
, args
.uploadpack
,
715 args
.verbose
? CONNECT_VERBOSE
: 0);
717 get_remote_heads(fd
[0], &ref
, 0, NULL
, 0);
719 ref
= fetch_pack(&args
, fd
, conn
, ref
, dest
, nr_heads
, heads
, NULL
);
722 if (finish_connect(conn
))
729 if (!ret
&& nr_heads
) {
730 /* If the heads to pull were given, we should have
731 * consumed all of them by matching the remote.
732 * Otherwise, 'git-fetch remote no-such-ref' would
733 * silently succeed without issuing an error.
735 for (i
= 0; i
< nr_heads
; i
++)
736 if (heads
[i
] && heads
[i
][0]) {
737 error("no such remote ref %s", heads
[i
]);
743 sha1_to_hex(ref
->old_sha1
), ref
->name
);
750 struct ref
*fetch_pack(struct fetch_pack_args
*my_args
,
751 int fd
[], struct child_process
*conn
,
752 const struct ref
*ref
,
756 char **pack_lockfile
)
762 memcpy(&args
, my_args
, sizeof(args
));
763 if (args
.depth
> 0) {
764 if (stat(git_path("shallow"), &st
))
768 if (heads
&& nr_heads
)
769 nr_heads
= remove_duplicates(nr_heads
, heads
);
772 die("no matching remote head");
774 ref_cpy
= do_fetch_pack(fd
, ref
, nr_heads
, heads
, pack_lockfile
);
776 if (args
.depth
> 0) {
777 struct cache_time mtime
;
778 char *shallow
= git_path("shallow");
781 mtime
.sec
= st
.st_mtime
;
783 mtime
.usec
= st
.st_mtim
.usec
;
785 if (stat(shallow
, &st
)) {
787 die("shallow file was removed during fetch");
788 } else if (st
.st_mtime
!= mtime
.sec
790 || st
.st_mtim
.usec
!= mtime
.usec
793 die("shallow file was changed during fetch");
795 fd
= hold_lock_file_for_update(&lock
, shallow
, 1);
796 if (!write_shallow_commits(fd
, 0)) {
798 rollback_lock_file(&lock
);
800 commit_lock_file(&lock
);