Teach fetch-pack/upload-pack about --include-tag
[git/dscho.git] / builtin-fetch-pack.c
blob34fb6d5c61e723278e70fe5949ed5a529904d958
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9 #include "fetch-pack.h"
10 #include "remote.h"
11 #include "run-command.h"
13 static int transfer_unpack_limit = -1;
14 static int fetch_unpack_limit = -1;
15 static int unpack_limit = 100;
16 static struct fetch_pack_args args = {
17 /* .uploadpack = */ "git-upload-pack",
20 static const char fetch_pack_usage[] =
21 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
23 #define COMPLETE (1U << 0)
24 #define COMMON (1U << 1)
25 #define COMMON_REF (1U << 2)
26 #define SEEN (1U << 3)
27 #define POPPED (1U << 4)
30 * After sending this many "have"s if we do not get any new ACK , we
31 * give up traversing our history.
33 #define MAX_IN_VAIN 256
35 static struct commit_list *rev_list;
36 static int non_common_revs, multi_ack, use_sideband;
38 static void rev_list_push(struct commit *commit, int mark)
40 if (!(commit->object.flags & mark)) {
41 commit->object.flags |= mark;
43 if (!(commit->object.parsed))
44 parse_commit(commit);
46 insert_by_date(commit, &rev_list);
48 if (!(commit->object.flags & COMMON))
49 non_common_revs++;
53 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
55 struct object *o = deref_tag(parse_object(sha1), path, 0);
57 if (o && o->type == OBJ_COMMIT)
58 rev_list_push((struct commit *)o, SEEN);
60 return 0;
64 This function marks a rev and its ancestors as common.
65 In some cases, it is desirable to mark only the ancestors (for example
66 when only the server does not yet know that they are common).
69 static void mark_common(struct commit *commit,
70 int ancestors_only, int dont_parse)
72 if (commit != NULL && !(commit->object.flags & COMMON)) {
73 struct object *o = (struct object *)commit;
75 if (!ancestors_only)
76 o->flags |= COMMON;
78 if (!(o->flags & SEEN))
79 rev_list_push(commit, SEEN);
80 else {
81 struct commit_list *parents;
83 if (!ancestors_only && !(o->flags & POPPED))
84 non_common_revs--;
85 if (!o->parsed && !dont_parse)
86 parse_commit(commit);
88 for (parents = commit->parents;
89 parents;
90 parents = parents->next)
91 mark_common(parents->item, 0, dont_parse);
97 Get the next rev to send, ignoring the common.
100 static const unsigned char* get_rev(void)
102 struct commit *commit = NULL;
104 while (commit == NULL) {
105 unsigned int mark;
106 struct commit_list* parents;
108 if (rev_list == NULL || non_common_revs == 0)
109 return NULL;
111 commit = rev_list->item;
112 if (!(commit->object.parsed))
113 parse_commit(commit);
114 commit->object.flags |= POPPED;
115 if (!(commit->object.flags & COMMON))
116 non_common_revs--;
118 parents = commit->parents;
120 if (commit->object.flags & COMMON) {
121 /* do not send "have", and ignore ancestors */
122 commit = NULL;
123 mark = COMMON | SEEN;
124 } else if (commit->object.flags & COMMON_REF)
125 /* send "have", and ignore ancestors */
126 mark = COMMON | SEEN;
127 else
128 /* send "have", also for its ancestors */
129 mark = SEEN;
131 while (parents) {
132 if (!(parents->item->object.flags & SEEN))
133 rev_list_push(parents->item, mark);
134 if (mark & COMMON)
135 mark_common(parents->item, 1, 0);
136 parents = parents->next;
139 rev_list = rev_list->next;
142 return commit->object.sha1;
145 static int find_common(int fd[2], unsigned char *result_sha1,
146 struct ref *refs)
148 int fetching;
149 int count = 0, flushes = 0, retval;
150 const unsigned char *sha1;
151 unsigned in_vain = 0;
152 int got_continue = 0;
154 for_each_ref(rev_list_insert_ref, NULL);
156 fetching = 0;
157 for ( ; refs ; refs = refs->next) {
158 unsigned char *remote = refs->old_sha1;
159 struct object *o;
162 * If that object is complete (i.e. it is an ancestor of a
163 * local ref), we tell them we have it but do not have to
164 * tell them about its ancestors, which they already know
165 * about.
167 * We use lookup_object here because we are only
168 * interested in the case we *know* the object is
169 * reachable and we have already scanned it.
171 if (((o = lookup_object(remote)) != NULL) &&
172 (o->flags & COMPLETE)) {
173 continue;
176 if (!fetching)
177 packet_write(fd[1], "want %s%s%s%s%s%s%s%s\n",
178 sha1_to_hex(remote),
179 (multi_ack ? " multi_ack" : ""),
180 (use_sideband == 2 ? " side-band-64k" : ""),
181 (use_sideband == 1 ? " side-band" : ""),
182 (args.use_thin_pack ? " thin-pack" : ""),
183 (args.no_progress ? " no-progress" : ""),
184 (args.include_tag ? " include-tag" : ""),
185 " ofs-delta");
186 else
187 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
188 fetching++;
190 if (is_repository_shallow())
191 write_shallow_commits(fd[1], 1);
192 if (args.depth > 0)
193 packet_write(fd[1], "deepen %d", args.depth);
194 packet_flush(fd[1]);
195 if (!fetching)
196 return 1;
198 if (args.depth > 0) {
199 char line[1024];
200 unsigned char sha1[20];
201 int len;
203 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
204 if (!prefixcmp(line, "shallow ")) {
205 if (get_sha1_hex(line + 8, sha1))
206 die("invalid shallow line: %s", line);
207 register_shallow(sha1);
208 continue;
210 if (!prefixcmp(line, "unshallow ")) {
211 if (get_sha1_hex(line + 10, sha1))
212 die("invalid unshallow line: %s", line);
213 if (!lookup_object(sha1))
214 die("object not found: %s", line);
215 /* make sure that it is parsed as shallow */
216 parse_object(sha1);
217 if (unregister_shallow(sha1))
218 die("no shallow found: %s", line);
219 continue;
221 die("expected shallow/unshallow, got %s", line);
225 flushes = 0;
226 retval = -1;
227 while ((sha1 = get_rev())) {
228 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
229 if (args.verbose)
230 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
231 in_vain++;
232 if (!(31 & ++count)) {
233 int ack;
235 packet_flush(fd[1]);
236 flushes++;
239 * We keep one window "ahead" of the other side, and
240 * will wait for an ACK only on the next one
242 if (count == 32)
243 continue;
245 do {
246 ack = get_ack(fd[0], result_sha1);
247 if (args.verbose && ack)
248 fprintf(stderr, "got ack %d %s\n", ack,
249 sha1_to_hex(result_sha1));
250 if (ack == 1) {
251 flushes = 0;
252 multi_ack = 0;
253 retval = 0;
254 goto done;
255 } else if (ack == 2) {
256 struct commit *commit =
257 lookup_commit(result_sha1);
258 mark_common(commit, 0, 1);
259 retval = 0;
260 in_vain = 0;
261 got_continue = 1;
263 } while (ack);
264 flushes--;
265 if (got_continue && MAX_IN_VAIN < in_vain) {
266 if (args.verbose)
267 fprintf(stderr, "giving up\n");
268 break; /* give up */
272 done:
273 packet_write(fd[1], "done\n");
274 if (args.verbose)
275 fprintf(stderr, "done\n");
276 if (retval != 0) {
277 multi_ack = 0;
278 flushes++;
280 while (flushes || multi_ack) {
281 int ack = get_ack(fd[0], result_sha1);
282 if (ack) {
283 if (args.verbose)
284 fprintf(stderr, "got ack (%d) %s\n", ack,
285 sha1_to_hex(result_sha1));
286 if (ack == 1)
287 return 0;
288 multi_ack = 1;
289 continue;
291 flushes--;
293 return retval;
296 static struct commit_list *complete;
298 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
300 struct object *o = parse_object(sha1);
302 while (o && o->type == OBJ_TAG) {
303 struct tag *t = (struct tag *) o;
304 if (!t->tagged)
305 break; /* broken repository */
306 o->flags |= COMPLETE;
307 o = parse_object(t->tagged->sha1);
309 if (o && o->type == OBJ_COMMIT) {
310 struct commit *commit = (struct commit *)o;
311 commit->object.flags |= COMPLETE;
312 insert_by_date(commit, &complete);
314 return 0;
317 static void mark_recent_complete_commits(unsigned long cutoff)
319 while (complete && cutoff <= complete->item->date) {
320 if (args.verbose)
321 fprintf(stderr, "Marking %s as complete\n",
322 sha1_to_hex(complete->item->object.sha1));
323 pop_most_recent_commit(&complete, COMPLETE);
327 static void filter_refs(struct ref **refs, int nr_match, char **match)
329 struct ref **return_refs;
330 struct ref *newlist = NULL;
331 struct ref **newtail = &newlist;
332 struct ref *ref, *next;
333 struct ref *fastarray[32];
335 if (nr_match && !args.fetch_all) {
336 if (ARRAY_SIZE(fastarray) < nr_match)
337 return_refs = xcalloc(nr_match, sizeof(struct ref *));
338 else {
339 return_refs = fastarray;
340 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
343 else
344 return_refs = NULL;
346 for (ref = *refs; ref; ref = next) {
347 next = ref->next;
348 if (!memcmp(ref->name, "refs/", 5) &&
349 check_ref_format(ref->name + 5))
350 ; /* trash */
351 else if (args.fetch_all &&
352 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
353 *newtail = ref;
354 ref->next = NULL;
355 newtail = &ref->next;
356 continue;
358 else {
359 int order = path_match(ref->name, nr_match, match);
360 if (order) {
361 return_refs[order-1] = ref;
362 continue; /* we will link it later */
365 free(ref);
368 if (!args.fetch_all) {
369 int i;
370 for (i = 0; i < nr_match; i++) {
371 ref = return_refs[i];
372 if (ref) {
373 *newtail = ref;
374 ref->next = NULL;
375 newtail = &ref->next;
378 if (return_refs != fastarray)
379 free(return_refs);
381 *refs = newlist;
384 static int everything_local(struct ref **refs, int nr_match, char **match)
386 struct ref *ref;
387 int retval;
388 unsigned long cutoff = 0;
390 save_commit_buffer = 0;
392 for (ref = *refs; ref; ref = ref->next) {
393 struct object *o;
395 o = parse_object(ref->old_sha1);
396 if (!o)
397 continue;
399 /* We already have it -- which may mean that we were
400 * in sync with the other side at some time after
401 * that (it is OK if we guess wrong here).
403 if (o->type == OBJ_COMMIT) {
404 struct commit *commit = (struct commit *)o;
405 if (!cutoff || cutoff < commit->date)
406 cutoff = commit->date;
410 if (!args.depth) {
411 for_each_ref(mark_complete, NULL);
412 if (cutoff)
413 mark_recent_complete_commits(cutoff);
417 * Mark all complete remote refs as common refs.
418 * Don't mark them common yet; the server has to be told so first.
420 for (ref = *refs; ref; ref = ref->next) {
421 struct object *o = deref_tag(lookup_object(ref->old_sha1),
422 NULL, 0);
424 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
425 continue;
427 if (!(o->flags & SEEN)) {
428 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
430 mark_common((struct commit *)o, 1, 1);
434 filter_refs(refs, nr_match, match);
436 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
437 const unsigned char *remote = ref->old_sha1;
438 unsigned char local[20];
439 struct object *o;
441 o = lookup_object(remote);
442 if (!o || !(o->flags & COMPLETE)) {
443 retval = 0;
444 if (!args.verbose)
445 continue;
446 fprintf(stderr,
447 "want %s (%s)\n", sha1_to_hex(remote),
448 ref->name);
449 continue;
452 hashcpy(ref->new_sha1, local);
453 if (!args.verbose)
454 continue;
455 fprintf(stderr,
456 "already have %s (%s)\n", sha1_to_hex(remote),
457 ref->name);
459 return retval;
462 static int sideband_demux(int fd, void *data)
464 int *xd = data;
466 return recv_sideband("fetch-pack", xd[0], fd, 2);
469 static int get_pack(int xd[2], char **pack_lockfile)
471 struct async demux;
472 const char *argv[20];
473 char keep_arg[256];
474 char hdr_arg[256];
475 const char **av;
476 int do_keep = args.keep_pack;
477 struct child_process cmd;
479 memset(&demux, 0, sizeof(demux));
480 if (use_sideband) {
481 /* xd[] is talking with upload-pack; subprocess reads from
482 * xd[0], spits out band#2 to stderr, and feeds us band#1
483 * through demux->out.
485 demux.proc = sideband_demux;
486 demux.data = xd;
487 if (start_async(&demux))
488 die("fetch-pack: unable to fork off sideband"
489 " demultiplexer");
491 else
492 demux.out = xd[0];
494 memset(&cmd, 0, sizeof(cmd));
495 cmd.argv = argv;
496 av = argv;
497 *hdr_arg = 0;
498 if (!args.keep_pack && unpack_limit) {
499 struct pack_header header;
501 if (read_pack_header(demux.out, &header))
502 die("protocol error: bad pack header");
503 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
504 ntohl(header.hdr_version), ntohl(header.hdr_entries));
505 if (ntohl(header.hdr_entries) < unpack_limit)
506 do_keep = 0;
507 else
508 do_keep = 1;
511 if (do_keep) {
512 if (pack_lockfile)
513 cmd.out = -1;
514 *av++ = "index-pack";
515 *av++ = "--stdin";
516 if (!args.quiet && !args.no_progress)
517 *av++ = "-v";
518 if (args.use_thin_pack)
519 *av++ = "--fix-thin";
520 if (args.lock_pack || unpack_limit) {
521 int s = sprintf(keep_arg,
522 "--keep=fetch-pack %d on ", getpid());
523 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
524 strcpy(keep_arg + s, "localhost");
525 *av++ = keep_arg;
528 else {
529 *av++ = "unpack-objects";
530 if (args.quiet)
531 *av++ = "-q";
533 if (*hdr_arg)
534 *av++ = hdr_arg;
535 *av++ = NULL;
537 cmd.in = demux.out;
538 cmd.git_cmd = 1;
539 if (start_command(&cmd))
540 die("fetch-pack: unable to fork off %s", argv[0]);
541 if (do_keep && pack_lockfile) {
542 *pack_lockfile = index_pack_lockfile(cmd.out);
543 close(cmd.out);
546 if (finish_command(&cmd))
547 die("%s failed", argv[0]);
548 if (use_sideband && finish_async(&demux))
549 die("error in sideband demultiplexer");
550 return 0;
553 static struct ref *do_fetch_pack(int fd[2],
554 const struct ref *orig_ref,
555 int nr_match,
556 char **match,
557 char **pack_lockfile)
559 struct ref *ref = copy_ref_list(orig_ref);
560 unsigned char sha1[20];
562 if (is_repository_shallow() && !server_supports("shallow"))
563 die("Server does not support shallow clients");
564 if (server_supports("multi_ack")) {
565 if (args.verbose)
566 fprintf(stderr, "Server supports multi_ack\n");
567 multi_ack = 1;
569 if (server_supports("side-band-64k")) {
570 if (args.verbose)
571 fprintf(stderr, "Server supports side-band-64k\n");
572 use_sideband = 2;
574 else if (server_supports("side-band")) {
575 if (args.verbose)
576 fprintf(stderr, "Server supports side-band\n");
577 use_sideband = 1;
579 if (everything_local(&ref, nr_match, match)) {
580 packet_flush(fd[1]);
581 goto all_done;
583 if (find_common(fd, sha1, ref) < 0)
584 if (!args.keep_pack)
585 /* When cloning, it is not unusual to have
586 * no common commit.
588 fprintf(stderr, "warning: no common commits\n");
590 if (get_pack(fd, pack_lockfile))
591 die("git-fetch-pack: fetch failed.");
593 all_done:
594 return ref;
597 static int remove_duplicates(int nr_heads, char **heads)
599 int src, dst;
601 for (src = dst = 0; src < nr_heads; src++) {
602 /* If heads[src] is different from any of
603 * heads[0..dst], push it in.
605 int i;
606 for (i = 0; i < dst; i++) {
607 if (!strcmp(heads[i], heads[src]))
608 break;
610 if (i < dst)
611 continue;
612 if (src != dst)
613 heads[dst] = heads[src];
614 dst++;
616 return dst;
619 static int fetch_pack_config(const char *var, const char *value)
621 if (strcmp(var, "fetch.unpacklimit") == 0) {
622 fetch_unpack_limit = git_config_int(var, value);
623 return 0;
626 if (strcmp(var, "transfer.unpacklimit") == 0) {
627 transfer_unpack_limit = git_config_int(var, value);
628 return 0;
631 return git_default_config(var, value);
634 static struct lock_file lock;
636 static void fetch_pack_setup(void)
638 static int did_setup;
639 if (did_setup)
640 return;
641 git_config(fetch_pack_config);
642 if (0 <= transfer_unpack_limit)
643 unpack_limit = transfer_unpack_limit;
644 else if (0 <= fetch_unpack_limit)
645 unpack_limit = fetch_unpack_limit;
646 did_setup = 1;
649 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
651 int i, ret, nr_heads;
652 struct ref *ref = NULL;
653 char *dest = NULL, **heads;
654 int fd[2];
655 struct child_process *conn;
657 nr_heads = 0;
658 heads = NULL;
659 for (i = 1; i < argc; i++) {
660 const char *arg = argv[i];
662 if (*arg == '-') {
663 if (!prefixcmp(arg, "--upload-pack=")) {
664 args.uploadpack = arg + 14;
665 continue;
667 if (!prefixcmp(arg, "--exec=")) {
668 args.uploadpack = arg + 7;
669 continue;
671 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
672 args.quiet = 1;
673 continue;
675 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
676 args.lock_pack = args.keep_pack;
677 args.keep_pack = 1;
678 continue;
680 if (!strcmp("--thin", arg)) {
681 args.use_thin_pack = 1;
682 continue;
684 if (!strcmp("--include-tag", arg)) {
685 args.include_tag = 1;
686 continue;
688 if (!strcmp("--all", arg)) {
689 args.fetch_all = 1;
690 continue;
692 if (!strcmp("-v", arg)) {
693 args.verbose = 1;
694 continue;
696 if (!prefixcmp(arg, "--depth=")) {
697 args.depth = strtol(arg + 8, NULL, 0);
698 continue;
700 if (!strcmp("--no-progress", arg)) {
701 args.no_progress = 1;
702 continue;
704 usage(fetch_pack_usage);
706 dest = (char *)arg;
707 heads = (char **)(argv + i + 1);
708 nr_heads = argc - i - 1;
709 break;
711 if (!dest)
712 usage(fetch_pack_usage);
714 conn = git_connect(fd, (char *)dest, args.uploadpack,
715 args.verbose ? CONNECT_VERBOSE : 0);
716 if (conn) {
717 get_remote_heads(fd[0], &ref, 0, NULL, 0);
719 ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
720 close(fd[0]);
721 close(fd[1]);
722 if (finish_connect(conn))
723 ref = NULL;
724 } else {
725 ref = NULL;
727 ret = !ref;
729 if (!ret && nr_heads) {
730 /* If the heads to pull were given, we should have
731 * consumed all of them by matching the remote.
732 * Otherwise, 'git-fetch remote no-such-ref' would
733 * silently succeed without issuing an error.
735 for (i = 0; i < nr_heads; i++)
736 if (heads[i] && heads[i][0]) {
737 error("no such remote ref %s", heads[i]);
738 ret = 1;
741 while (ref) {
742 printf("%s %s\n",
743 sha1_to_hex(ref->old_sha1), ref->name);
744 ref = ref->next;
747 return ret;
750 struct ref *fetch_pack(struct fetch_pack_args *my_args,
751 int fd[], struct child_process *conn,
752 const struct ref *ref,
753 const char *dest,
754 int nr_heads,
755 char **heads,
756 char **pack_lockfile)
758 struct stat st;
759 struct ref *ref_cpy;
761 fetch_pack_setup();
762 memcpy(&args, my_args, sizeof(args));
763 if (args.depth > 0) {
764 if (stat(git_path("shallow"), &st))
765 st.st_mtime = 0;
768 if (heads && nr_heads)
769 nr_heads = remove_duplicates(nr_heads, heads);
770 if (!ref) {
771 packet_flush(fd[1]);
772 die("no matching remote head");
774 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
776 if (args.depth > 0) {
777 struct cache_time mtime;
778 char *shallow = git_path("shallow");
779 int fd;
781 mtime.sec = st.st_mtime;
782 #ifdef USE_NSEC
783 mtime.usec = st.st_mtim.usec;
784 #endif
785 if (stat(shallow, &st)) {
786 if (mtime.sec)
787 die("shallow file was removed during fetch");
788 } else if (st.st_mtime != mtime.sec
789 #ifdef USE_NSEC
790 || st.st_mtim.usec != mtime.usec
791 #endif
793 die("shallow file was changed during fetch");
795 fd = hold_lock_file_for_update(&lock, shallow, 1);
796 if (!write_shallow_commits(fd, 0)) {
797 unlink(shallow);
798 rollback_lock_file(&lock);
799 } else {
800 commit_lock_file(&lock);
804 return ref_cpy;