builtin-fsck: reports missing parent commits
[git/jrn.git] / builtin-fetch-pack.c
blob410414d91fc5249a2152600f5a7fb5d90745942d
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9 #include "fetch-pack.h"
10 #include "run-command.h"
12 static int transfer_unpack_limit = -1;
13 static int fetch_unpack_limit = -1;
14 static int unpack_limit = 100;
15 static struct fetch_pack_args args = {
16 /* .uploadpack = */ "git-upload-pack",
19 static const char fetch_pack_usage[] =
20 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
22 #define COMPLETE (1U << 0)
23 #define COMMON (1U << 1)
24 #define COMMON_REF (1U << 2)
25 #define SEEN (1U << 3)
26 #define POPPED (1U << 4)
29 * After sending this many "have"s if we do not get any new ACK , we
30 * give up traversing our history.
32 #define MAX_IN_VAIN 256
34 static struct commit_list *rev_list;
35 static int non_common_revs, multi_ack, use_sideband;
37 static void rev_list_push(struct commit *commit, int mark)
39 if (!(commit->object.flags & mark)) {
40 commit->object.flags |= mark;
42 if (!(commit->object.parsed))
43 parse_commit(commit);
45 insert_by_date(commit, &rev_list);
47 if (!(commit->object.flags & COMMON))
48 non_common_revs++;
52 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
54 struct object *o = deref_tag(parse_object(sha1), path, 0);
56 if (o && o->type == OBJ_COMMIT)
57 rev_list_push((struct commit *)o, SEEN);
59 return 0;
63 This function marks a rev and its ancestors as common.
64 In some cases, it is desirable to mark only the ancestors (for example
65 when only the server does not yet know that they are common).
68 static void mark_common(struct commit *commit,
69 int ancestors_only, int dont_parse)
71 if (commit != NULL && !(commit->object.flags & COMMON)) {
72 struct object *o = (struct object *)commit;
74 if (!ancestors_only)
75 o->flags |= COMMON;
77 if (!(o->flags & SEEN))
78 rev_list_push(commit, SEEN);
79 else {
80 struct commit_list *parents;
82 if (!ancestors_only && !(o->flags & POPPED))
83 non_common_revs--;
84 if (!o->parsed && !dont_parse)
85 parse_commit(commit);
87 for (parents = commit->parents;
88 parents;
89 parents = parents->next)
90 mark_common(parents->item, 0, dont_parse);
96 Get the next rev to send, ignoring the common.
99 static const unsigned char* get_rev(void)
101 struct commit *commit = NULL;
103 while (commit == NULL) {
104 unsigned int mark;
105 struct commit_list* parents;
107 if (rev_list == NULL || non_common_revs == 0)
108 return NULL;
110 commit = rev_list->item;
111 if (!(commit->object.parsed))
112 parse_commit(commit);
113 commit->object.flags |= POPPED;
114 if (!(commit->object.flags & COMMON))
115 non_common_revs--;
117 parents = commit->parents;
119 if (commit->object.flags & COMMON) {
120 /* do not send "have", and ignore ancestors */
121 commit = NULL;
122 mark = COMMON | SEEN;
123 } else if (commit->object.flags & COMMON_REF)
124 /* send "have", and ignore ancestors */
125 mark = COMMON | SEEN;
126 else
127 /* send "have", also for its ancestors */
128 mark = SEEN;
130 while (parents) {
131 if (!(parents->item->object.flags & SEEN))
132 rev_list_push(parents->item, mark);
133 if (mark & COMMON)
134 mark_common(parents->item, 1, 0);
135 parents = parents->next;
138 rev_list = rev_list->next;
141 return commit->object.sha1;
144 static int find_common(int fd[2], unsigned char *result_sha1,
145 struct ref *refs)
147 int fetching;
148 int count = 0, flushes = 0, retval;
149 const unsigned char *sha1;
150 unsigned in_vain = 0;
151 int got_continue = 0;
153 for_each_ref(rev_list_insert_ref, NULL);
155 fetching = 0;
156 for ( ; refs ; refs = refs->next) {
157 unsigned char *remote = refs->old_sha1;
158 struct object *o;
161 * If that object is complete (i.e. it is an ancestor of a
162 * local ref), we tell them we have it but do not have to
163 * tell them about its ancestors, which they already know
164 * about.
166 * We use lookup_object here because we are only
167 * interested in the case we *know* the object is
168 * reachable and we have already scanned it.
170 if (((o = lookup_object(remote)) != NULL) &&
171 (o->flags & COMPLETE)) {
172 continue;
175 if (!fetching)
176 packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
177 sha1_to_hex(remote),
178 (multi_ack ? " multi_ack" : ""),
179 (use_sideband == 2 ? " side-band-64k" : ""),
180 (use_sideband == 1 ? " side-band" : ""),
181 (args.use_thin_pack ? " thin-pack" : ""),
182 (args.no_progress ? " no-progress" : ""),
183 " ofs-delta");
184 else
185 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
186 fetching++;
188 if (is_repository_shallow())
189 write_shallow_commits(fd[1], 1);
190 if (args.depth > 0)
191 packet_write(fd[1], "deepen %d", args.depth);
192 packet_flush(fd[1]);
193 if (!fetching)
194 return 1;
196 if (args.depth > 0) {
197 char line[1024];
198 unsigned char sha1[20];
199 int len;
201 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
202 if (!prefixcmp(line, "shallow ")) {
203 if (get_sha1_hex(line + 8, sha1))
204 die("invalid shallow line: %s", line);
205 register_shallow(sha1);
206 continue;
208 if (!prefixcmp(line, "unshallow ")) {
209 if (get_sha1_hex(line + 10, sha1))
210 die("invalid unshallow line: %s", line);
211 if (!lookup_object(sha1))
212 die("object not found: %s", line);
213 /* make sure that it is parsed as shallow */
214 parse_object(sha1);
215 if (unregister_shallow(sha1))
216 die("no shallow found: %s", line);
217 continue;
219 die("expected shallow/unshallow, got %s", line);
223 flushes = 0;
224 retval = -1;
225 while ((sha1 = get_rev())) {
226 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
227 if (args.verbose)
228 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
229 in_vain++;
230 if (!(31 & ++count)) {
231 int ack;
233 packet_flush(fd[1]);
234 flushes++;
237 * We keep one window "ahead" of the other side, and
238 * will wait for an ACK only on the next one
240 if (count == 32)
241 continue;
243 do {
244 ack = get_ack(fd[0], result_sha1);
245 if (args.verbose && ack)
246 fprintf(stderr, "got ack %d %s\n", ack,
247 sha1_to_hex(result_sha1));
248 if (ack == 1) {
249 flushes = 0;
250 multi_ack = 0;
251 retval = 0;
252 goto done;
253 } else if (ack == 2) {
254 struct commit *commit =
255 lookup_commit(result_sha1);
256 mark_common(commit, 0, 1);
257 retval = 0;
258 in_vain = 0;
259 got_continue = 1;
261 } while (ack);
262 flushes--;
263 if (got_continue && MAX_IN_VAIN < in_vain) {
264 if (args.verbose)
265 fprintf(stderr, "giving up\n");
266 break; /* give up */
270 done:
271 packet_write(fd[1], "done\n");
272 if (args.verbose)
273 fprintf(stderr, "done\n");
274 if (retval != 0) {
275 multi_ack = 0;
276 flushes++;
278 while (flushes || multi_ack) {
279 int ack = get_ack(fd[0], result_sha1);
280 if (ack) {
281 if (args.verbose)
282 fprintf(stderr, "got ack (%d) %s\n", ack,
283 sha1_to_hex(result_sha1));
284 if (ack == 1)
285 return 0;
286 multi_ack = 1;
287 continue;
289 flushes--;
291 return retval;
294 static struct commit_list *complete;
296 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
298 struct object *o = parse_object(sha1);
300 while (o && o->type == OBJ_TAG) {
301 struct tag *t = (struct tag *) o;
302 if (!t->tagged)
303 break; /* broken repository */
304 o->flags |= COMPLETE;
305 o = parse_object(t->tagged->sha1);
307 if (o && o->type == OBJ_COMMIT) {
308 struct commit *commit = (struct commit *)o;
309 commit->object.flags |= COMPLETE;
310 insert_by_date(commit, &complete);
312 return 0;
315 static void mark_recent_complete_commits(unsigned long cutoff)
317 while (complete && cutoff <= complete->item->date) {
318 if (args.verbose)
319 fprintf(stderr, "Marking %s as complete\n",
320 sha1_to_hex(complete->item->object.sha1));
321 pop_most_recent_commit(&complete, COMPLETE);
325 static void filter_refs(struct ref **refs, int nr_match, char **match)
327 struct ref **return_refs;
328 struct ref *newlist = NULL;
329 struct ref **newtail = &newlist;
330 struct ref *ref, *next;
331 struct ref *fastarray[32];
333 if (nr_match && !args.fetch_all) {
334 if (ARRAY_SIZE(fastarray) < nr_match)
335 return_refs = xcalloc(nr_match, sizeof(struct ref *));
336 else {
337 return_refs = fastarray;
338 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
341 else
342 return_refs = NULL;
344 for (ref = *refs; ref; ref = next) {
345 next = ref->next;
346 if (!memcmp(ref->name, "refs/", 5) &&
347 check_ref_format(ref->name + 5))
348 ; /* trash */
349 else if (args.fetch_all &&
350 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
351 *newtail = ref;
352 ref->next = NULL;
353 newtail = &ref->next;
354 continue;
356 else {
357 int order = path_match(ref->name, nr_match, match);
358 if (order) {
359 return_refs[order-1] = ref;
360 continue; /* we will link it later */
363 free(ref);
366 if (!args.fetch_all) {
367 int i;
368 for (i = 0; i < nr_match; i++) {
369 ref = return_refs[i];
370 if (ref) {
371 *newtail = ref;
372 ref->next = NULL;
373 newtail = &ref->next;
376 if (return_refs != fastarray)
377 free(return_refs);
379 *refs = newlist;
382 static int everything_local(struct ref **refs, int nr_match, char **match)
384 struct ref *ref;
385 int retval;
386 unsigned long cutoff = 0;
388 save_commit_buffer = 0;
390 for (ref = *refs; ref; ref = ref->next) {
391 struct object *o;
393 o = parse_object(ref->old_sha1);
394 if (!o)
395 continue;
397 /* We already have it -- which may mean that we were
398 * in sync with the other side at some time after
399 * that (it is OK if we guess wrong here).
401 if (o->type == OBJ_COMMIT) {
402 struct commit *commit = (struct commit *)o;
403 if (!cutoff || cutoff < commit->date)
404 cutoff = commit->date;
408 if (!args.depth) {
409 for_each_ref(mark_complete, NULL);
410 if (cutoff)
411 mark_recent_complete_commits(cutoff);
415 * Mark all complete remote refs as common refs.
416 * Don't mark them common yet; the server has to be told so first.
418 for (ref = *refs; ref; ref = ref->next) {
419 struct object *o = deref_tag(lookup_object(ref->old_sha1),
420 NULL, 0);
422 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
423 continue;
425 if (!(o->flags & SEEN)) {
426 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
428 mark_common((struct commit *)o, 1, 1);
432 filter_refs(refs, nr_match, match);
434 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
435 const unsigned char *remote = ref->old_sha1;
436 unsigned char local[20];
437 struct object *o;
439 o = lookup_object(remote);
440 if (!o || !(o->flags & COMPLETE)) {
441 retval = 0;
442 if (!args.verbose)
443 continue;
444 fprintf(stderr,
445 "want %s (%s)\n", sha1_to_hex(remote),
446 ref->name);
447 continue;
450 hashcpy(ref->new_sha1, local);
451 if (!args.verbose)
452 continue;
453 fprintf(stderr,
454 "already have %s (%s)\n", sha1_to_hex(remote),
455 ref->name);
457 return retval;
460 static int sideband_demux(int fd, void *data)
462 int *xd = data;
464 return recv_sideband("fetch-pack", xd[0], fd, 2);
467 static int get_pack(int xd[2], char **pack_lockfile)
469 struct async demux;
470 const char *argv[20];
471 char keep_arg[256];
472 char hdr_arg[256];
473 const char **av;
474 int do_keep = args.keep_pack;
475 struct child_process cmd;
477 memset(&demux, 0, sizeof(demux));
478 if (use_sideband) {
479 /* xd[] is talking with upload-pack; subprocess reads from
480 * xd[0], spits out band#2 to stderr, and feeds us band#1
481 * through demux->out.
483 demux.proc = sideband_demux;
484 demux.data = xd;
485 if (start_async(&demux))
486 die("fetch-pack: unable to fork off sideband"
487 " demultiplexer");
489 else
490 demux.out = xd[0];
492 memset(&cmd, 0, sizeof(cmd));
493 cmd.argv = argv;
494 av = argv;
495 *hdr_arg = 0;
496 if (!args.keep_pack && unpack_limit) {
497 struct pack_header header;
499 if (read_pack_header(demux.out, &header))
500 die("protocol error: bad pack header");
501 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
502 ntohl(header.hdr_version), ntohl(header.hdr_entries));
503 if (ntohl(header.hdr_entries) < unpack_limit)
504 do_keep = 0;
505 else
506 do_keep = 1;
509 if (do_keep) {
510 if (pack_lockfile)
511 cmd.out = -1;
512 *av++ = "index-pack";
513 *av++ = "--stdin";
514 if (!args.quiet && !args.no_progress)
515 *av++ = "-v";
516 if (args.use_thin_pack)
517 *av++ = "--fix-thin";
518 if (args.lock_pack || unpack_limit) {
519 int s = sprintf(keep_arg,
520 "--keep=fetch-pack %d on ", getpid());
521 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
522 strcpy(keep_arg + s, "localhost");
523 *av++ = keep_arg;
526 else {
527 *av++ = "unpack-objects";
528 if (args.quiet)
529 *av++ = "-q";
531 if (*hdr_arg)
532 *av++ = hdr_arg;
533 *av++ = NULL;
535 cmd.in = demux.out;
536 cmd.git_cmd = 1;
537 if (start_command(&cmd))
538 die("fetch-pack: unable to fork off %s", argv[0]);
539 if (do_keep && pack_lockfile)
540 *pack_lockfile = index_pack_lockfile(cmd.out);
542 if (finish_command(&cmd))
543 die("%s failed", argv[0]);
544 if (use_sideband && finish_async(&demux))
545 die("error in sideband demultiplexer");
546 return 0;
549 static struct ref *do_fetch_pack(int fd[2],
550 int nr_match,
551 char **match,
552 char **pack_lockfile)
554 struct ref *ref;
555 unsigned char sha1[20];
557 get_remote_heads(fd[0], &ref, 0, NULL, 0);
558 if (is_repository_shallow() && !server_supports("shallow"))
559 die("Server does not support shallow clients");
560 if (server_supports("multi_ack")) {
561 if (args.verbose)
562 fprintf(stderr, "Server supports multi_ack\n");
563 multi_ack = 1;
565 if (server_supports("side-band-64k")) {
566 if (args.verbose)
567 fprintf(stderr, "Server supports side-band-64k\n");
568 use_sideband = 2;
570 else if (server_supports("side-band")) {
571 if (args.verbose)
572 fprintf(stderr, "Server supports side-band\n");
573 use_sideband = 1;
575 if (!ref) {
576 packet_flush(fd[1]);
577 die("no matching remote head");
579 if (everything_local(&ref, nr_match, match)) {
580 packet_flush(fd[1]);
581 goto all_done;
583 if (find_common(fd, sha1, ref) < 0)
584 if (!args.keep_pack)
585 /* When cloning, it is not unusual to have
586 * no common commit.
588 fprintf(stderr, "warning: no common commits\n");
590 if (get_pack(fd, pack_lockfile))
591 die("git-fetch-pack: fetch failed.");
593 all_done:
594 return ref;
597 static int remove_duplicates(int nr_heads, char **heads)
599 int src, dst;
601 for (src = dst = 0; src < nr_heads; src++) {
602 /* If heads[src] is different from any of
603 * heads[0..dst], push it in.
605 int i;
606 for (i = 0; i < dst; i++) {
607 if (!strcmp(heads[i], heads[src]))
608 break;
610 if (i < dst)
611 continue;
612 if (src != dst)
613 heads[dst] = heads[src];
614 dst++;
616 return dst;
619 static int fetch_pack_config(const char *var, const char *value)
621 if (strcmp(var, "fetch.unpacklimit") == 0) {
622 fetch_unpack_limit = git_config_int(var, value);
623 return 0;
626 if (strcmp(var, "transfer.unpacklimit") == 0) {
627 transfer_unpack_limit = git_config_int(var, value);
628 return 0;
631 return git_default_config(var, value);
634 static struct lock_file lock;
636 static void fetch_pack_setup(void)
638 static int did_setup;
639 if (did_setup)
640 return;
641 git_config(fetch_pack_config);
642 if (0 <= transfer_unpack_limit)
643 unpack_limit = transfer_unpack_limit;
644 else if (0 <= fetch_unpack_limit)
645 unpack_limit = fetch_unpack_limit;
646 did_setup = 1;
649 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
651 int i, ret, nr_heads;
652 struct ref *ref;
653 char *dest = NULL, **heads;
655 nr_heads = 0;
656 heads = NULL;
657 for (i = 1; i < argc; i++) {
658 const char *arg = argv[i];
660 if (*arg == '-') {
661 if (!prefixcmp(arg, "--upload-pack=")) {
662 args.uploadpack = arg + 14;
663 continue;
665 if (!prefixcmp(arg, "--exec=")) {
666 args.uploadpack = arg + 7;
667 continue;
669 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
670 args.quiet = 1;
671 continue;
673 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
674 args.lock_pack = args.keep_pack;
675 args.keep_pack = 1;
676 continue;
678 if (!strcmp("--thin", arg)) {
679 args.use_thin_pack = 1;
680 continue;
682 if (!strcmp("--all", arg)) {
683 args.fetch_all = 1;
684 continue;
686 if (!strcmp("-v", arg)) {
687 args.verbose = 1;
688 continue;
690 if (!prefixcmp(arg, "--depth=")) {
691 args.depth = strtol(arg + 8, NULL, 0);
692 continue;
694 if (!strcmp("--no-progress", arg)) {
695 args.no_progress = 1;
696 continue;
698 usage(fetch_pack_usage);
700 dest = (char *)arg;
701 heads = (char **)(argv + i + 1);
702 nr_heads = argc - i - 1;
703 break;
705 if (!dest)
706 usage(fetch_pack_usage);
708 ref = fetch_pack(&args, dest, nr_heads, heads, NULL);
709 ret = !ref;
711 while (ref) {
712 printf("%s %s\n",
713 sha1_to_hex(ref->old_sha1), ref->name);
714 ref = ref->next;
717 return ret;
720 struct ref *fetch_pack(struct fetch_pack_args *my_args,
721 const char *dest,
722 int nr_heads,
723 char **heads,
724 char **pack_lockfile)
726 int i, ret;
727 int fd[2];
728 struct child_process *conn;
729 struct ref *ref;
730 struct stat st;
732 fetch_pack_setup();
733 memcpy(&args, my_args, sizeof(args));
734 if (args.depth > 0) {
735 if (stat(git_path("shallow"), &st))
736 st.st_mtime = 0;
739 conn = git_connect(fd, (char *)dest, args.uploadpack,
740 args.verbose ? CONNECT_VERBOSE : 0);
741 if (heads && nr_heads)
742 nr_heads = remove_duplicates(nr_heads, heads);
743 ref = do_fetch_pack(fd, nr_heads, heads, pack_lockfile);
744 close(fd[0]);
745 close(fd[1]);
746 ret = finish_connect(conn);
748 if (!ret && nr_heads) {
749 /* If the heads to pull were given, we should have
750 * consumed all of them by matching the remote.
751 * Otherwise, 'git-fetch remote no-such-ref' would
752 * silently succeed without issuing an error.
754 for (i = 0; i < nr_heads; i++)
755 if (heads[i] && heads[i][0]) {
756 error("no such remote ref %s", heads[i]);
757 ret = 1;
761 if (!ret && args.depth > 0) {
762 struct cache_time mtime;
763 char *shallow = git_path("shallow");
764 int fd;
766 mtime.sec = st.st_mtime;
767 #ifdef USE_NSEC
768 mtime.usec = st.st_mtim.usec;
769 #endif
770 if (stat(shallow, &st)) {
771 if (mtime.sec)
772 die("shallow file was removed during fetch");
773 } else if (st.st_mtime != mtime.sec
774 #ifdef USE_NSEC
775 || st.st_mtim.usec != mtime.usec
776 #endif
778 die("shallow file was changed during fetch");
780 fd = hold_lock_file_for_update(&lock, shallow, 1);
781 if (!write_shallow_commits(fd, 0)) {
782 unlink(shallow);
783 rollback_lock_file(&lock);
784 } else {
785 commit_lock_file(&lock);
789 if (ret)
790 ref = NULL;
792 return ref;