Allow fetch-pack to decide keeping the fetched pack without exploding
[git/gitweb.git] / fetch-pack.c
blobdd67e48fc7cb945d4f1e11f30abca89d16adfc9d
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
10 static int keep_pack;
11 static int keep_auto;
12 static int quiet;
13 static int verbose;
14 static int fetch_all;
15 static int depth;
16 static const char fetch_pack_usage[] =
17 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [-v] [<host>:]<directory> [<refs>...]";
18 static const char *uploadpack = "git-upload-pack";
20 #define COMPLETE (1U << 0)
21 #define COMMON (1U << 1)
22 #define COMMON_REF (1U << 2)
23 #define SEEN (1U << 3)
24 #define POPPED (1U << 4)
27 * After sending this many "have"s if we do not get any new ACK , we
28 * give up traversing our history.
30 #define MAX_IN_VAIN 256
32 static struct commit_list *rev_list;
33 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
35 static void rev_list_push(struct commit *commit, int mark)
37 if (!(commit->object.flags & mark)) {
38 commit->object.flags |= mark;
40 if (!(commit->object.parsed))
41 parse_commit(commit);
43 insert_by_date(commit, &rev_list);
45 if (!(commit->object.flags & COMMON))
46 non_common_revs++;
50 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
52 struct object *o = deref_tag(parse_object(sha1), path, 0);
54 if (o && o->type == OBJ_COMMIT)
55 rev_list_push((struct commit *)o, SEEN);
57 return 0;
61 This function marks a rev and its ancestors as common.
62 In some cases, it is desirable to mark only the ancestors (for example
63 when only the server does not yet know that they are common).
66 static void mark_common(struct commit *commit,
67 int ancestors_only, int dont_parse)
69 if (commit != NULL && !(commit->object.flags & COMMON)) {
70 struct object *o = (struct object *)commit;
72 if (!ancestors_only)
73 o->flags |= COMMON;
75 if (!(o->flags & SEEN))
76 rev_list_push(commit, SEEN);
77 else {
78 struct commit_list *parents;
80 if (!ancestors_only && !(o->flags & POPPED))
81 non_common_revs--;
82 if (!o->parsed && !dont_parse)
83 parse_commit(commit);
85 for (parents = commit->parents;
86 parents;
87 parents = parents->next)
88 mark_common(parents->item, 0, dont_parse);
94 Get the next rev to send, ignoring the common.
97 static const unsigned char* get_rev(void)
99 struct commit *commit = NULL;
101 while (commit == NULL) {
102 unsigned int mark;
103 struct commit_list* parents;
105 if (rev_list == NULL || non_common_revs == 0)
106 return NULL;
108 commit = rev_list->item;
109 if (!(commit->object.parsed))
110 parse_commit(commit);
111 commit->object.flags |= POPPED;
112 if (!(commit->object.flags & COMMON))
113 non_common_revs--;
115 parents = commit->parents;
117 if (commit->object.flags & COMMON) {
118 /* do not send "have", and ignore ancestors */
119 commit = NULL;
120 mark = COMMON | SEEN;
121 } else if (commit->object.flags & COMMON_REF)
122 /* send "have", and ignore ancestors */
123 mark = COMMON | SEEN;
124 else
125 /* send "have", also for its ancestors */
126 mark = SEEN;
128 while (parents) {
129 if (!(parents->item->object.flags & SEEN))
130 rev_list_push(parents->item, mark);
131 if (mark & COMMON)
132 mark_common(parents->item, 1, 0);
133 parents = parents->next;
136 rev_list = rev_list->next;
139 return commit->object.sha1;
142 static int find_common(int fd[2], unsigned char *result_sha1,
143 struct ref *refs)
145 int fetching;
146 int count = 0, flushes = 0, retval;
147 const unsigned char *sha1;
148 unsigned in_vain = 0;
149 int got_continue = 0;
151 for_each_ref(rev_list_insert_ref, NULL);
153 fetching = 0;
154 for ( ; refs ; refs = refs->next) {
155 unsigned char *remote = refs->old_sha1;
156 struct object *o;
159 * If that object is complete (i.e. it is an ancestor of a
160 * local ref), we tell them we have it but do not have to
161 * tell them about its ancestors, which they already know
162 * about.
164 * We use lookup_object here because we are only
165 * interested in the case we *know* the object is
166 * reachable and we have already scanned it.
168 if (((o = lookup_object(remote)) != NULL) &&
169 (o->flags & COMPLETE)) {
170 continue;
173 if (!fetching)
174 packet_write(fd[1], "want %s%s%s%s%s%s\n",
175 sha1_to_hex(remote),
176 (multi_ack ? " multi_ack" : ""),
177 (use_sideband == 2 ? " side-band-64k" : ""),
178 (use_sideband == 1 ? " side-band" : ""),
179 (use_thin_pack ? " thin-pack" : ""),
180 " ofs-delta");
181 else
182 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
183 fetching++;
185 if (is_repository_shallow())
186 write_shallow_commits(fd[1], 1);
187 if (depth > 0)
188 packet_write(fd[1], "deepen %d", depth);
189 packet_flush(fd[1]);
190 if (!fetching)
191 return 1;
193 if (depth > 0) {
194 char line[1024];
195 unsigned char sha1[20];
196 int len;
198 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
199 if (!strncmp("shallow ", line, 8)) {
200 if (get_sha1_hex(line + 8, sha1))
201 die("invalid shallow line: %s", line);
202 register_shallow(sha1);
203 continue;
205 if (!strncmp("unshallow ", line, 10)) {
206 if (get_sha1_hex(line + 10, sha1))
207 die("invalid unshallow line: %s", line);
208 if (!lookup_object(sha1))
209 die("object not found: %s", line);
210 /* make sure that it is parsed as shallow */
211 parse_object(sha1);
212 if (unregister_shallow(sha1))
213 die("no shallow found: %s", line);
214 continue;
216 die("expected shallow/unshallow, got %s", line);
220 flushes = 0;
221 retval = -1;
222 while ((sha1 = get_rev())) {
223 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
224 if (verbose)
225 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
226 in_vain++;
227 if (!(31 & ++count)) {
228 int ack;
230 packet_flush(fd[1]);
231 flushes++;
234 * We keep one window "ahead" of the other side, and
235 * will wait for an ACK only on the next one
237 if (count == 32)
238 continue;
240 do {
241 ack = get_ack(fd[0], result_sha1);
242 if (verbose && ack)
243 fprintf(stderr, "got ack %d %s\n", ack,
244 sha1_to_hex(result_sha1));
245 if (ack == 1) {
246 flushes = 0;
247 multi_ack = 0;
248 retval = 0;
249 goto done;
250 } else if (ack == 2) {
251 struct commit *commit =
252 lookup_commit(result_sha1);
253 mark_common(commit, 0, 1);
254 retval = 0;
255 in_vain = 0;
256 got_continue = 1;
258 } while (ack);
259 flushes--;
260 if (got_continue && MAX_IN_VAIN < in_vain) {
261 if (verbose)
262 fprintf(stderr, "giving up\n");
263 break; /* give up */
267 done:
268 packet_write(fd[1], "done\n");
269 if (verbose)
270 fprintf(stderr, "done\n");
271 if (retval != 0) {
272 multi_ack = 0;
273 flushes++;
275 while (flushes || multi_ack) {
276 int ack = get_ack(fd[0], result_sha1);
277 if (ack) {
278 if (verbose)
279 fprintf(stderr, "got ack (%d) %s\n", ack,
280 sha1_to_hex(result_sha1));
281 if (ack == 1)
282 return 0;
283 multi_ack = 1;
284 continue;
286 flushes--;
288 return retval;
291 static struct commit_list *complete;
293 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
295 struct object *o = parse_object(sha1);
297 while (o && o->type == OBJ_TAG) {
298 struct tag *t = (struct tag *) o;
299 if (!t->tagged)
300 break; /* broken repository */
301 o->flags |= COMPLETE;
302 o = parse_object(t->tagged->sha1);
304 if (o && o->type == OBJ_COMMIT) {
305 struct commit *commit = (struct commit *)o;
306 commit->object.flags |= COMPLETE;
307 insert_by_date(commit, &complete);
309 return 0;
312 static void mark_recent_complete_commits(unsigned long cutoff)
314 while (complete && cutoff <= complete->item->date) {
315 if (verbose)
316 fprintf(stderr, "Marking %s as complete\n",
317 sha1_to_hex(complete->item->object.sha1));
318 pop_most_recent_commit(&complete, COMPLETE);
322 static void filter_refs(struct ref **refs, int nr_match, char **match)
324 struct ref **return_refs;
325 struct ref *newlist = NULL;
326 struct ref **newtail = &newlist;
327 struct ref *ref, *next;
328 struct ref *fastarray[32];
330 if (nr_match && !fetch_all) {
331 if (ARRAY_SIZE(fastarray) < nr_match)
332 return_refs = xcalloc(nr_match, sizeof(struct ref *));
333 else {
334 return_refs = fastarray;
335 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
338 else
339 return_refs = NULL;
341 for (ref = *refs; ref; ref = next) {
342 next = ref->next;
343 if (!memcmp(ref->name, "refs/", 5) &&
344 check_ref_format(ref->name + 5))
345 ; /* trash */
346 else if (fetch_all &&
347 (!depth || strncmp(ref->name, "refs/tags/", 10) )) {
348 *newtail = ref;
349 ref->next = NULL;
350 newtail = &ref->next;
351 continue;
353 else {
354 int order = path_match(ref->name, nr_match, match);
355 if (order) {
356 return_refs[order-1] = ref;
357 continue; /* we will link it later */
360 free(ref);
363 if (!fetch_all) {
364 int i;
365 for (i = 0; i < nr_match; i++) {
366 ref = return_refs[i];
367 if (ref) {
368 *newtail = ref;
369 ref->next = NULL;
370 newtail = &ref->next;
373 if (return_refs != fastarray)
374 free(return_refs);
376 *refs = newlist;
379 static int everything_local(struct ref **refs, int nr_match, char **match)
381 struct ref *ref;
382 int retval;
383 unsigned long cutoff = 0;
385 track_object_refs = 0;
386 save_commit_buffer = 0;
388 for (ref = *refs; ref; ref = ref->next) {
389 struct object *o;
391 o = parse_object(ref->old_sha1);
392 if (!o)
393 continue;
395 /* We already have it -- which may mean that we were
396 * in sync with the other side at some time after
397 * that (it is OK if we guess wrong here).
399 if (o->type == OBJ_COMMIT) {
400 struct commit *commit = (struct commit *)o;
401 if (!cutoff || cutoff < commit->date)
402 cutoff = commit->date;
406 if (!depth) {
407 for_each_ref(mark_complete, NULL);
408 if (cutoff)
409 mark_recent_complete_commits(cutoff);
413 * Mark all complete remote refs as common refs.
414 * Don't mark them common yet; the server has to be told so first.
416 for (ref = *refs; ref; ref = ref->next) {
417 struct object *o = deref_tag(lookup_object(ref->old_sha1),
418 NULL, 0);
420 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
421 continue;
423 if (!(o->flags & SEEN)) {
424 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
426 mark_common((struct commit *)o, 1, 1);
430 filter_refs(refs, nr_match, match);
432 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
433 const unsigned char *remote = ref->old_sha1;
434 unsigned char local[20];
435 struct object *o;
437 o = lookup_object(remote);
438 if (!o || !(o->flags & COMPLETE)) {
439 retval = 0;
440 if (!verbose)
441 continue;
442 fprintf(stderr,
443 "want %s (%s)\n", sha1_to_hex(remote),
444 ref->name);
445 continue;
448 hashcpy(ref->new_sha1, local);
449 if (!verbose)
450 continue;
451 fprintf(stderr,
452 "already have %s (%s)\n", sha1_to_hex(remote),
453 ref->name);
455 return retval;
458 static pid_t setup_sideband(int fd[2], int xd[2])
460 pid_t side_pid;
462 if (!use_sideband) {
463 fd[0] = xd[0];
464 fd[1] = xd[1];
465 return 0;
467 /* xd[] is talking with upload-pack; subprocess reads from
468 * xd[0], spits out band#2 to stderr, and feeds us band#1
469 * through our fd[0].
471 if (pipe(fd) < 0)
472 die("fetch-pack: unable to set up pipe");
473 side_pid = fork();
474 if (side_pid < 0)
475 die("fetch-pack: unable to fork off sideband demultiplexer");
476 if (!side_pid) {
477 /* subprocess */
478 close(fd[0]);
479 if (xd[0] != xd[1])
480 close(xd[1]);
481 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
482 exit(1);
483 exit(0);
485 close(xd[0]);
486 close(fd[1]);
487 fd[1] = xd[1];
488 return side_pid;
491 static int get_pack(int xd[2])
493 int status;
494 pid_t pid, side_pid;
495 int fd[2];
496 const char *argv[20];
497 char keep_arg[256];
498 char hdr_arg[256];
499 const char **av;
500 int do_keep = keep_pack;
502 side_pid = setup_sideband(fd, xd);
504 av = argv;
505 *hdr_arg = 0;
506 if (keep_auto) {
507 struct pack_header header;
509 if (read_pack_header(fd[0], &header))
510 die("protocol error: bad pack header");
511 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
512 ntohl(header.hdr_version), ntohl(header.hdr_entries));
513 if (ntohl(header.hdr_entries) < keep_auto)
514 do_keep = 0;
515 else
516 do_keep = 1;
519 if (do_keep) {
520 *av++ = "index-pack";
521 *av++ = "--stdin";
522 if (!quiet)
523 *av++ = "-v";
524 if (use_thin_pack)
525 *av++ = "--fix-thin";
526 if (keep_pack > 1 || keep_auto) {
527 int s = sprintf(keep_arg,
528 "--keep=fetch-pack %d on ", getpid());
529 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
530 strcpy(keep_arg + s, "localhost");
531 *av++ = keep_arg;
534 else {
535 *av++ = "unpack-objects";
536 if (quiet)
537 *av++ = "-q";
539 if (*hdr_arg)
540 *av++ = hdr_arg;
541 *av++ = NULL;
543 pid = fork();
544 if (pid < 0)
545 die("fetch-pack: unable to fork off %s", argv[0]);
546 if (!pid) {
547 dup2(fd[0], 0);
548 close(fd[0]);
549 close(fd[1]);
550 execv_git_cmd(argv);
551 die("%s exec failed", argv[0]);
553 close(fd[0]);
554 close(fd[1]);
555 while (waitpid(pid, &status, 0) < 0) {
556 if (errno != EINTR)
557 die("waiting for %s: %s", argv[0], strerror(errno));
559 if (WIFEXITED(status)) {
560 int code = WEXITSTATUS(status);
561 if (code)
562 die("%s died with error code %d", argv[0], code);
563 return 0;
565 if (WIFSIGNALED(status)) {
566 int sig = WTERMSIG(status);
567 die("%s died of signal %d", argv[0], sig);
569 die("%s died of unnatural causes %d", argv[0], status);
572 static int fetch_pack(int fd[2], int nr_match, char **match)
574 struct ref *ref;
575 unsigned char sha1[20];
577 get_remote_heads(fd[0], &ref, 0, NULL, 0);
578 if (is_repository_shallow() && !server_supports("shallow"))
579 die("Server does not support shallow clients");
580 if (server_supports("multi_ack")) {
581 if (verbose)
582 fprintf(stderr, "Server supports multi_ack\n");
583 multi_ack = 1;
585 if (server_supports("side-band-64k")) {
586 if (verbose)
587 fprintf(stderr, "Server supports side-band-64k\n");
588 use_sideband = 2;
590 else if (server_supports("side-band")) {
591 if (verbose)
592 fprintf(stderr, "Server supports side-band\n");
593 use_sideband = 1;
595 if (!ref) {
596 packet_flush(fd[1]);
597 die("no matching remote head");
599 if (everything_local(&ref, nr_match, match)) {
600 packet_flush(fd[1]);
601 goto all_done;
603 if (find_common(fd, sha1, ref) < 0)
604 if (keep_pack != 1)
605 /* When cloning, it is not unusual to have
606 * no common commit.
608 fprintf(stderr, "warning: no common commits\n");
610 if (get_pack(fd))
611 die("git-fetch-pack: fetch failed.");
613 all_done:
614 while (ref) {
615 printf("%s %s\n",
616 sha1_to_hex(ref->old_sha1), ref->name);
617 ref = ref->next;
619 return 0;
622 static int remove_duplicates(int nr_heads, char **heads)
624 int src, dst;
626 for (src = dst = 0; src < nr_heads; src++) {
627 /* If heads[src] is different from any of
628 * heads[0..dst], push it in.
630 int i;
631 for (i = 0; i < dst; i++) {
632 if (!strcmp(heads[i], heads[src]))
633 break;
635 if (i < dst)
636 continue;
637 if (src != dst)
638 heads[dst] = heads[src];
639 dst++;
641 heads[dst] = 0;
642 return dst;
645 static struct lock_file lock;
647 int main(int argc, char **argv)
649 int i, ret, nr_heads;
650 char *dest = NULL, **heads;
651 int fd[2];
652 pid_t pid;
653 struct stat st;
655 setup_git_directory();
657 nr_heads = 0;
658 heads = NULL;
659 for (i = 1; i < argc; i++) {
660 char *arg = argv[i];
662 if (*arg == '-') {
663 if (!strncmp("--upload-pack=", arg, 14)) {
664 uploadpack = arg + 14;
665 continue;
667 if (!strncmp("--exec=", arg, 7)) {
668 uploadpack = arg + 7;
669 continue;
671 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
672 quiet = 1;
673 continue;
675 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
676 keep_pack++;
677 continue;
679 if (!strcmp("--keep-auto", arg)) {
680 keep_auto = 100;
681 continue;
683 if (!strncmp("--keep-auto=", arg, 12)) {
684 keep_auto = strtoul(arg + 12, NULL, 0);
685 if (keep_auto < 20)
686 keep_auto = 20;
687 continue;
689 if (!strcmp("--thin", arg)) {
690 use_thin_pack = 1;
691 continue;
693 if (!strcmp("--all", arg)) {
694 fetch_all = 1;
695 continue;
697 if (!strcmp("-v", arg)) {
698 verbose = 1;
699 continue;
701 if (!strncmp("--depth=", arg, 8)) {
702 depth = strtol(arg + 8, NULL, 0);
703 if (stat(git_path("shallow"), &st))
704 st.st_mtime = 0;
705 continue;
707 usage(fetch_pack_usage);
709 dest = arg;
710 heads = argv + i + 1;
711 nr_heads = argc - i - 1;
712 break;
714 if (!dest)
715 usage(fetch_pack_usage);
716 pid = git_connect(fd, dest, uploadpack);
717 if (pid < 0)
718 return 1;
719 if (heads && nr_heads)
720 nr_heads = remove_duplicates(nr_heads, heads);
721 ret = fetch_pack(fd, nr_heads, heads);
722 close(fd[0]);
723 close(fd[1]);
724 ret |= finish_connect(pid);
726 if (!ret && nr_heads) {
727 /* If the heads to pull were given, we should have
728 * consumed all of them by matching the remote.
729 * Otherwise, 'git-fetch remote no-such-ref' would
730 * silently succeed without issuing an error.
732 for (i = 0; i < nr_heads; i++)
733 if (heads[i] && heads[i][0]) {
734 error("no such remote ref %s", heads[i]);
735 ret = 1;
739 if (!ret && depth > 0) {
740 struct cache_time mtime;
741 char *shallow = git_path("shallow");
742 int fd;
744 mtime.sec = st.st_mtime;
745 #ifdef USE_NSEC
746 mtime.usec = st.st_mtim.usec;
747 #endif
748 if (stat(shallow, &st)) {
749 if (mtime.sec)
750 die("shallow file was removed during fetch");
751 } else if (st.st_mtime != mtime.sec
752 #ifdef USE_NSEC
753 || st.st_mtim.usec != mtime.usec
754 #endif
756 die("shallow file was changed during fetch");
758 fd = hold_lock_file_for_update(&lock, shallow, 1);
759 if (!write_shallow_commits(fd, 0)) {
760 unlink(shallow);
761 rollback_lock_file(&lock);
762 } else {
763 close(fd);
764 commit_lock_file(&lock);
768 return !!ret;