fetch-pack: remove --keep-auto and make it the default.
[git/jnareb-git.git] / fetch-pack.c
blobfc0534ce03a69f45671d009772865cbfcad3b026
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
10 static int keep_pack;
11 static int unpack_limit = 100;
12 static int quiet;
13 static int verbose;
14 static int fetch_all;
15 static int depth;
16 static const char fetch_pack_usage[] =
17 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [-v] [<host>:]<directory> [<refs>...]";
18 static const char *uploadpack = "git-upload-pack";
20 #define COMPLETE (1U << 0)
21 #define COMMON (1U << 1)
22 #define COMMON_REF (1U << 2)
23 #define SEEN (1U << 3)
24 #define POPPED (1U << 4)
27 * After sending this many "have"s if we do not get any new ACK , we
28 * give up traversing our history.
30 #define MAX_IN_VAIN 256
32 static struct commit_list *rev_list;
33 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
35 static void rev_list_push(struct commit *commit, int mark)
37 if (!(commit->object.flags & mark)) {
38 commit->object.flags |= mark;
40 if (!(commit->object.parsed))
41 parse_commit(commit);
43 insert_by_date(commit, &rev_list);
45 if (!(commit->object.flags & COMMON))
46 non_common_revs++;
50 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
52 struct object *o = deref_tag(parse_object(sha1), path, 0);
54 if (o && o->type == OBJ_COMMIT)
55 rev_list_push((struct commit *)o, SEEN);
57 return 0;
61 This function marks a rev and its ancestors as common.
62 In some cases, it is desirable to mark only the ancestors (for example
63 when only the server does not yet know that they are common).
66 static void mark_common(struct commit *commit,
67 int ancestors_only, int dont_parse)
69 if (commit != NULL && !(commit->object.flags & COMMON)) {
70 struct object *o = (struct object *)commit;
72 if (!ancestors_only)
73 o->flags |= COMMON;
75 if (!(o->flags & SEEN))
76 rev_list_push(commit, SEEN);
77 else {
78 struct commit_list *parents;
80 if (!ancestors_only && !(o->flags & POPPED))
81 non_common_revs--;
82 if (!o->parsed && !dont_parse)
83 parse_commit(commit);
85 for (parents = commit->parents;
86 parents;
87 parents = parents->next)
88 mark_common(parents->item, 0, dont_parse);
94 Get the next rev to send, ignoring the common.
97 static const unsigned char* get_rev(void)
99 struct commit *commit = NULL;
101 while (commit == NULL) {
102 unsigned int mark;
103 struct commit_list* parents;
105 if (rev_list == NULL || non_common_revs == 0)
106 return NULL;
108 commit = rev_list->item;
109 if (!(commit->object.parsed))
110 parse_commit(commit);
111 commit->object.flags |= POPPED;
112 if (!(commit->object.flags & COMMON))
113 non_common_revs--;
115 parents = commit->parents;
117 if (commit->object.flags & COMMON) {
118 /* do not send "have", and ignore ancestors */
119 commit = NULL;
120 mark = COMMON | SEEN;
121 } else if (commit->object.flags & COMMON_REF)
122 /* send "have", and ignore ancestors */
123 mark = COMMON | SEEN;
124 else
125 /* send "have", also for its ancestors */
126 mark = SEEN;
128 while (parents) {
129 if (!(parents->item->object.flags & SEEN))
130 rev_list_push(parents->item, mark);
131 if (mark & COMMON)
132 mark_common(parents->item, 1, 0);
133 parents = parents->next;
136 rev_list = rev_list->next;
139 return commit->object.sha1;
142 static int find_common(int fd[2], unsigned char *result_sha1,
143 struct ref *refs)
145 int fetching;
146 int count = 0, flushes = 0, retval;
147 const unsigned char *sha1;
148 unsigned in_vain = 0;
149 int got_continue = 0;
151 for_each_ref(rev_list_insert_ref, NULL);
153 fetching = 0;
154 for ( ; refs ; refs = refs->next) {
155 unsigned char *remote = refs->old_sha1;
156 struct object *o;
159 * If that object is complete (i.e. it is an ancestor of a
160 * local ref), we tell them we have it but do not have to
161 * tell them about its ancestors, which they already know
162 * about.
164 * We use lookup_object here because we are only
165 * interested in the case we *know* the object is
166 * reachable and we have already scanned it.
168 if (((o = lookup_object(remote)) != NULL) &&
169 (o->flags & COMPLETE)) {
170 continue;
173 if (!fetching)
174 packet_write(fd[1], "want %s%s%s%s%s%s\n",
175 sha1_to_hex(remote),
176 (multi_ack ? " multi_ack" : ""),
177 (use_sideband == 2 ? " side-band-64k" : ""),
178 (use_sideband == 1 ? " side-band" : ""),
179 (use_thin_pack ? " thin-pack" : ""),
180 " ofs-delta");
181 else
182 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
183 fetching++;
185 if (is_repository_shallow())
186 write_shallow_commits(fd[1], 1);
187 if (depth > 0)
188 packet_write(fd[1], "deepen %d", depth);
189 packet_flush(fd[1]);
190 if (!fetching)
191 return 1;
193 if (depth > 0) {
194 char line[1024];
195 unsigned char sha1[20];
196 int len;
198 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
199 if (!strncmp("shallow ", line, 8)) {
200 if (get_sha1_hex(line + 8, sha1))
201 die("invalid shallow line: %s", line);
202 register_shallow(sha1);
203 continue;
205 if (!strncmp("unshallow ", line, 10)) {
206 if (get_sha1_hex(line + 10, sha1))
207 die("invalid unshallow line: %s", line);
208 if (!lookup_object(sha1))
209 die("object not found: %s", line);
210 /* make sure that it is parsed as shallow */
211 parse_object(sha1);
212 if (unregister_shallow(sha1))
213 die("no shallow found: %s", line);
214 continue;
216 die("expected shallow/unshallow, got %s", line);
220 flushes = 0;
221 retval = -1;
222 while ((sha1 = get_rev())) {
223 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
224 if (verbose)
225 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
226 in_vain++;
227 if (!(31 & ++count)) {
228 int ack;
230 packet_flush(fd[1]);
231 flushes++;
234 * We keep one window "ahead" of the other side, and
235 * will wait for an ACK only on the next one
237 if (count == 32)
238 continue;
240 do {
241 ack = get_ack(fd[0], result_sha1);
242 if (verbose && ack)
243 fprintf(stderr, "got ack %d %s\n", ack,
244 sha1_to_hex(result_sha1));
245 if (ack == 1) {
246 flushes = 0;
247 multi_ack = 0;
248 retval = 0;
249 goto done;
250 } else if (ack == 2) {
251 struct commit *commit =
252 lookup_commit(result_sha1);
253 mark_common(commit, 0, 1);
254 retval = 0;
255 in_vain = 0;
256 got_continue = 1;
258 } while (ack);
259 flushes--;
260 if (got_continue && MAX_IN_VAIN < in_vain) {
261 if (verbose)
262 fprintf(stderr, "giving up\n");
263 break; /* give up */
267 done:
268 packet_write(fd[1], "done\n");
269 if (verbose)
270 fprintf(stderr, "done\n");
271 if (retval != 0) {
272 multi_ack = 0;
273 flushes++;
275 while (flushes || multi_ack) {
276 int ack = get_ack(fd[0], result_sha1);
277 if (ack) {
278 if (verbose)
279 fprintf(stderr, "got ack (%d) %s\n", ack,
280 sha1_to_hex(result_sha1));
281 if (ack == 1)
282 return 0;
283 multi_ack = 1;
284 continue;
286 flushes--;
288 return retval;
291 static struct commit_list *complete;
293 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
295 struct object *o = parse_object(sha1);
297 while (o && o->type == OBJ_TAG) {
298 struct tag *t = (struct tag *) o;
299 if (!t->tagged)
300 break; /* broken repository */
301 o->flags |= COMPLETE;
302 o = parse_object(t->tagged->sha1);
304 if (o && o->type == OBJ_COMMIT) {
305 struct commit *commit = (struct commit *)o;
306 commit->object.flags |= COMPLETE;
307 insert_by_date(commit, &complete);
309 return 0;
312 static void mark_recent_complete_commits(unsigned long cutoff)
314 while (complete && cutoff <= complete->item->date) {
315 if (verbose)
316 fprintf(stderr, "Marking %s as complete\n",
317 sha1_to_hex(complete->item->object.sha1));
318 pop_most_recent_commit(&complete, COMPLETE);
322 static void filter_refs(struct ref **refs, int nr_match, char **match)
324 struct ref **return_refs;
325 struct ref *newlist = NULL;
326 struct ref **newtail = &newlist;
327 struct ref *ref, *next;
328 struct ref *fastarray[32];
330 if (nr_match && !fetch_all) {
331 if (ARRAY_SIZE(fastarray) < nr_match)
332 return_refs = xcalloc(nr_match, sizeof(struct ref *));
333 else {
334 return_refs = fastarray;
335 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
338 else
339 return_refs = NULL;
341 for (ref = *refs; ref; ref = next) {
342 next = ref->next;
343 if (!memcmp(ref->name, "refs/", 5) &&
344 check_ref_format(ref->name + 5))
345 ; /* trash */
346 else if (fetch_all &&
347 (!depth || strncmp(ref->name, "refs/tags/", 10) )) {
348 *newtail = ref;
349 ref->next = NULL;
350 newtail = &ref->next;
351 continue;
353 else {
354 int order = path_match(ref->name, nr_match, match);
355 if (order) {
356 return_refs[order-1] = ref;
357 continue; /* we will link it later */
360 free(ref);
363 if (!fetch_all) {
364 int i;
365 for (i = 0; i < nr_match; i++) {
366 ref = return_refs[i];
367 if (ref) {
368 *newtail = ref;
369 ref->next = NULL;
370 newtail = &ref->next;
373 if (return_refs != fastarray)
374 free(return_refs);
376 *refs = newlist;
379 static int everything_local(struct ref **refs, int nr_match, char **match)
381 struct ref *ref;
382 int retval;
383 unsigned long cutoff = 0;
385 track_object_refs = 0;
386 save_commit_buffer = 0;
388 for (ref = *refs; ref; ref = ref->next) {
389 struct object *o;
391 o = parse_object(ref->old_sha1);
392 if (!o)
393 continue;
395 /* We already have it -- which may mean that we were
396 * in sync with the other side at some time after
397 * that (it is OK if we guess wrong here).
399 if (o->type == OBJ_COMMIT) {
400 struct commit *commit = (struct commit *)o;
401 if (!cutoff || cutoff < commit->date)
402 cutoff = commit->date;
406 if (!depth) {
407 for_each_ref(mark_complete, NULL);
408 if (cutoff)
409 mark_recent_complete_commits(cutoff);
413 * Mark all complete remote refs as common refs.
414 * Don't mark them common yet; the server has to be told so first.
416 for (ref = *refs; ref; ref = ref->next) {
417 struct object *o = deref_tag(lookup_object(ref->old_sha1),
418 NULL, 0);
420 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
421 continue;
423 if (!(o->flags & SEEN)) {
424 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
426 mark_common((struct commit *)o, 1, 1);
430 filter_refs(refs, nr_match, match);
432 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
433 const unsigned char *remote = ref->old_sha1;
434 unsigned char local[20];
435 struct object *o;
437 o = lookup_object(remote);
438 if (!o || !(o->flags & COMPLETE)) {
439 retval = 0;
440 if (!verbose)
441 continue;
442 fprintf(stderr,
443 "want %s (%s)\n", sha1_to_hex(remote),
444 ref->name);
445 continue;
448 hashcpy(ref->new_sha1, local);
449 if (!verbose)
450 continue;
451 fprintf(stderr,
452 "already have %s (%s)\n", sha1_to_hex(remote),
453 ref->name);
455 return retval;
458 static pid_t setup_sideband(int fd[2], int xd[2])
460 pid_t side_pid;
462 if (!use_sideband) {
463 fd[0] = xd[0];
464 fd[1] = xd[1];
465 return 0;
467 /* xd[] is talking with upload-pack; subprocess reads from
468 * xd[0], spits out band#2 to stderr, and feeds us band#1
469 * through our fd[0].
471 if (pipe(fd) < 0)
472 die("fetch-pack: unable to set up pipe");
473 side_pid = fork();
474 if (side_pid < 0)
475 die("fetch-pack: unable to fork off sideband demultiplexer");
476 if (!side_pid) {
477 /* subprocess */
478 close(fd[0]);
479 if (xd[0] != xd[1])
480 close(xd[1]);
481 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
482 exit(1);
483 exit(0);
485 close(xd[0]);
486 close(fd[1]);
487 fd[1] = xd[1];
488 return side_pid;
491 static int get_pack(int xd[2])
493 int status;
494 pid_t pid, side_pid;
495 int fd[2];
496 const char *argv[20];
497 char keep_arg[256];
498 char hdr_arg[256];
499 const char **av;
500 int do_keep = keep_pack;
502 side_pid = setup_sideband(fd, xd);
504 av = argv;
505 *hdr_arg = 0;
506 if (unpack_limit) {
507 struct pack_header header;
509 if (read_pack_header(fd[0], &header))
510 die("protocol error: bad pack header");
511 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
512 ntohl(header.hdr_version), ntohl(header.hdr_entries));
513 if (ntohl(header.hdr_entries) < unpack_limit)
514 do_keep = 0;
515 else
516 do_keep = 1;
519 if (do_keep) {
520 *av++ = "index-pack";
521 *av++ = "--stdin";
522 if (!quiet)
523 *av++ = "-v";
524 if (use_thin_pack)
525 *av++ = "--fix-thin";
526 if (keep_pack > 1 || unpack_limit) {
527 int s = sprintf(keep_arg,
528 "--keep=fetch-pack %d on ", getpid());
529 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
530 strcpy(keep_arg + s, "localhost");
531 *av++ = keep_arg;
534 else {
535 *av++ = "unpack-objects";
536 if (quiet)
537 *av++ = "-q";
539 if (*hdr_arg)
540 *av++ = hdr_arg;
541 *av++ = NULL;
543 pid = fork();
544 if (pid < 0)
545 die("fetch-pack: unable to fork off %s", argv[0]);
546 if (!pid) {
547 dup2(fd[0], 0);
548 close(fd[0]);
549 close(fd[1]);
550 execv_git_cmd(argv);
551 die("%s exec failed", argv[0]);
553 close(fd[0]);
554 close(fd[1]);
555 while (waitpid(pid, &status, 0) < 0) {
556 if (errno != EINTR)
557 die("waiting for %s: %s", argv[0], strerror(errno));
559 if (WIFEXITED(status)) {
560 int code = WEXITSTATUS(status);
561 if (code)
562 die("%s died with error code %d", argv[0], code);
563 return 0;
565 if (WIFSIGNALED(status)) {
566 int sig = WTERMSIG(status);
567 die("%s died of signal %d", argv[0], sig);
569 die("%s died of unnatural causes %d", argv[0], status);
572 static int fetch_pack(int fd[2], int nr_match, char **match)
574 struct ref *ref;
575 unsigned char sha1[20];
577 get_remote_heads(fd[0], &ref, 0, NULL, 0);
578 if (is_repository_shallow() && !server_supports("shallow"))
579 die("Server does not support shallow clients");
580 if (server_supports("multi_ack")) {
581 if (verbose)
582 fprintf(stderr, "Server supports multi_ack\n");
583 multi_ack = 1;
585 if (server_supports("side-band-64k")) {
586 if (verbose)
587 fprintf(stderr, "Server supports side-band-64k\n");
588 use_sideband = 2;
590 else if (server_supports("side-band")) {
591 if (verbose)
592 fprintf(stderr, "Server supports side-band\n");
593 use_sideband = 1;
595 if (!ref) {
596 packet_flush(fd[1]);
597 die("no matching remote head");
599 if (everything_local(&ref, nr_match, match)) {
600 packet_flush(fd[1]);
601 goto all_done;
603 if (find_common(fd, sha1, ref) < 0)
604 if (keep_pack != 1)
605 /* When cloning, it is not unusual to have
606 * no common commit.
608 fprintf(stderr, "warning: no common commits\n");
610 if (get_pack(fd))
611 die("git-fetch-pack: fetch failed.");
613 all_done:
614 while (ref) {
615 printf("%s %s\n",
616 sha1_to_hex(ref->old_sha1), ref->name);
617 ref = ref->next;
619 return 0;
622 static int remove_duplicates(int nr_heads, char **heads)
624 int src, dst;
626 for (src = dst = 0; src < nr_heads; src++) {
627 /* If heads[src] is different from any of
628 * heads[0..dst], push it in.
630 int i;
631 for (i = 0; i < dst; i++) {
632 if (!strcmp(heads[i], heads[src]))
633 break;
635 if (i < dst)
636 continue;
637 if (src != dst)
638 heads[dst] = heads[src];
639 dst++;
641 heads[dst] = 0;
642 return dst;
645 static int fetch_pack_config(const char *var, const char *value)
647 if (strcmp(var, "fetch.unpacklimit") == 0) {
648 unpack_limit = git_config_int(var, value);
649 return 0;
652 return git_default_config(var, value);
655 static struct lock_file lock;
657 int main(int argc, char **argv)
659 int i, ret, nr_heads;
660 char *dest = NULL, **heads;
661 int fd[2];
662 pid_t pid;
663 struct stat st;
665 setup_git_directory();
666 setup_ident();
667 git_config(fetch_pack_config);
669 nr_heads = 0;
670 heads = NULL;
671 for (i = 1; i < argc; i++) {
672 char *arg = argv[i];
674 if (*arg == '-') {
675 if (!strncmp("--upload-pack=", arg, 14)) {
676 uploadpack = arg + 14;
677 continue;
679 if (!strncmp("--exec=", arg, 7)) {
680 uploadpack = arg + 7;
681 continue;
683 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
684 quiet = 1;
685 continue;
687 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
688 keep_pack++;
689 unpack_limit = 0;
690 continue;
692 if (!strcmp("--thin", arg)) {
693 use_thin_pack = 1;
694 continue;
696 if (!strcmp("--all", arg)) {
697 fetch_all = 1;
698 continue;
700 if (!strcmp("-v", arg)) {
701 verbose = 1;
702 continue;
704 if (!strncmp("--depth=", arg, 8)) {
705 depth = strtol(arg + 8, NULL, 0);
706 if (stat(git_path("shallow"), &st))
707 st.st_mtime = 0;
708 continue;
710 usage(fetch_pack_usage);
712 dest = arg;
713 heads = argv + i + 1;
714 nr_heads = argc - i - 1;
715 break;
717 if (!dest)
718 usage(fetch_pack_usage);
719 pid = git_connect(fd, dest, uploadpack);
720 if (pid < 0)
721 return 1;
722 if (heads && nr_heads)
723 nr_heads = remove_duplicates(nr_heads, heads);
724 ret = fetch_pack(fd, nr_heads, heads);
725 close(fd[0]);
726 close(fd[1]);
727 ret |= finish_connect(pid);
729 if (!ret && nr_heads) {
730 /* If the heads to pull were given, we should have
731 * consumed all of them by matching the remote.
732 * Otherwise, 'git-fetch remote no-such-ref' would
733 * silently succeed without issuing an error.
735 for (i = 0; i < nr_heads; i++)
736 if (heads[i] && heads[i][0]) {
737 error("no such remote ref %s", heads[i]);
738 ret = 1;
742 if (!ret && depth > 0) {
743 struct cache_time mtime;
744 char *shallow = git_path("shallow");
745 int fd;
747 mtime.sec = st.st_mtime;
748 #ifdef USE_NSEC
749 mtime.usec = st.st_mtim.usec;
750 #endif
751 if (stat(shallow, &st)) {
752 if (mtime.sec)
753 die("shallow file was removed during fetch");
754 } else if (st.st_mtime != mtime.sec
755 #ifdef USE_NSEC
756 || st.st_mtim.usec != mtime.usec
757 #endif
759 die("shallow file was changed during fetch");
761 fd = hold_lock_file_for_update(&lock, shallow, 1);
762 if (!write_shallow_commits(fd, 0)) {
763 unlink(shallow);
764 rollback_lock_file(&lock);
765 } else {
766 close(fd);
767 commit_lock_file(&lock);
771 return !!ret;