Merge branch 'maint'
[git/git-svn.git] / builtin-fetch-pack.c
blob629735f54723a3bc3b24f8c91e9a6226ae28d742
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9 #include "fetch-pack.h"
10 #include "remote.h"
11 #include "run-command.h"
13 static int transfer_unpack_limit = -1;
14 static int fetch_unpack_limit = -1;
15 static int unpack_limit = 100;
16 static int prefer_ofs_delta = 1;
17 static struct fetch_pack_args args = {
18 /* .uploadpack = */ "git-upload-pack",
21 static const char fetch_pack_usage[] =
22 "git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
24 #define COMPLETE (1U << 0)
25 #define COMMON (1U << 1)
26 #define COMMON_REF (1U << 2)
27 #define SEEN (1U << 3)
28 #define POPPED (1U << 4)
30 static int marked;
33 * After sending this many "have"s if we do not get any new ACK , we
34 * give up traversing our history.
36 #define MAX_IN_VAIN 256
38 static struct commit_list *rev_list;
39 static int non_common_revs, multi_ack, use_sideband;
41 static void rev_list_push(struct commit *commit, int mark)
43 if (!(commit->object.flags & mark)) {
44 commit->object.flags |= mark;
46 if (!(commit->object.parsed))
47 if (parse_commit(commit))
48 return;
50 insert_by_date(commit, &rev_list);
52 if (!(commit->object.flags & COMMON))
53 non_common_revs++;
57 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
59 struct object *o = deref_tag(parse_object(sha1), path, 0);
61 if (o && o->type == OBJ_COMMIT)
62 rev_list_push((struct commit *)o, SEEN);
64 return 0;
67 static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data)
69 struct object *o = deref_tag(parse_object(sha1), path, 0);
71 if (o && o->type == OBJ_COMMIT)
72 clear_commit_marks((struct commit *)o,
73 COMMON | COMMON_REF | SEEN | POPPED);
74 return 0;
78 This function marks a rev and its ancestors as common.
79 In some cases, it is desirable to mark only the ancestors (for example
80 when only the server does not yet know that they are common).
83 static void mark_common(struct commit *commit,
84 int ancestors_only, int dont_parse)
86 if (commit != NULL && !(commit->object.flags & COMMON)) {
87 struct object *o = (struct object *)commit;
89 if (!ancestors_only)
90 o->flags |= COMMON;
92 if (!(o->flags & SEEN))
93 rev_list_push(commit, SEEN);
94 else {
95 struct commit_list *parents;
97 if (!ancestors_only && !(o->flags & POPPED))
98 non_common_revs--;
99 if (!o->parsed && !dont_parse)
100 if (parse_commit(commit))
101 return;
103 for (parents = commit->parents;
104 parents;
105 parents = parents->next)
106 mark_common(parents->item, 0, dont_parse);
112 Get the next rev to send, ignoring the common.
115 static const unsigned char *get_rev(void)
117 struct commit *commit = NULL;
119 while (commit == NULL) {
120 unsigned int mark;
121 struct commit_list *parents;
123 if (rev_list == NULL || non_common_revs == 0)
124 return NULL;
126 commit = rev_list->item;
127 if (!commit->object.parsed)
128 parse_commit(commit);
129 parents = commit->parents;
131 commit->object.flags |= POPPED;
132 if (!(commit->object.flags & COMMON))
133 non_common_revs--;
135 if (commit->object.flags & COMMON) {
136 /* do not send "have", and ignore ancestors */
137 commit = NULL;
138 mark = COMMON | SEEN;
139 } else if (commit->object.flags & COMMON_REF)
140 /* send "have", and ignore ancestors */
141 mark = COMMON | SEEN;
142 else
143 /* send "have", also for its ancestors */
144 mark = SEEN;
146 while (parents) {
147 if (!(parents->item->object.flags & SEEN))
148 rev_list_push(parents->item, mark);
149 if (mark & COMMON)
150 mark_common(parents->item, 1, 0);
151 parents = parents->next;
154 rev_list = rev_list->next;
157 return commit->object.sha1;
160 static int find_common(int fd[2], unsigned char *result_sha1,
161 struct ref *refs)
163 int fetching;
164 int count = 0, flushes = 0, retval;
165 const unsigned char *sha1;
166 unsigned in_vain = 0;
167 int got_continue = 0;
169 if (marked)
170 for_each_ref(clear_marks, NULL);
171 marked = 1;
173 for_each_ref(rev_list_insert_ref, NULL);
175 fetching = 0;
176 for ( ; refs ; refs = refs->next) {
177 unsigned char *remote = refs->old_sha1;
178 struct object *o;
181 * If that object is complete (i.e. it is an ancestor of a
182 * local ref), we tell them we have it but do not have to
183 * tell them about its ancestors, which they already know
184 * about.
186 * We use lookup_object here because we are only
187 * interested in the case we *know* the object is
188 * reachable and we have already scanned it.
190 if (((o = lookup_object(remote)) != NULL) &&
191 (o->flags & COMPLETE)) {
192 continue;
195 if (!fetching)
196 packet_write(fd[1], "want %s%s%s%s%s%s%s%s\n",
197 sha1_to_hex(remote),
198 (multi_ack ? " multi_ack" : ""),
199 (use_sideband == 2 ? " side-band-64k" : ""),
200 (use_sideband == 1 ? " side-band" : ""),
201 (args.use_thin_pack ? " thin-pack" : ""),
202 (args.no_progress ? " no-progress" : ""),
203 (args.include_tag ? " include-tag" : ""),
204 (prefer_ofs_delta ? " ofs-delta" : ""));
205 else
206 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
207 fetching++;
209 if (is_repository_shallow())
210 write_shallow_commits(fd[1], 1);
211 if (args.depth > 0)
212 packet_write(fd[1], "deepen %d", args.depth);
213 packet_flush(fd[1]);
214 if (!fetching)
215 return 1;
217 if (args.depth > 0) {
218 char line[1024];
219 unsigned char sha1[20];
221 while (packet_read_line(fd[0], line, sizeof(line))) {
222 if (!prefixcmp(line, "shallow ")) {
223 if (get_sha1_hex(line + 8, sha1))
224 die("invalid shallow line: %s", line);
225 register_shallow(sha1);
226 continue;
228 if (!prefixcmp(line, "unshallow ")) {
229 if (get_sha1_hex(line + 10, sha1))
230 die("invalid unshallow line: %s", line);
231 if (!lookup_object(sha1))
232 die("object not found: %s", line);
233 /* make sure that it is parsed as shallow */
234 if (!parse_object(sha1))
235 die("error in object: %s", line);
236 if (unregister_shallow(sha1))
237 die("no shallow found: %s", line);
238 continue;
240 die("expected shallow/unshallow, got %s", line);
244 flushes = 0;
245 retval = -1;
246 while ((sha1 = get_rev())) {
247 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
248 if (args.verbose)
249 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
250 in_vain++;
251 if (!(31 & ++count)) {
252 int ack;
254 packet_flush(fd[1]);
255 flushes++;
258 * We keep one window "ahead" of the other side, and
259 * will wait for an ACK only on the next one
261 if (count == 32)
262 continue;
264 do {
265 ack = get_ack(fd[0], result_sha1);
266 if (args.verbose && ack)
267 fprintf(stderr, "got ack %d %s\n", ack,
268 sha1_to_hex(result_sha1));
269 if (ack == 1) {
270 flushes = 0;
271 multi_ack = 0;
272 retval = 0;
273 goto done;
274 } else if (ack == 2) {
275 struct commit *commit =
276 lookup_commit(result_sha1);
277 mark_common(commit, 0, 1);
278 retval = 0;
279 in_vain = 0;
280 got_continue = 1;
282 } while (ack);
283 flushes--;
284 if (got_continue && MAX_IN_VAIN < in_vain) {
285 if (args.verbose)
286 fprintf(stderr, "giving up\n");
287 break; /* give up */
291 done:
292 packet_write(fd[1], "done\n");
293 if (args.verbose)
294 fprintf(stderr, "done\n");
295 if (retval != 0) {
296 multi_ack = 0;
297 flushes++;
299 while (flushes || multi_ack) {
300 int ack = get_ack(fd[0], result_sha1);
301 if (ack) {
302 if (args.verbose)
303 fprintf(stderr, "got ack (%d) %s\n", ack,
304 sha1_to_hex(result_sha1));
305 if (ack == 1)
306 return 0;
307 multi_ack = 1;
308 continue;
310 flushes--;
312 /* it is no error to fetch into a completely empty repo */
313 return count ? retval : 0;
316 static struct commit_list *complete;
318 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
320 struct object *o = parse_object(sha1);
322 while (o && o->type == OBJ_TAG) {
323 struct tag *t = (struct tag *) o;
324 if (!t->tagged)
325 break; /* broken repository */
326 o->flags |= COMPLETE;
327 o = parse_object(t->tagged->sha1);
329 if (o && o->type == OBJ_COMMIT) {
330 struct commit *commit = (struct commit *)o;
331 commit->object.flags |= COMPLETE;
332 insert_by_date(commit, &complete);
334 return 0;
337 static void mark_recent_complete_commits(unsigned long cutoff)
339 while (complete && cutoff <= complete->item->date) {
340 if (args.verbose)
341 fprintf(stderr, "Marking %s as complete\n",
342 sha1_to_hex(complete->item->object.sha1));
343 pop_most_recent_commit(&complete, COMPLETE);
347 static void filter_refs(struct ref **refs, int nr_match, char **match)
349 struct ref **return_refs;
350 struct ref *newlist = NULL;
351 struct ref **newtail = &newlist;
352 struct ref *ref, *next;
353 struct ref *fastarray[32];
355 if (nr_match && !args.fetch_all) {
356 if (ARRAY_SIZE(fastarray) < nr_match)
357 return_refs = xcalloc(nr_match, sizeof(struct ref *));
358 else {
359 return_refs = fastarray;
360 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
363 else
364 return_refs = NULL;
366 for (ref = *refs; ref; ref = next) {
367 next = ref->next;
368 if (!memcmp(ref->name, "refs/", 5) &&
369 check_ref_format(ref->name + 5))
370 ; /* trash */
371 else if (args.fetch_all &&
372 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
373 *newtail = ref;
374 ref->next = NULL;
375 newtail = &ref->next;
376 continue;
378 else {
379 int order = path_match(ref->name, nr_match, match);
380 if (order) {
381 return_refs[order-1] = ref;
382 continue; /* we will link it later */
385 free(ref);
388 if (!args.fetch_all) {
389 int i;
390 for (i = 0; i < nr_match; i++) {
391 ref = return_refs[i];
392 if (ref) {
393 *newtail = ref;
394 ref->next = NULL;
395 newtail = &ref->next;
398 if (return_refs != fastarray)
399 free(return_refs);
401 *refs = newlist;
404 static int everything_local(struct ref **refs, int nr_match, char **match)
406 struct ref *ref;
407 int retval;
408 unsigned long cutoff = 0;
410 save_commit_buffer = 0;
412 for (ref = *refs; ref; ref = ref->next) {
413 struct object *o;
415 o = parse_object(ref->old_sha1);
416 if (!o)
417 continue;
419 /* We already have it -- which may mean that we were
420 * in sync with the other side at some time after
421 * that (it is OK if we guess wrong here).
423 if (o->type == OBJ_COMMIT) {
424 struct commit *commit = (struct commit *)o;
425 if (!cutoff || cutoff < commit->date)
426 cutoff = commit->date;
430 if (!args.depth) {
431 for_each_ref(mark_complete, NULL);
432 if (cutoff)
433 mark_recent_complete_commits(cutoff);
437 * Mark all complete remote refs as common refs.
438 * Don't mark them common yet; the server has to be told so first.
440 for (ref = *refs; ref; ref = ref->next) {
441 struct object *o = deref_tag(lookup_object(ref->old_sha1),
442 NULL, 0);
444 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
445 continue;
447 if (!(o->flags & SEEN)) {
448 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
450 mark_common((struct commit *)o, 1, 1);
454 filter_refs(refs, nr_match, match);
456 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
457 const unsigned char *remote = ref->old_sha1;
458 unsigned char local[20];
459 struct object *o;
461 o = lookup_object(remote);
462 if (!o || !(o->flags & COMPLETE)) {
463 retval = 0;
464 if (!args.verbose)
465 continue;
466 fprintf(stderr,
467 "want %s (%s)\n", sha1_to_hex(remote),
468 ref->name);
469 continue;
472 hashcpy(ref->new_sha1, local);
473 if (!args.verbose)
474 continue;
475 fprintf(stderr,
476 "already have %s (%s)\n", sha1_to_hex(remote),
477 ref->name);
479 return retval;
482 static int sideband_demux(int fd, void *data)
484 int *xd = data;
486 int ret = recv_sideband("fetch-pack", xd[0], fd);
487 close(fd);
488 return ret;
491 static int get_pack(int xd[2], char **pack_lockfile)
493 struct async demux;
494 const char *argv[20];
495 char keep_arg[256];
496 char hdr_arg[256];
497 const char **av;
498 int do_keep = args.keep_pack;
499 struct child_process cmd;
501 memset(&demux, 0, sizeof(demux));
502 if (use_sideband) {
503 /* xd[] is talking with upload-pack; subprocess reads from
504 * xd[0], spits out band#2 to stderr, and feeds us band#1
505 * through demux->out.
507 demux.proc = sideband_demux;
508 demux.data = xd;
509 if (start_async(&demux))
510 die("fetch-pack: unable to fork off sideband"
511 " demultiplexer");
513 else
514 demux.out = xd[0];
516 memset(&cmd, 0, sizeof(cmd));
517 cmd.argv = argv;
518 av = argv;
519 *hdr_arg = 0;
520 if (!args.keep_pack && unpack_limit) {
521 struct pack_header header;
523 if (read_pack_header(demux.out, &header))
524 die("protocol error: bad pack header");
525 snprintf(hdr_arg, sizeof(hdr_arg),
526 "--pack_header=%"PRIu32",%"PRIu32,
527 ntohl(header.hdr_version), ntohl(header.hdr_entries));
528 if (ntohl(header.hdr_entries) < unpack_limit)
529 do_keep = 0;
530 else
531 do_keep = 1;
534 if (do_keep) {
535 if (pack_lockfile)
536 cmd.out = -1;
537 *av++ = "index-pack";
538 *av++ = "--stdin";
539 if (!args.quiet && !args.no_progress)
540 *av++ = "-v";
541 if (args.use_thin_pack)
542 *av++ = "--fix-thin";
543 if (args.lock_pack || unpack_limit) {
544 int s = sprintf(keep_arg,
545 "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
546 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
547 strcpy(keep_arg + s, "localhost");
548 *av++ = keep_arg;
551 else {
552 *av++ = "unpack-objects";
553 if (args.quiet)
554 *av++ = "-q";
556 if (*hdr_arg)
557 *av++ = hdr_arg;
558 *av++ = NULL;
560 cmd.in = demux.out;
561 cmd.git_cmd = 1;
562 if (start_command(&cmd))
563 die("fetch-pack: unable to fork off %s", argv[0]);
564 if (do_keep && pack_lockfile) {
565 *pack_lockfile = index_pack_lockfile(cmd.out);
566 close(cmd.out);
569 if (finish_command(&cmd))
570 die("%s failed", argv[0]);
571 if (use_sideband && finish_async(&demux))
572 die("error in sideband demultiplexer");
573 return 0;
576 static struct ref *do_fetch_pack(int fd[2],
577 const struct ref *orig_ref,
578 int nr_match,
579 char **match,
580 char **pack_lockfile)
582 struct ref *ref = copy_ref_list(orig_ref);
583 unsigned char sha1[20];
585 if (is_repository_shallow() && !server_supports("shallow"))
586 die("Server does not support shallow clients");
587 if (server_supports("multi_ack")) {
588 if (args.verbose)
589 fprintf(stderr, "Server supports multi_ack\n");
590 multi_ack = 1;
592 if (server_supports("side-band-64k")) {
593 if (args.verbose)
594 fprintf(stderr, "Server supports side-band-64k\n");
595 use_sideband = 2;
597 else if (server_supports("side-band")) {
598 if (args.verbose)
599 fprintf(stderr, "Server supports side-band\n");
600 use_sideband = 1;
602 if (server_supports("ofs-delta")) {
603 if (args.verbose)
604 fprintf(stderr, "Server supports ofs-delta\n");
605 } else
606 prefer_ofs_delta = 0;
607 if (everything_local(&ref, nr_match, match)) {
608 packet_flush(fd[1]);
609 goto all_done;
611 if (find_common(fd, sha1, ref) < 0)
612 if (!args.keep_pack)
613 /* When cloning, it is not unusual to have
614 * no common commit.
616 warning("no common commits");
618 if (get_pack(fd, pack_lockfile))
619 die("git fetch-pack: fetch failed.");
621 all_done:
622 return ref;
625 static int remove_duplicates(int nr_heads, char **heads)
627 int src, dst;
629 for (src = dst = 0; src < nr_heads; src++) {
630 /* If heads[src] is different from any of
631 * heads[0..dst], push it in.
633 int i;
634 for (i = 0; i < dst; i++) {
635 if (!strcmp(heads[i], heads[src]))
636 break;
638 if (i < dst)
639 continue;
640 if (src != dst)
641 heads[dst] = heads[src];
642 dst++;
644 return dst;
647 static int fetch_pack_config(const char *var, const char *value, void *cb)
649 if (strcmp(var, "fetch.unpacklimit") == 0) {
650 fetch_unpack_limit = git_config_int(var, value);
651 return 0;
654 if (strcmp(var, "transfer.unpacklimit") == 0) {
655 transfer_unpack_limit = git_config_int(var, value);
656 return 0;
659 if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
660 prefer_ofs_delta = git_config_bool(var, value);
661 return 0;
664 return git_default_config(var, value, cb);
667 static struct lock_file lock;
669 static void fetch_pack_setup(void)
671 static int did_setup;
672 if (did_setup)
673 return;
674 git_config(fetch_pack_config, NULL);
675 if (0 <= transfer_unpack_limit)
676 unpack_limit = transfer_unpack_limit;
677 else if (0 <= fetch_unpack_limit)
678 unpack_limit = fetch_unpack_limit;
679 did_setup = 1;
682 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
684 int i, ret, nr_heads;
685 struct ref *ref = NULL;
686 char *dest = NULL, **heads;
687 int fd[2];
688 struct child_process *conn;
690 nr_heads = 0;
691 heads = NULL;
692 for (i = 1; i < argc; i++) {
693 const char *arg = argv[i];
695 if (*arg == '-') {
696 if (!prefixcmp(arg, "--upload-pack=")) {
697 args.uploadpack = arg + 14;
698 continue;
700 if (!prefixcmp(arg, "--exec=")) {
701 args.uploadpack = arg + 7;
702 continue;
704 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
705 args.quiet = 1;
706 continue;
708 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
709 args.lock_pack = args.keep_pack;
710 args.keep_pack = 1;
711 continue;
713 if (!strcmp("--thin", arg)) {
714 args.use_thin_pack = 1;
715 continue;
717 if (!strcmp("--include-tag", arg)) {
718 args.include_tag = 1;
719 continue;
721 if (!strcmp("--all", arg)) {
722 args.fetch_all = 1;
723 continue;
725 if (!strcmp("-v", arg)) {
726 args.verbose = 1;
727 continue;
729 if (!prefixcmp(arg, "--depth=")) {
730 args.depth = strtol(arg + 8, NULL, 0);
731 continue;
733 if (!strcmp("--no-progress", arg)) {
734 args.no_progress = 1;
735 continue;
737 usage(fetch_pack_usage);
739 dest = (char *)arg;
740 heads = (char **)(argv + i + 1);
741 nr_heads = argc - i - 1;
742 break;
744 if (!dest)
745 usage(fetch_pack_usage);
747 conn = git_connect(fd, (char *)dest, args.uploadpack,
748 args.verbose ? CONNECT_VERBOSE : 0);
749 if (conn) {
750 get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL);
752 ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
753 close(fd[0]);
754 close(fd[1]);
755 if (finish_connect(conn))
756 ref = NULL;
757 } else {
758 ref = NULL;
760 ret = !ref;
762 if (!ret && nr_heads) {
763 /* If the heads to pull were given, we should have
764 * consumed all of them by matching the remote.
765 * Otherwise, 'git fetch remote no-such-ref' would
766 * silently succeed without issuing an error.
768 for (i = 0; i < nr_heads; i++)
769 if (heads[i] && heads[i][0]) {
770 error("no such remote ref %s", heads[i]);
771 ret = 1;
774 while (ref) {
775 printf("%s %s\n",
776 sha1_to_hex(ref->old_sha1), ref->name);
777 ref = ref->next;
780 return ret;
783 struct ref *fetch_pack(struct fetch_pack_args *my_args,
784 int fd[], struct child_process *conn,
785 const struct ref *ref,
786 const char *dest,
787 int nr_heads,
788 char **heads,
789 char **pack_lockfile)
791 struct stat st;
792 struct ref *ref_cpy;
794 fetch_pack_setup();
795 if (&args != my_args)
796 memcpy(&args, my_args, sizeof(args));
797 if (args.depth > 0) {
798 if (stat(git_path("shallow"), &st))
799 st.st_mtime = 0;
802 if (heads && nr_heads)
803 nr_heads = remove_duplicates(nr_heads, heads);
804 if (!ref) {
805 packet_flush(fd[1]);
806 die("no matching remote head");
808 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
810 if (args.depth > 0) {
811 struct cache_time mtime;
812 char *shallow = git_path("shallow");
813 int fd;
815 mtime.sec = st.st_mtime;
816 mtime.nsec = ST_MTIME_NSEC(st);
817 if (stat(shallow, &st)) {
818 if (mtime.sec)
819 die("shallow file was removed during fetch");
820 } else if (st.st_mtime != mtime.sec
821 #ifdef USE_NSEC
822 || ST_MTIME_NSEC(st) != mtime.nsec
823 #endif
825 die("shallow file was changed during fetch");
827 fd = hold_lock_file_for_update(&lock, shallow,
828 LOCK_DIE_ON_ERROR);
829 if (!write_shallow_commits(fd, 0)) {
830 unlink_or_warn(shallow);
831 rollback_lock_file(&lock);
832 } else {
833 commit_lock_file(&lock);
837 reprepare_packed_git();
838 return ref_cpy;