initial commit (same functionality as wingit)
[git/mingw/4msysgit/wingit-dll.git] / builtin-fetch-pack.c
blob6202462216f3b8c3cbe924d4cc002616e69d75ab
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9 #include "fetch-pack.h"
10 #include "remote.h"
11 #include "run-command.h"
13 static int transfer_unpack_limit = -1;
14 static int fetch_unpack_limit = -1;
15 static int unpack_limit = 100;
16 static int prefer_ofs_delta = 1;
17 static struct fetch_pack_args args = {
18 /* .uploadpack = */ "git-upload-pack",
21 static const char fetch_pack_usage[] =
22 "git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
24 #define COMPLETE (1U << 0)
25 #define COMMON (1U << 1)
26 #define COMMON_REF (1U << 2)
27 #define SEEN (1U << 3)
28 #define POPPED (1U << 4)
30 static int marked;
33 * After sending this many "have"s if we do not get any new ACK , we
34 * give up traversing our history.
36 #define MAX_IN_VAIN 256
38 static struct commit_list *rev_list;
39 static int non_common_revs, multi_ack, use_sideband;
41 static void rev_list_push(struct commit *commit, int mark)
43 if (!(commit->object.flags & mark)) {
44 commit->object.flags |= mark;
46 if (!(commit->object.parsed))
47 if (parse_commit(commit))
48 return;
50 insert_by_date(commit, &rev_list);
52 if (!(commit->object.flags & COMMON))
53 non_common_revs++;
57 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
59 struct object *o = deref_tag(parse_object(sha1), path, 0);
61 if (o && o->type == OBJ_COMMIT)
62 rev_list_push((struct commit *)o, SEEN);
64 return 0;
67 static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data)
69 struct object *o = deref_tag(parse_object(sha1), path, 0);
71 if (o && o->type == OBJ_COMMIT)
72 clear_commit_marks((struct commit *)o,
73 COMMON | COMMON_REF | SEEN | POPPED);
74 return 0;
78 This function marks a rev and its ancestors as common.
79 In some cases, it is desirable to mark only the ancestors (for example
80 when only the server does not yet know that they are common).
83 static void mark_common(struct commit *commit,
84 int ancestors_only, int dont_parse)
86 if (commit != NULL && !(commit->object.flags & COMMON)) {
87 struct object *o = (struct object *)commit;
89 if (!ancestors_only)
90 o->flags |= COMMON;
92 if (!(o->flags & SEEN))
93 rev_list_push(commit, SEEN);
94 else {
95 struct commit_list *parents;
97 if (!ancestors_only && !(o->flags & POPPED))
98 non_common_revs--;
99 if (!o->parsed && !dont_parse)
100 if (parse_commit(commit))
101 return;
103 for (parents = commit->parents;
104 parents;
105 parents = parents->next)
106 mark_common(parents->item, 0, dont_parse);
112 Get the next rev to send, ignoring the common.
115 static const unsigned char *get_rev(void)
117 struct commit *commit = NULL;
119 while (commit == NULL) {
120 unsigned int mark;
121 struct commit_list *parents;
123 if (rev_list == NULL || non_common_revs == 0)
124 return NULL;
126 commit = rev_list->item;
127 if (!commit->object.parsed)
128 parse_commit(commit);
129 parents = commit->parents;
131 commit->object.flags |= POPPED;
132 if (!(commit->object.flags & COMMON))
133 non_common_revs--;
135 if (commit->object.flags & COMMON) {
136 /* do not send "have", and ignore ancestors */
137 commit = NULL;
138 mark = COMMON | SEEN;
139 } else if (commit->object.flags & COMMON_REF)
140 /* send "have", and ignore ancestors */
141 mark = COMMON | SEEN;
142 else
143 /* send "have", also for its ancestors */
144 mark = SEEN;
146 while (parents) {
147 if (!(parents->item->object.flags & SEEN))
148 rev_list_push(parents->item, mark);
149 if (mark & COMMON)
150 mark_common(parents->item, 1, 0);
151 parents = parents->next;
154 rev_list = rev_list->next;
157 return commit->object.sha1;
160 static int find_common(int fd[2], unsigned char *result_sha1,
161 struct ref *refs)
163 int fetching;
164 int count = 0, flushes = 0, retval;
165 const unsigned char *sha1;
166 unsigned in_vain = 0;
167 int got_continue = 0;
169 if (marked)
170 for_each_ref(clear_marks, NULL);
171 marked = 1;
173 for_each_ref(rev_list_insert_ref, NULL);
175 fetching = 0;
176 for ( ; refs ; refs = refs->next) {
177 unsigned char *remote = refs->old_sha1;
178 struct object *o;
181 * If that object is complete (i.e. it is an ancestor of a
182 * local ref), we tell them we have it but do not have to
183 * tell them about its ancestors, which they already know
184 * about.
186 * We use lookup_object here because we are only
187 * interested in the case we *know* the object is
188 * reachable and we have already scanned it.
190 if (((o = lookup_object(remote)) != NULL) &&
191 (o->flags & COMPLETE)) {
192 continue;
195 if (!fetching)
196 packet_write(fd[1], "want %s%s%s%s%s%s%s%s\n",
197 sha1_to_hex(remote),
198 (multi_ack ? " multi_ack" : ""),
199 (use_sideband == 2 ? " side-band-64k" : ""),
200 (use_sideband == 1 ? " side-band" : ""),
201 (args.use_thin_pack ? " thin-pack" : ""),
202 (args.no_progress ? " no-progress" : ""),
203 (args.include_tag ? " include-tag" : ""),
204 (prefer_ofs_delta ? " ofs-delta" : ""));
205 else
206 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
207 fetching++;
209 if (is_repository_shallow())
210 write_shallow_commits(fd[1], 1);
211 if (args.depth > 0)
212 packet_write(fd[1], "deepen %d", args.depth);
213 packet_flush(fd[1]);
214 if (!fetching)
215 return 1;
217 if (args.depth > 0) {
218 char line[1024];
219 unsigned char sha1[20];
221 while (packet_read_line(fd[0], line, sizeof(line))) {
222 if (!prefixcmp(line, "shallow ")) {
223 if (get_sha1_hex(line + 8, sha1))
224 die("invalid shallow line: %s", line);
225 register_shallow(sha1);
226 continue;
228 if (!prefixcmp(line, "unshallow ")) {
229 if (get_sha1_hex(line + 10, sha1))
230 die("invalid unshallow line: %s", line);
231 if (!lookup_object(sha1))
232 die("object not found: %s", line);
233 /* make sure that it is parsed as shallow */
234 if (!parse_object(sha1))
235 die("error in object: %s", line);
236 if (unregister_shallow(sha1))
237 die("no shallow found: %s", line);
238 continue;
240 die("expected shallow/unshallow, got %s", line);
244 flushes = 0;
245 retval = -1;
246 while ((sha1 = get_rev())) {
247 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
248 if (args.verbose)
249 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
250 in_vain++;
251 if (!(31 & ++count)) {
252 int ack;
254 packet_flush(fd[1]);
255 flushes++;
258 * We keep one window "ahead" of the other side, and
259 * will wait for an ACK only on the next one
261 if (count == 32)
262 continue;
264 do {
265 ack = get_ack(fd[0], result_sha1);
266 if (args.verbose && ack)
267 fprintf(stderr, "got ack %d %s\n", ack,
268 sha1_to_hex(result_sha1));
269 if (ack == 1) {
270 flushes = 0;
271 multi_ack = 0;
272 retval = 0;
273 goto done;
274 } else if (ack == 2) {
275 struct commit *commit =
276 lookup_commit(result_sha1);
277 mark_common(commit, 0, 1);
278 retval = 0;
279 in_vain = 0;
280 got_continue = 1;
282 } while (ack);
283 flushes--;
284 if (got_continue && MAX_IN_VAIN < in_vain) {
285 if (args.verbose)
286 fprintf(stderr, "giving up\n");
287 break; /* give up */
291 done:
292 packet_write(fd[1], "done\n");
293 if (args.verbose)
294 fprintf(stderr, "done\n");
295 if (retval != 0) {
296 multi_ack = 0;
297 flushes++;
299 while (flushes || multi_ack) {
300 int ack = get_ack(fd[0], result_sha1);
301 if (ack) {
302 if (args.verbose)
303 fprintf(stderr, "got ack (%d) %s\n", ack,
304 sha1_to_hex(result_sha1));
305 if (ack == 1)
306 return 0;
307 multi_ack = 1;
308 continue;
310 flushes--;
312 /* it is no error to fetch into a completely empty repo */
313 return count ? retval : 0;
316 static struct commit_list *complete;
318 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
320 struct object *o = parse_object(sha1);
322 while (o && o->type == OBJ_TAG) {
323 struct tag *t = (struct tag *) o;
324 if (!t->tagged)
325 break; /* broken repository */
326 o->flags |= COMPLETE;
327 o = parse_object(t->tagged->sha1);
329 if (o && o->type == OBJ_COMMIT) {
330 struct commit *commit = (struct commit *)o;
331 commit->object.flags |= COMPLETE;
332 insert_by_date(commit, &complete);
334 return 0;
337 static void mark_recent_complete_commits(unsigned long cutoff)
339 while (complete && cutoff <= complete->item->date) {
340 if (args.verbose)
341 fprintf(stderr, "Marking %s as complete\n",
342 sha1_to_hex(complete->item->object.sha1));
343 pop_most_recent_commit(&complete, COMPLETE);
347 static void filter_refs(struct ref **refs, int nr_match, char **match)
349 struct ref **return_refs;
350 struct ref *newlist = NULL;
351 struct ref **newtail = &newlist;
352 struct ref *ref, *next;
353 struct ref *fastarray[32];
355 if (nr_match && !args.fetch_all) {
356 if (ARRAY_SIZE(fastarray) < nr_match)
357 return_refs = xcalloc(nr_match, sizeof(struct ref *));
358 else {
359 return_refs = fastarray;
360 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
363 else
364 return_refs = NULL;
366 for (ref = *refs; ref; ref = next) {
367 next = ref->next;
368 if (!memcmp(ref->name, "refs/", 5) &&
369 check_ref_format(ref->name + 5))
370 ; /* trash */
371 else if (args.fetch_all &&
372 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
373 *newtail = ref;
374 ref->next = NULL;
375 newtail = &ref->next;
376 continue;
378 else {
379 int order = path_match(ref->name, nr_match, match);
380 if (order) {
381 return_refs[order-1] = ref;
382 continue; /* we will link it later */
385 free(ref);
388 if (!args.fetch_all) {
389 int i;
390 for (i = 0; i < nr_match; i++) {
391 ref = return_refs[i];
392 if (ref) {
393 *newtail = ref;
394 ref->next = NULL;
395 newtail = &ref->next;
398 if (return_refs != fastarray)
399 free(return_refs);
401 *refs = newlist;
404 static int everything_local(struct ref **refs, int nr_match, char **match)
406 struct ref *ref;
407 int retval;
408 unsigned long cutoff = 0;
410 save_commit_buffer = 0;
412 for (ref = *refs; ref; ref = ref->next) {
413 struct object *o;
415 o = parse_object(ref->old_sha1);
416 if (!o)
417 continue;
419 /* We already have it -- which may mean that we were
420 * in sync with the other side at some time after
421 * that (it is OK if we guess wrong here).
423 if (o->type == OBJ_COMMIT) {
424 struct commit *commit = (struct commit *)o;
425 if (!cutoff || cutoff < commit->date)
426 cutoff = commit->date;
430 if (!args.depth) {
431 for_each_ref(mark_complete, NULL);
432 if (cutoff)
433 mark_recent_complete_commits(cutoff);
437 * Mark all complete remote refs as common refs.
438 * Don't mark them common yet; the server has to be told so first.
440 for (ref = *refs; ref; ref = ref->next) {
441 struct object *o = deref_tag(lookup_object(ref->old_sha1),
442 NULL, 0);
444 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
445 continue;
447 if (!(o->flags & SEEN)) {
448 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
450 mark_common((struct commit *)o, 1, 1);
454 filter_refs(refs, nr_match, match);
456 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
457 const unsigned char *remote = ref->old_sha1;
458 unsigned char local[20];
459 struct object *o;
461 o = lookup_object(remote);
462 if (!o || !(o->flags & COMPLETE)) {
463 retval = 0;
464 if (!args.verbose)
465 continue;
466 fprintf(stderr,
467 "want %s (%s)\n", sha1_to_hex(remote),
468 ref->name);
469 continue;
472 hashcpy(ref->new_sha1, local);
473 if (!args.verbose)
474 continue;
475 fprintf(stderr,
476 "already have %s (%s)\n", sha1_to_hex(remote),
477 ref->name);
479 return retval;
482 static int sideband_demux(int fd, void *data)
484 int *xd = data;
486 return recv_sideband("fetch-pack", xd[0], fd);
489 static int get_pack(int xd[2], char **pack_lockfile)
491 struct async demux;
492 const char *argv[20];
493 char keep_arg[256];
494 char hdr_arg[256];
495 const char **av;
496 int do_keep = args.keep_pack;
497 struct child_process cmd;
499 memset(&demux, 0, sizeof(demux));
500 if (use_sideband) {
501 /* xd[] is talking with upload-pack; subprocess reads from
502 * xd[0], spits out band#2 to stderr, and feeds us band#1
503 * through demux->out.
505 demux.proc = sideband_demux;
506 demux.data = xd;
507 if (start_async(&demux))
508 die("fetch-pack: unable to fork off sideband"
509 " demultiplexer");
511 else
512 demux.out = xd[0];
514 memset(&cmd, 0, sizeof(cmd));
515 cmd.argv = argv;
516 av = argv;
517 *hdr_arg = 0;
518 if (!args.keep_pack && unpack_limit) {
519 struct pack_header header;
521 if (read_pack_header(demux.out, &header))
522 die("protocol error: bad pack header");
523 snprintf(hdr_arg, sizeof(hdr_arg),
524 "--pack_header=%"PRIu32",%"PRIu32,
525 ntohl(header.hdr_version), ntohl(header.hdr_entries));
526 if (ntohl(header.hdr_entries) < unpack_limit)
527 do_keep = 0;
528 else
529 do_keep = 1;
532 if (do_keep) {
533 if (pack_lockfile)
534 cmd.out = -1;
535 *av++ = "index-pack";
536 *av++ = "--stdin";
537 if (!args.quiet && !args.no_progress)
538 *av++ = "-v";
539 if (args.use_thin_pack)
540 *av++ = "--fix-thin";
541 if (args.lock_pack || unpack_limit) {
542 int s = sprintf(keep_arg,
543 "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
544 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
545 strcpy(keep_arg + s, "localhost");
546 *av++ = keep_arg;
549 else {
550 *av++ = "unpack-objects";
551 if (args.quiet)
552 *av++ = "-q";
554 if (*hdr_arg)
555 *av++ = hdr_arg;
556 *av++ = NULL;
558 cmd.in = demux.out;
559 cmd.git_cmd = 1;
560 if (start_command(&cmd))
561 die("fetch-pack: unable to fork off %s", argv[0]);
562 if (do_keep && pack_lockfile) {
563 *pack_lockfile = index_pack_lockfile(cmd.out);
564 close(cmd.out);
567 if (finish_command(&cmd))
568 die("%s failed", argv[0]);
569 if (use_sideband && finish_async(&demux))
570 die("error in sideband demultiplexer");
571 return 0;
574 static struct ref *do_fetch_pack(int fd[2],
575 const struct ref *orig_ref,
576 int nr_match,
577 char **match,
578 char **pack_lockfile)
580 struct ref *ref = copy_ref_list(orig_ref);
581 unsigned char sha1[20];
583 if (is_repository_shallow() && !server_supports("shallow"))
584 die("Server does not support shallow clients");
585 if (server_supports("multi_ack")) {
586 if (args.verbose)
587 fprintf(stderr, "Server supports multi_ack\n");
588 multi_ack = 1;
590 if (server_supports("side-band-64k")) {
591 if (args.verbose)
592 fprintf(stderr, "Server supports side-band-64k\n");
593 use_sideband = 2;
595 else if (server_supports("side-band")) {
596 if (args.verbose)
597 fprintf(stderr, "Server supports side-band\n");
598 use_sideband = 1;
600 if (server_supports("ofs-delta")) {
601 if (args.verbose)
602 fprintf(stderr, "Server supports ofs-delta\n");
603 } else
604 prefer_ofs_delta = 0;
605 if (everything_local(&ref, nr_match, match)) {
606 packet_flush(fd[1]);
607 goto all_done;
609 if (find_common(fd, sha1, ref) < 0)
610 if (!args.keep_pack)
611 /* When cloning, it is not unusual to have
612 * no common commit.
614 warning("no common commits");
616 if (get_pack(fd, pack_lockfile))
617 die("git fetch-pack: fetch failed.");
619 all_done:
620 return ref;
623 static int remove_duplicates(int nr_heads, char **heads)
625 int src, dst;
627 for (src = dst = 0; src < nr_heads; src++) {
628 /* If heads[src] is different from any of
629 * heads[0..dst], push it in.
631 int i;
632 for (i = 0; i < dst; i++) {
633 if (!strcmp(heads[i], heads[src]))
634 break;
636 if (i < dst)
637 continue;
638 if (src != dst)
639 heads[dst] = heads[src];
640 dst++;
642 return dst;
645 static int fetch_pack_config(const char *var, const char *value, void *cb)
647 if (strcmp(var, "fetch.unpacklimit") == 0) {
648 fetch_unpack_limit = git_config_int(var, value);
649 return 0;
652 if (strcmp(var, "transfer.unpacklimit") == 0) {
653 transfer_unpack_limit = git_config_int(var, value);
654 return 0;
657 if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
658 prefer_ofs_delta = git_config_bool(var, value);
659 return 0;
662 return git_default_config(var, value, cb);
665 static struct lock_file lock;
667 static void fetch_pack_setup(void)
669 static int did_setup;
670 if (did_setup)
671 return;
672 git_config(fetch_pack_config, NULL);
673 if (0 <= transfer_unpack_limit)
674 unpack_limit = transfer_unpack_limit;
675 else if (0 <= fetch_unpack_limit)
676 unpack_limit = fetch_unpack_limit;
677 did_setup = 1;
680 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
682 int i, ret, nr_heads;
683 struct ref *ref = NULL;
684 char *dest = NULL, **heads;
685 int fd[2];
686 struct child_process *conn;
688 nr_heads = 0;
689 heads = NULL;
690 for (i = 1; i < argc; i++) {
691 const char *arg = argv[i];
693 if (*arg == '-') {
694 if (!prefixcmp(arg, "--upload-pack=")) {
695 args.uploadpack = arg + 14;
696 continue;
698 if (!prefixcmp(arg, "--exec=")) {
699 args.uploadpack = arg + 7;
700 continue;
702 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
703 args.quiet = 1;
704 continue;
706 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
707 args.lock_pack = args.keep_pack;
708 args.keep_pack = 1;
709 continue;
711 if (!strcmp("--thin", arg)) {
712 args.use_thin_pack = 1;
713 continue;
715 if (!strcmp("--include-tag", arg)) {
716 args.include_tag = 1;
717 continue;
719 if (!strcmp("--all", arg)) {
720 args.fetch_all = 1;
721 continue;
723 if (!strcmp("-v", arg)) {
724 args.verbose = 1;
725 continue;
727 if (!prefixcmp(arg, "--depth=")) {
728 args.depth = strtol(arg + 8, NULL, 0);
729 continue;
731 if (!strcmp("--no-progress", arg)) {
732 args.no_progress = 1;
733 continue;
735 usage(fetch_pack_usage);
737 dest = (char *)arg;
738 heads = (char **)(argv + i + 1);
739 nr_heads = argc - i - 1;
740 break;
742 if (!dest)
743 usage(fetch_pack_usage);
745 conn = git_connect(fd, (char *)dest, args.uploadpack,
746 args.verbose ? CONNECT_VERBOSE : 0);
747 if (conn) {
748 get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL);
750 ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
751 close(fd[0]);
752 close(fd[1]);
753 if (finish_connect(conn))
754 ref = NULL;
755 } else {
756 ref = NULL;
758 ret = !ref;
760 if (!ret && nr_heads) {
761 /* If the heads to pull were given, we should have
762 * consumed all of them by matching the remote.
763 * Otherwise, 'git fetch remote no-such-ref' would
764 * silently succeed without issuing an error.
766 for (i = 0; i < nr_heads; i++)
767 if (heads[i] && heads[i][0]) {
768 error("no such remote ref %s", heads[i]);
769 ret = 1;
772 while (ref) {
773 printf("%s %s\n",
774 sha1_to_hex(ref->old_sha1), ref->name);
775 ref = ref->next;
778 return ret;
781 struct ref *fetch_pack(struct fetch_pack_args *my_args,
782 int fd[], struct child_process *conn,
783 const struct ref *ref,
784 const char *dest,
785 int nr_heads,
786 char **heads,
787 char **pack_lockfile)
789 struct stat st;
790 struct ref *ref_cpy;
792 fetch_pack_setup();
793 if (&args != my_args)
794 memcpy(&args, my_args, sizeof(args));
795 if (args.depth > 0) {
796 if (stat(git_path("shallow"), &st))
797 st.st_mtime = 0;
800 if (heads && nr_heads)
801 nr_heads = remove_duplicates(nr_heads, heads);
802 if (!ref) {
803 packet_flush(fd[1]);
804 die("no matching remote head");
806 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
808 if (args.depth > 0) {
809 struct cache_time mtime;
810 char *shallow = git_path("shallow");
811 int fd;
813 mtime.sec = st.st_mtime;
814 mtime.nsec = ST_MTIME_NSEC(st);
815 if (stat(shallow, &st)) {
816 if (mtime.sec)
817 die("shallow file was removed during fetch");
818 } else if (st.st_mtime != mtime.sec
819 #ifdef USE_NSEC
820 || ST_MTIME_NSEC(st) != mtime.nsec
821 #endif
823 die("shallow file was changed during fetch");
825 fd = hold_lock_file_for_update(&lock, shallow,
826 LOCK_DIE_ON_ERROR);
827 if (!write_shallow_commits(fd, 0)) {
828 unlink_or_warn(shallow);
829 rollback_lock_file(&lock);
830 } else {
831 commit_lock_file(&lock);
835 reprepare_packed_git();
836 return ref_cpy;