skip t5512 because remote does not yet work
[git/platforms/storm.git] / builtin-fetch-pack.c
blobb21ba2eb95267c7402dfe5a9112fb77cd4fe3268
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9 #include "fetch-pack.h"
10 #include "run-command.h"
12 static int transfer_unpack_limit = -1;
13 static int fetch_unpack_limit = -1;
14 static int unpack_limit = 100;
15 static struct fetch_pack_args args = {
16 /* .uploadpack = */ "git-upload-pack",
19 static const char fetch_pack_usage[] =
20 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
22 #define COMPLETE (1U << 0)
23 #define COMMON (1U << 1)
24 #define COMMON_REF (1U << 2)
25 #define SEEN (1U << 3)
26 #define POPPED (1U << 4)
29 * After sending this many "have"s if we do not get any new ACK , we
30 * give up traversing our history.
32 #define MAX_IN_VAIN 256
34 static struct commit_list *rev_list;
35 static int non_common_revs, multi_ack, use_sideband;
37 static void rev_list_push(struct commit *commit, int mark)
39 if (!(commit->object.flags & mark)) {
40 commit->object.flags |= mark;
42 if (!(commit->object.parsed))
43 parse_commit(commit);
45 insert_by_date(commit, &rev_list);
47 if (!(commit->object.flags & COMMON))
48 non_common_revs++;
52 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
54 struct object *o = deref_tag(parse_object(sha1), path, 0);
56 if (o && o->type == OBJ_COMMIT)
57 rev_list_push((struct commit *)o, SEEN);
59 return 0;
63 This function marks a rev and its ancestors as common.
64 In some cases, it is desirable to mark only the ancestors (for example
65 when only the server does not yet know that they are common).
68 static void mark_common(struct commit *commit,
69 int ancestors_only, int dont_parse)
71 if (commit != NULL && !(commit->object.flags & COMMON)) {
72 struct object *o = (struct object *)commit;
74 if (!ancestors_only)
75 o->flags |= COMMON;
77 if (!(o->flags & SEEN))
78 rev_list_push(commit, SEEN);
79 else {
80 struct commit_list *parents;
82 if (!ancestors_only && !(o->flags & POPPED))
83 non_common_revs--;
84 if (!o->parsed && !dont_parse)
85 parse_commit(commit);
87 for (parents = commit->parents;
88 parents;
89 parents = parents->next)
90 mark_common(parents->item, 0, dont_parse);
96 Get the next rev to send, ignoring the common.
99 static const unsigned char* get_rev(void)
101 struct commit *commit = NULL;
103 while (commit == NULL) {
104 unsigned int mark;
105 struct commit_list* parents;
107 if (rev_list == NULL || non_common_revs == 0)
108 return NULL;
110 commit = rev_list->item;
111 if (!(commit->object.parsed))
112 parse_commit(commit);
113 commit->object.flags |= POPPED;
114 if (!(commit->object.flags & COMMON))
115 non_common_revs--;
117 parents = commit->parents;
119 if (commit->object.flags & COMMON) {
120 /* do not send "have", and ignore ancestors */
121 commit = NULL;
122 mark = COMMON | SEEN;
123 } else if (commit->object.flags & COMMON_REF)
124 /* send "have", and ignore ancestors */
125 mark = COMMON | SEEN;
126 else
127 /* send "have", also for its ancestors */
128 mark = SEEN;
130 while (parents) {
131 if (!(parents->item->object.flags & SEEN))
132 rev_list_push(parents->item, mark);
133 if (mark & COMMON)
134 mark_common(parents->item, 1, 0);
135 parents = parents->next;
138 rev_list = rev_list->next;
141 return commit->object.sha1;
144 static int find_common(int fd[2], unsigned char *result_sha1,
145 struct ref *refs)
147 int fetching;
148 int count = 0, flushes = 0, retval;
149 const unsigned char *sha1;
150 unsigned in_vain = 0;
151 int got_continue = 0;
153 for_each_ref(rev_list_insert_ref, NULL);
155 fetching = 0;
156 for ( ; refs ; refs = refs->next) {
157 unsigned char *remote = refs->old_sha1;
158 struct object *o;
161 * If that object is complete (i.e. it is an ancestor of a
162 * local ref), we tell them we have it but do not have to
163 * tell them about its ancestors, which they already know
164 * about.
166 * We use lookup_object here because we are only
167 * interested in the case we *know* the object is
168 * reachable and we have already scanned it.
170 if (((o = lookup_object(remote)) != NULL) &&
171 (o->flags & COMPLETE)) {
172 continue;
175 if (!fetching)
176 packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
177 sha1_to_hex(remote),
178 (multi_ack ? " multi_ack" : ""),
179 (use_sideband == 2 ? " side-band-64k" : ""),
180 (use_sideband == 1 ? " side-band" : ""),
181 (args.use_thin_pack ? " thin-pack" : ""),
182 (args.no_progress ? " no-progress" : ""),
183 " ofs-delta");
184 else
185 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
186 fetching++;
188 if (is_repository_shallow())
189 write_shallow_commits(fd[1], 1);
190 if (args.depth > 0)
191 packet_write(fd[1], "deepen %d", args.depth);
192 packet_flush(fd[1]);
193 if (!fetching)
194 return 1;
196 if (args.depth > 0) {
197 char line[1024];
198 unsigned char sha1[20];
199 int len;
201 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
202 if (!prefixcmp(line, "shallow ")) {
203 if (get_sha1_hex(line + 8, sha1))
204 die("invalid shallow line: %s", line);
205 register_shallow(sha1);
206 continue;
208 if (!prefixcmp(line, "unshallow ")) {
209 if (get_sha1_hex(line + 10, sha1))
210 die("invalid unshallow line: %s", line);
211 if (!lookup_object(sha1))
212 die("object not found: %s", line);
213 /* make sure that it is parsed as shallow */
214 parse_object(sha1);
215 if (unregister_shallow(sha1))
216 die("no shallow found: %s", line);
217 continue;
219 die("expected shallow/unshallow, got %s", line);
223 flushes = 0;
224 retval = -1;
225 while ((sha1 = get_rev())) {
226 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
227 if (args.verbose)
228 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
229 in_vain++;
230 if (!(31 & ++count)) {
231 int ack;
233 packet_flush(fd[1]);
234 flushes++;
237 * We keep one window "ahead" of the other side, and
238 * will wait for an ACK only on the next one
240 if (count == 32)
241 continue;
243 do {
244 ack = get_ack(fd[0], result_sha1);
245 if (args.verbose && ack)
246 fprintf(stderr, "got ack %d %s\n", ack,
247 sha1_to_hex(result_sha1));
248 if (ack == 1) {
249 flushes = 0;
250 multi_ack = 0;
251 retval = 0;
252 goto done;
253 } else if (ack == 2) {
254 struct commit *commit =
255 lookup_commit(result_sha1);
256 mark_common(commit, 0, 1);
257 retval = 0;
258 in_vain = 0;
259 got_continue = 1;
261 } while (ack);
262 flushes--;
263 if (got_continue && MAX_IN_VAIN < in_vain) {
264 if (args.verbose)
265 fprintf(stderr, "giving up\n");
266 break; /* give up */
270 done:
271 packet_write(fd[1], "done\n");
272 if (args.verbose)
273 fprintf(stderr, "done\n");
274 if (retval != 0) {
275 multi_ack = 0;
276 flushes++;
278 while (flushes || multi_ack) {
279 int ack = get_ack(fd[0], result_sha1);
280 if (ack) {
281 if (args.verbose)
282 fprintf(stderr, "got ack (%d) %s\n", ack,
283 sha1_to_hex(result_sha1));
284 if (ack == 1)
285 return 0;
286 multi_ack = 1;
287 continue;
289 flushes--;
291 /* it is no error to fetch into a completely empty repo */
292 return count ? retval : 0;
295 static struct commit_list *complete;
297 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
299 struct object *o = parse_object(sha1);
301 while (o && o->type == OBJ_TAG) {
302 struct tag *t = (struct tag *) o;
303 if (!t->tagged)
304 break; /* broken repository */
305 o->flags |= COMPLETE;
306 o = parse_object(t->tagged->sha1);
308 if (o && o->type == OBJ_COMMIT) {
309 struct commit *commit = (struct commit *)o;
310 commit->object.flags |= COMPLETE;
311 insert_by_date(commit, &complete);
313 return 0;
316 static void mark_recent_complete_commits(unsigned long cutoff)
318 while (complete && cutoff <= complete->item->date) {
319 if (args.verbose)
320 fprintf(stderr, "Marking %s as complete\n",
321 sha1_to_hex(complete->item->object.sha1));
322 pop_most_recent_commit(&complete, COMPLETE);
326 static void filter_refs(struct ref **refs, int nr_match, char **match)
328 struct ref **return_refs;
329 struct ref *newlist = NULL;
330 struct ref **newtail = &newlist;
331 struct ref *ref, *next;
332 struct ref *fastarray[32];
334 if (nr_match && !args.fetch_all) {
335 if (ARRAY_SIZE(fastarray) < nr_match)
336 return_refs = xcalloc(nr_match, sizeof(struct ref *));
337 else {
338 return_refs = fastarray;
339 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
342 else
343 return_refs = NULL;
345 for (ref = *refs; ref; ref = next) {
346 next = ref->next;
347 if (!memcmp(ref->name, "refs/", 5) &&
348 check_ref_format(ref->name + 5))
349 ; /* trash */
350 else if (args.fetch_all &&
351 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
352 *newtail = ref;
353 ref->next = NULL;
354 newtail = &ref->next;
355 continue;
357 else {
358 int order = path_match(ref->name, nr_match, match);
359 if (order) {
360 return_refs[order-1] = ref;
361 continue; /* we will link it later */
364 free(ref);
367 if (!args.fetch_all) {
368 int i;
369 for (i = 0; i < nr_match; i++) {
370 ref = return_refs[i];
371 if (ref) {
372 *newtail = ref;
373 ref->next = NULL;
374 newtail = &ref->next;
377 if (return_refs != fastarray)
378 free(return_refs);
380 *refs = newlist;
383 static int everything_local(struct ref **refs, int nr_match, char **match)
385 struct ref *ref;
386 int retval;
387 unsigned long cutoff = 0;
389 track_object_refs = 0;
390 save_commit_buffer = 0;
392 for (ref = *refs; ref; ref = ref->next) {
393 struct object *o;
395 o = parse_object(ref->old_sha1);
396 if (!o)
397 continue;
399 /* We already have it -- which may mean that we were
400 * in sync with the other side at some time after
401 * that (it is OK if we guess wrong here).
403 if (o->type == OBJ_COMMIT) {
404 struct commit *commit = (struct commit *)o;
405 if (!cutoff || cutoff < commit->date)
406 cutoff = commit->date;
410 if (!args.depth) {
411 for_each_ref(mark_complete, NULL);
412 if (cutoff)
413 mark_recent_complete_commits(cutoff);
417 * Mark all complete remote refs as common refs.
418 * Don't mark them common yet; the server has to be told so first.
420 for (ref = *refs; ref; ref = ref->next) {
421 struct object *o = deref_tag(lookup_object(ref->old_sha1),
422 NULL, 0);
424 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
425 continue;
427 if (!(o->flags & SEEN)) {
428 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
430 mark_common((struct commit *)o, 1, 1);
434 filter_refs(refs, nr_match, match);
436 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
437 const unsigned char *remote = ref->old_sha1;
438 unsigned char local[20];
439 struct object *o;
441 o = lookup_object(remote);
442 if (!o || !(o->flags & COMPLETE)) {
443 retval = 0;
444 if (!args.verbose)
445 continue;
446 fprintf(stderr,
447 "want %s (%s)\n", sha1_to_hex(remote),
448 ref->name);
449 continue;
452 hashcpy(ref->new_sha1, local);
453 if (!args.verbose)
454 continue;
455 fprintf(stderr,
456 "already have %s (%s)\n", sha1_to_hex(remote),
457 ref->name);
459 return retval;
462 static int sideband_demux(int fd, void *data)
464 int *xd = data;
466 return recv_sideband("fetch-pack", xd[0], fd, 2);
469 static int get_pack(int xd[2], char **pack_lockfile)
471 struct async demux;
472 const char *argv[20];
473 char keep_arg[256];
474 char hdr_arg[256];
475 const char **av;
476 int do_keep = args.keep_pack;
477 struct child_process cmd;
479 memset(&demux, 0, sizeof(demux));
480 if (use_sideband) {
481 /* xd[] is talking with upload-pack; subprocess reads from
482 * xd[0], spits out band#2 to stderr, and feeds us band#1
483 * through demux->out.
485 demux.proc = sideband_demux;
486 demux.data = xd;
487 if (start_async(&demux))
488 die("fetch-pack: unable to fork off sideband"
489 " demultiplexer");
491 else
492 demux.out = xd[0];
494 memset(&cmd, 0, sizeof(cmd));
495 cmd.argv = argv;
496 av = argv;
497 *hdr_arg = 0;
498 if (!args.keep_pack && unpack_limit) {
499 struct pack_header header;
501 if (read_pack_header(demux.out, &header))
502 die("protocol error: bad pack header");
503 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
504 ntohl(header.hdr_version), ntohl(header.hdr_entries));
505 if (ntohl(header.hdr_entries) < unpack_limit)
506 do_keep = 0;
507 else
508 do_keep = 1;
511 if (do_keep) {
512 if (pack_lockfile)
513 cmd.out = -1;
514 *av++ = "index-pack";
515 *av++ = "--stdin";
516 if (!args.quiet && !args.no_progress)
517 *av++ = "-v";
518 if (args.use_thin_pack)
519 *av++ = "--fix-thin";
520 if (args.lock_pack || unpack_limit) {
521 int s = sprintf(keep_arg,
522 "--keep=fetch-pack %d on ", getpid());
523 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
524 strcpy(keep_arg + s, "localhost");
525 *av++ = keep_arg;
528 else {
529 *av++ = "unpack-objects";
530 if (args.quiet)
531 *av++ = "-q";
533 if (*hdr_arg)
534 *av++ = hdr_arg;
535 *av++ = NULL;
537 cmd.in = demux.out;
538 cmd.git_cmd = 1;
539 if (start_command(&cmd))
540 die("fetch-pack: unable to fork off %s", argv[0]);
541 if (do_keep && pack_lockfile)
542 *pack_lockfile = index_pack_lockfile(cmd.out);
544 if (finish_command(&cmd))
545 die("%s failed", argv[0]);
546 if (use_sideband && finish_async(&demux))
547 die("error in sideband demultiplexer");
548 return 0;
551 static struct ref *do_fetch_pack(int fd[2],
552 int nr_match,
553 char **match,
554 char **pack_lockfile)
556 struct ref *ref;
557 unsigned char sha1[20];
559 get_remote_heads(fd[0], &ref, 0, NULL, 0);
560 if (is_repository_shallow() && !server_supports("shallow"))
561 die("Server does not support shallow clients");
562 if (server_supports("multi_ack")) {
563 if (args.verbose)
564 fprintf(stderr, "Server supports multi_ack\n");
565 multi_ack = 1;
567 if (server_supports("side-band-64k")) {
568 if (args.verbose)
569 fprintf(stderr, "Server supports side-band-64k\n");
570 use_sideband = 2;
572 else if (server_supports("side-band")) {
573 if (args.verbose)
574 fprintf(stderr, "Server supports side-band\n");
575 use_sideband = 1;
577 if (!ref) {
578 packet_flush(fd[1]);
579 die("no matching remote head");
581 if (everything_local(&ref, nr_match, match)) {
582 packet_flush(fd[1]);
583 goto all_done;
585 if (find_common(fd, sha1, ref) < 0)
586 if (!args.keep_pack)
587 /* When cloning, it is not unusual to have
588 * no common commit.
590 fprintf(stderr, "warning: no common commits\n");
592 if (get_pack(fd, pack_lockfile))
593 die("git-fetch-pack: fetch failed.");
595 all_done:
596 return ref;
599 static int remove_duplicates(int nr_heads, char **heads)
601 int src, dst;
603 for (src = dst = 0; src < nr_heads; src++) {
604 /* If heads[src] is different from any of
605 * heads[0..dst], push it in.
607 int i;
608 for (i = 0; i < dst; i++) {
609 if (!strcmp(heads[i], heads[src]))
610 break;
612 if (i < dst)
613 continue;
614 if (src != dst)
615 heads[dst] = heads[src];
616 dst++;
618 return dst;
621 static int fetch_pack_config(const char *var, const char *value)
623 if (strcmp(var, "fetch.unpacklimit") == 0) {
624 fetch_unpack_limit = git_config_int(var, value);
625 return 0;
628 if (strcmp(var, "transfer.unpacklimit") == 0) {
629 transfer_unpack_limit = git_config_int(var, value);
630 return 0;
633 return git_default_config(var, value);
636 static struct lock_file lock;
638 static void fetch_pack_setup(void)
640 static int did_setup;
641 if (did_setup)
642 return;
643 git_config(fetch_pack_config);
644 if (0 <= transfer_unpack_limit)
645 unpack_limit = transfer_unpack_limit;
646 else if (0 <= fetch_unpack_limit)
647 unpack_limit = fetch_unpack_limit;
648 did_setup = 1;
651 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
653 int i, ret, nr_heads;
654 struct ref *ref;
655 char *dest = NULL, **heads;
657 nr_heads = 0;
658 heads = NULL;
659 for (i = 1; i < argc; i++) {
660 const char *arg = argv[i];
662 if (*arg == '-') {
663 if (!prefixcmp(arg, "--upload-pack=")) {
664 args.uploadpack = arg + 14;
665 continue;
667 if (!prefixcmp(arg, "--exec=")) {
668 args.uploadpack = arg + 7;
669 continue;
671 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
672 args.quiet = 1;
673 continue;
675 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
676 args.lock_pack = args.keep_pack;
677 args.keep_pack = 1;
678 continue;
680 if (!strcmp("--thin", arg)) {
681 args.use_thin_pack = 1;
682 continue;
684 if (!strcmp("--all", arg)) {
685 args.fetch_all = 1;
686 continue;
688 if (!strcmp("-v", arg)) {
689 args.verbose = 1;
690 continue;
692 if (!prefixcmp(arg, "--depth=")) {
693 args.depth = strtol(arg + 8, NULL, 0);
694 continue;
696 if (!strcmp("--no-progress", arg)) {
697 args.no_progress = 1;
698 continue;
700 usage(fetch_pack_usage);
702 dest = (char *)arg;
703 heads = (char **)(argv + i + 1);
704 nr_heads = argc - i - 1;
705 break;
707 if (!dest)
708 usage(fetch_pack_usage);
710 ref = fetch_pack(&args, dest, nr_heads, heads, NULL);
711 ret = !ref;
713 while (ref) {
714 printf("%s %s\n",
715 sha1_to_hex(ref->old_sha1), ref->name);
716 ref = ref->next;
719 return ret;
722 struct ref *fetch_pack(struct fetch_pack_args *my_args,
723 const char *dest,
724 int nr_heads,
725 char **heads,
726 char **pack_lockfile)
728 int i, ret;
729 int fd[2];
730 struct child_process *conn;
731 struct ref *ref;
732 struct stat st;
734 fetch_pack_setup();
735 memcpy(&args, my_args, sizeof(args));
736 if (args.depth > 0) {
737 if (stat(git_path("shallow"), &st))
738 st.st_mtime = 0;
741 conn = git_connect(fd, (char *)dest, args.uploadpack,
742 args.verbose ? CONNECT_VERBOSE : 0);
743 if (heads && nr_heads)
744 nr_heads = remove_duplicates(nr_heads, heads);
745 ref = do_fetch_pack(fd, nr_heads, heads, pack_lockfile);
746 close(fd[0]);
747 close(fd[1]);
748 ret = finish_connect(conn);
750 if (!ret && nr_heads) {
751 /* If the heads to pull were given, we should have
752 * consumed all of them by matching the remote.
753 * Otherwise, 'git-fetch remote no-such-ref' would
754 * silently succeed without issuing an error.
756 for (i = 0; i < nr_heads; i++)
757 if (heads[i] && heads[i][0]) {
758 error("no such remote ref %s", heads[i]);
759 ret = 1;
763 if (!ret && args.depth > 0) {
764 struct cache_time mtime;
765 char *shallow = git_path("shallow");
766 int fd;
768 mtime.sec = st.st_mtime;
769 #ifdef USE_NSEC
770 mtime.usec = st.st_mtim.usec;
771 #endif
772 if (stat(shallow, &st)) {
773 if (mtime.sec)
774 die("shallow file was removed during fetch");
775 } else if (st.st_mtime != mtime.sec
776 #ifdef USE_NSEC
777 || st.st_mtim.usec != mtime.usec
778 #endif
780 die("shallow file was changed during fetch");
782 fd = hold_lock_file_for_update(&lock, shallow, 1);
783 if (!write_shallow_commits(fd, 0)) {
784 unlink(shallow);
785 rollback_lock_file(&lock);
786 } else {
787 close(fd);
788 commit_lock_file(&lock);
792 if (ret)
793 ref = NULL;
795 return ref;