unpack-objects: prevent writing of inconsistent objects
[tgit.git] / builtin-fetch-pack.c
blob29b38e4650cbd032ecfae02b334ce2b62ea1e622
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9 #include "fetch-pack.h"
10 #include "remote.h"
11 #include "run-command.h"
13 static int transfer_unpack_limit = -1;
14 static int fetch_unpack_limit = -1;
15 static int unpack_limit = 100;
16 static struct fetch_pack_args args = {
17 /* .uploadpack = */ "git-upload-pack",
20 static const char fetch_pack_usage[] =
21 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
23 #define COMPLETE (1U << 0)
24 #define COMMON (1U << 1)
25 #define COMMON_REF (1U << 2)
26 #define SEEN (1U << 3)
27 #define POPPED (1U << 4)
30 * After sending this many "have"s if we do not get any new ACK , we
31 * give up traversing our history.
33 #define MAX_IN_VAIN 256
35 static struct commit_list *rev_list;
36 static int non_common_revs, multi_ack, use_sideband;
38 static void rev_list_push(struct commit *commit, int mark)
40 if (!(commit->object.flags & mark)) {
41 commit->object.flags |= mark;
43 if (!(commit->object.parsed))
44 if (parse_commit(commit))
45 return;
47 insert_by_date(commit, &rev_list);
49 if (!(commit->object.flags & COMMON))
50 non_common_revs++;
54 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
56 struct object *o = deref_tag(parse_object(sha1), path, 0);
58 if (o && o->type == OBJ_COMMIT)
59 rev_list_push((struct commit *)o, SEEN);
61 return 0;
65 This function marks a rev and its ancestors as common.
66 In some cases, it is desirable to mark only the ancestors (for example
67 when only the server does not yet know that they are common).
70 static void mark_common(struct commit *commit,
71 int ancestors_only, int dont_parse)
73 if (commit != NULL && !(commit->object.flags & COMMON)) {
74 struct object *o = (struct object *)commit;
76 if (!ancestors_only)
77 o->flags |= COMMON;
79 if (!(o->flags & SEEN))
80 rev_list_push(commit, SEEN);
81 else {
82 struct commit_list *parents;
84 if (!ancestors_only && !(o->flags & POPPED))
85 non_common_revs--;
86 if (!o->parsed && !dont_parse)
87 if (parse_commit(commit))
88 return;
90 for (parents = commit->parents;
91 parents;
92 parents = parents->next)
93 mark_common(parents->item, 0, dont_parse);
99 Get the next rev to send, ignoring the common.
102 static const unsigned char* get_rev(void)
104 struct commit *commit = NULL;
106 while (commit == NULL) {
107 unsigned int mark;
108 struct commit_list *parents = NULL;
110 if (rev_list == NULL || non_common_revs == 0)
111 return NULL;
113 commit = rev_list->item;
114 if (!(commit->object.parsed))
115 if (!parse_commit(commit))
116 parents = commit->parents;
118 commit->object.flags |= POPPED;
119 if (!(commit->object.flags & COMMON))
120 non_common_revs--;
122 if (commit->object.flags & COMMON) {
123 /* do not send "have", and ignore ancestors */
124 commit = NULL;
125 mark = COMMON | SEEN;
126 } else if (commit->object.flags & COMMON_REF)
127 /* send "have", and ignore ancestors */
128 mark = COMMON | SEEN;
129 else
130 /* send "have", also for its ancestors */
131 mark = SEEN;
133 while (parents) {
134 if (!(parents->item->object.flags & SEEN))
135 rev_list_push(parents->item, mark);
136 if (mark & COMMON)
137 mark_common(parents->item, 1, 0);
138 parents = parents->next;
141 rev_list = rev_list->next;
144 return commit->object.sha1;
147 static int find_common(int fd[2], unsigned char *result_sha1,
148 struct ref *refs)
150 int fetching;
151 int count = 0, flushes = 0, retval;
152 const unsigned char *sha1;
153 unsigned in_vain = 0;
154 int got_continue = 0;
156 for_each_ref(rev_list_insert_ref, NULL);
158 fetching = 0;
159 for ( ; refs ; refs = refs->next) {
160 unsigned char *remote = refs->old_sha1;
161 struct object *o;
164 * If that object is complete (i.e. it is an ancestor of a
165 * local ref), we tell them we have it but do not have to
166 * tell them about its ancestors, which they already know
167 * about.
169 * We use lookup_object here because we are only
170 * interested in the case we *know* the object is
171 * reachable and we have already scanned it.
173 if (((o = lookup_object(remote)) != NULL) &&
174 (o->flags & COMPLETE)) {
175 continue;
178 if (!fetching)
179 packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
180 sha1_to_hex(remote),
181 (multi_ack ? " multi_ack" : ""),
182 (use_sideband == 2 ? " side-band-64k" : ""),
183 (use_sideband == 1 ? " side-band" : ""),
184 (args.use_thin_pack ? " thin-pack" : ""),
185 (args.no_progress ? " no-progress" : ""),
186 " ofs-delta");
187 else
188 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
189 fetching++;
191 if (is_repository_shallow())
192 write_shallow_commits(fd[1], 1);
193 if (args.depth > 0)
194 packet_write(fd[1], "deepen %d", args.depth);
195 packet_flush(fd[1]);
196 if (!fetching)
197 return 1;
199 if (args.depth > 0) {
200 char line[1024];
201 unsigned char sha1[20];
202 int len;
204 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
205 if (!prefixcmp(line, "shallow ")) {
206 if (get_sha1_hex(line + 8, sha1))
207 die("invalid shallow line: %s", line);
208 register_shallow(sha1);
209 continue;
211 if (!prefixcmp(line, "unshallow ")) {
212 if (get_sha1_hex(line + 10, sha1))
213 die("invalid unshallow line: %s", line);
214 if (!lookup_object(sha1))
215 die("object not found: %s", line);
216 /* make sure that it is parsed as shallow */
217 if (!parse_object(sha1))
218 die("error in object: %s", line);
219 if (unregister_shallow(sha1))
220 die("no shallow found: %s", line);
221 continue;
223 die("expected shallow/unshallow, got %s", line);
227 flushes = 0;
228 retval = -1;
229 while ((sha1 = get_rev())) {
230 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
231 if (args.verbose)
232 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
233 in_vain++;
234 if (!(31 & ++count)) {
235 int ack;
237 packet_flush(fd[1]);
238 flushes++;
241 * We keep one window "ahead" of the other side, and
242 * will wait for an ACK only on the next one
244 if (count == 32)
245 continue;
247 do {
248 ack = get_ack(fd[0], result_sha1);
249 if (args.verbose && ack)
250 fprintf(stderr, "got ack %d %s\n", ack,
251 sha1_to_hex(result_sha1));
252 if (ack == 1) {
253 flushes = 0;
254 multi_ack = 0;
255 retval = 0;
256 goto done;
257 } else if (ack == 2) {
258 struct commit *commit =
259 lookup_commit(result_sha1);
260 mark_common(commit, 0, 1);
261 retval = 0;
262 in_vain = 0;
263 got_continue = 1;
265 } while (ack);
266 flushes--;
267 if (got_continue && MAX_IN_VAIN < in_vain) {
268 if (args.verbose)
269 fprintf(stderr, "giving up\n");
270 break; /* give up */
274 done:
275 packet_write(fd[1], "done\n");
276 if (args.verbose)
277 fprintf(stderr, "done\n");
278 if (retval != 0) {
279 multi_ack = 0;
280 flushes++;
282 while (flushes || multi_ack) {
283 int ack = get_ack(fd[0], result_sha1);
284 if (ack) {
285 if (args.verbose)
286 fprintf(stderr, "got ack (%d) %s\n", ack,
287 sha1_to_hex(result_sha1));
288 if (ack == 1)
289 return 0;
290 multi_ack = 1;
291 continue;
293 flushes--;
295 return retval;
298 static struct commit_list *complete;
300 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
302 struct object *o = parse_object(sha1);
304 while (o && o->type == OBJ_TAG) {
305 struct tag *t = (struct tag *) o;
306 if (!t->tagged)
307 break; /* broken repository */
308 o->flags |= COMPLETE;
309 o = parse_object(t->tagged->sha1);
311 if (o && o->type == OBJ_COMMIT) {
312 struct commit *commit = (struct commit *)o;
313 commit->object.flags |= COMPLETE;
314 insert_by_date(commit, &complete);
316 return 0;
319 static void mark_recent_complete_commits(unsigned long cutoff)
321 while (complete && cutoff <= complete->item->date) {
322 if (args.verbose)
323 fprintf(stderr, "Marking %s as complete\n",
324 sha1_to_hex(complete->item->object.sha1));
325 pop_most_recent_commit(&complete, COMPLETE);
329 static void filter_refs(struct ref **refs, int nr_match, char **match)
331 struct ref **return_refs;
332 struct ref *newlist = NULL;
333 struct ref **newtail = &newlist;
334 struct ref *ref, *next;
335 struct ref *fastarray[32];
337 if (nr_match && !args.fetch_all) {
338 if (ARRAY_SIZE(fastarray) < nr_match)
339 return_refs = xcalloc(nr_match, sizeof(struct ref *));
340 else {
341 return_refs = fastarray;
342 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
345 else
346 return_refs = NULL;
348 for (ref = *refs; ref; ref = next) {
349 next = ref->next;
350 if (!memcmp(ref->name, "refs/", 5) &&
351 check_ref_format(ref->name + 5))
352 ; /* trash */
353 else if (args.fetch_all &&
354 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
355 *newtail = ref;
356 ref->next = NULL;
357 newtail = &ref->next;
358 continue;
360 else {
361 int order = path_match(ref->name, nr_match, match);
362 if (order) {
363 return_refs[order-1] = ref;
364 continue; /* we will link it later */
367 free(ref);
370 if (!args.fetch_all) {
371 int i;
372 for (i = 0; i < nr_match; i++) {
373 ref = return_refs[i];
374 if (ref) {
375 *newtail = ref;
376 ref->next = NULL;
377 newtail = &ref->next;
380 if (return_refs != fastarray)
381 free(return_refs);
383 *refs = newlist;
386 static int everything_local(struct ref **refs, int nr_match, char **match)
388 struct ref *ref;
389 int retval;
390 unsigned long cutoff = 0;
392 save_commit_buffer = 0;
394 for (ref = *refs; ref; ref = ref->next) {
395 struct object *o;
397 o = parse_object(ref->old_sha1);
398 if (!o)
399 continue;
401 /* We already have it -- which may mean that we were
402 * in sync with the other side at some time after
403 * that (it is OK if we guess wrong here).
405 if (o->type == OBJ_COMMIT) {
406 struct commit *commit = (struct commit *)o;
407 if (!cutoff || cutoff < commit->date)
408 cutoff = commit->date;
412 if (!args.depth) {
413 for_each_ref(mark_complete, NULL);
414 if (cutoff)
415 mark_recent_complete_commits(cutoff);
419 * Mark all complete remote refs as common refs.
420 * Don't mark them common yet; the server has to be told so first.
422 for (ref = *refs; ref; ref = ref->next) {
423 struct object *o = deref_tag(lookup_object(ref->old_sha1),
424 NULL, 0);
426 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
427 continue;
429 if (!(o->flags & SEEN)) {
430 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
432 mark_common((struct commit *)o, 1, 1);
436 filter_refs(refs, nr_match, match);
438 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
439 const unsigned char *remote = ref->old_sha1;
440 unsigned char local[20];
441 struct object *o;
443 o = lookup_object(remote);
444 if (!o || !(o->flags & COMPLETE)) {
445 retval = 0;
446 if (!args.verbose)
447 continue;
448 fprintf(stderr,
449 "want %s (%s)\n", sha1_to_hex(remote),
450 ref->name);
451 continue;
454 hashcpy(ref->new_sha1, local);
455 if (!args.verbose)
456 continue;
457 fprintf(stderr,
458 "already have %s (%s)\n", sha1_to_hex(remote),
459 ref->name);
461 return retval;
464 static int sideband_demux(int fd, void *data)
466 int *xd = data;
468 return recv_sideband("fetch-pack", xd[0], fd, 2);
471 static int get_pack(int xd[2], char **pack_lockfile)
473 struct async demux;
474 const char *argv[20];
475 char keep_arg[256];
476 char hdr_arg[256];
477 const char **av;
478 int do_keep = args.keep_pack;
479 struct child_process cmd;
481 memset(&demux, 0, sizeof(demux));
482 if (use_sideband) {
483 /* xd[] is talking with upload-pack; subprocess reads from
484 * xd[0], spits out band#2 to stderr, and feeds us band#1
485 * through demux->out.
487 demux.proc = sideband_demux;
488 demux.data = xd;
489 if (start_async(&demux))
490 die("fetch-pack: unable to fork off sideband"
491 " demultiplexer");
493 else
494 demux.out = xd[0];
496 memset(&cmd, 0, sizeof(cmd));
497 cmd.argv = argv;
498 av = argv;
499 *hdr_arg = 0;
500 if (!args.keep_pack && unpack_limit) {
501 struct pack_header header;
503 if (read_pack_header(demux.out, &header))
504 die("protocol error: bad pack header");
505 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
506 ntohl(header.hdr_version), ntohl(header.hdr_entries));
507 if (ntohl(header.hdr_entries) < unpack_limit)
508 do_keep = 0;
509 else
510 do_keep = 1;
513 if (do_keep) {
514 if (pack_lockfile)
515 cmd.out = -1;
516 *av++ = "index-pack";
517 *av++ = "--stdin";
518 if (!args.quiet && !args.no_progress)
519 *av++ = "-v";
520 if (args.use_thin_pack)
521 *av++ = "--fix-thin";
522 if (args.lock_pack || unpack_limit) {
523 int s = sprintf(keep_arg,
524 "--keep=fetch-pack %d on ", getpid());
525 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
526 strcpy(keep_arg + s, "localhost");
527 *av++ = keep_arg;
530 else {
531 *av++ = "unpack-objects";
532 if (args.quiet)
533 *av++ = "-q";
535 if (*hdr_arg)
536 *av++ = hdr_arg;
537 *av++ = NULL;
539 cmd.in = demux.out;
540 cmd.git_cmd = 1;
541 if (start_command(&cmd))
542 die("fetch-pack: unable to fork off %s", argv[0]);
543 if (do_keep && pack_lockfile) {
544 *pack_lockfile = index_pack_lockfile(cmd.out);
545 close(cmd.out);
548 if (finish_command(&cmd))
549 die("%s failed", argv[0]);
550 if (use_sideband && finish_async(&demux))
551 die("error in sideband demultiplexer");
552 return 0;
555 static struct ref *do_fetch_pack(int fd[2],
556 const struct ref *orig_ref,
557 int nr_match,
558 char **match,
559 char **pack_lockfile)
561 struct ref *ref = copy_ref_list(orig_ref);
562 unsigned char sha1[20];
564 if (is_repository_shallow() && !server_supports("shallow"))
565 die("Server does not support shallow clients");
566 if (server_supports("multi_ack")) {
567 if (args.verbose)
568 fprintf(stderr, "Server supports multi_ack\n");
569 multi_ack = 1;
571 if (server_supports("side-band-64k")) {
572 if (args.verbose)
573 fprintf(stderr, "Server supports side-band-64k\n");
574 use_sideband = 2;
576 else if (server_supports("side-band")) {
577 if (args.verbose)
578 fprintf(stderr, "Server supports side-band\n");
579 use_sideband = 1;
581 if (everything_local(&ref, nr_match, match)) {
582 packet_flush(fd[1]);
583 goto all_done;
585 if (find_common(fd, sha1, ref) < 0)
586 if (!args.keep_pack)
587 /* When cloning, it is not unusual to have
588 * no common commit.
590 fprintf(stderr, "warning: no common commits\n");
592 if (get_pack(fd, pack_lockfile))
593 die("git-fetch-pack: fetch failed.");
595 all_done:
596 return ref;
599 static int remove_duplicates(int nr_heads, char **heads)
601 int src, dst;
603 for (src = dst = 0; src < nr_heads; src++) {
604 /* If heads[src] is different from any of
605 * heads[0..dst], push it in.
607 int i;
608 for (i = 0; i < dst; i++) {
609 if (!strcmp(heads[i], heads[src]))
610 break;
612 if (i < dst)
613 continue;
614 if (src != dst)
615 heads[dst] = heads[src];
616 dst++;
618 return dst;
621 static int fetch_pack_config(const char *var, const char *value)
623 if (strcmp(var, "fetch.unpacklimit") == 0) {
624 fetch_unpack_limit = git_config_int(var, value);
625 return 0;
628 if (strcmp(var, "transfer.unpacklimit") == 0) {
629 transfer_unpack_limit = git_config_int(var, value);
630 return 0;
633 return git_default_config(var, value);
636 static struct lock_file lock;
638 static void fetch_pack_setup(void)
640 static int did_setup;
641 if (did_setup)
642 return;
643 git_config(fetch_pack_config);
644 if (0 <= transfer_unpack_limit)
645 unpack_limit = transfer_unpack_limit;
646 else if (0 <= fetch_unpack_limit)
647 unpack_limit = fetch_unpack_limit;
648 did_setup = 1;
651 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
653 int i, ret, nr_heads;
654 struct ref *ref = NULL;
655 char *dest = NULL, **heads;
656 int fd[2];
657 struct child_process *conn;
659 nr_heads = 0;
660 heads = NULL;
661 for (i = 1; i < argc; i++) {
662 const char *arg = argv[i];
664 if (*arg == '-') {
665 if (!prefixcmp(arg, "--upload-pack=")) {
666 args.uploadpack = arg + 14;
667 continue;
669 if (!prefixcmp(arg, "--exec=")) {
670 args.uploadpack = arg + 7;
671 continue;
673 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
674 args.quiet = 1;
675 continue;
677 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
678 args.lock_pack = args.keep_pack;
679 args.keep_pack = 1;
680 continue;
682 if (!strcmp("--thin", arg)) {
683 args.use_thin_pack = 1;
684 continue;
686 if (!strcmp("--all", arg)) {
687 args.fetch_all = 1;
688 continue;
690 if (!strcmp("-v", arg)) {
691 args.verbose = 1;
692 continue;
694 if (!prefixcmp(arg, "--depth=")) {
695 args.depth = strtol(arg + 8, NULL, 0);
696 continue;
698 if (!strcmp("--no-progress", arg)) {
699 args.no_progress = 1;
700 continue;
702 usage(fetch_pack_usage);
704 dest = (char *)arg;
705 heads = (char **)(argv + i + 1);
706 nr_heads = argc - i - 1;
707 break;
709 if (!dest)
710 usage(fetch_pack_usage);
712 conn = git_connect(fd, (char *)dest, args.uploadpack,
713 args.verbose ? CONNECT_VERBOSE : 0);
714 if (conn) {
715 get_remote_heads(fd[0], &ref, 0, NULL, 0);
717 ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
718 close(fd[0]);
719 close(fd[1]);
720 if (finish_connect(conn))
721 ref = NULL;
722 } else {
723 ref = NULL;
725 ret = !ref;
727 if (!ret && nr_heads) {
728 /* If the heads to pull were given, we should have
729 * consumed all of them by matching the remote.
730 * Otherwise, 'git-fetch remote no-such-ref' would
731 * silently succeed without issuing an error.
733 for (i = 0; i < nr_heads; i++)
734 if (heads[i] && heads[i][0]) {
735 error("no such remote ref %s", heads[i]);
736 ret = 1;
739 while (ref) {
740 printf("%s %s\n",
741 sha1_to_hex(ref->old_sha1), ref->name);
742 ref = ref->next;
745 return ret;
748 struct ref *fetch_pack(struct fetch_pack_args *my_args,
749 int fd[], struct child_process *conn,
750 const struct ref *ref,
751 const char *dest,
752 int nr_heads,
753 char **heads,
754 char **pack_lockfile)
756 struct stat st;
757 struct ref *ref_cpy;
759 fetch_pack_setup();
760 memcpy(&args, my_args, sizeof(args));
761 if (args.depth > 0) {
762 if (stat(git_path("shallow"), &st))
763 st.st_mtime = 0;
766 if (heads && nr_heads)
767 nr_heads = remove_duplicates(nr_heads, heads);
768 if (!ref) {
769 packet_flush(fd[1]);
770 die("no matching remote head");
772 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
774 if (args.depth > 0) {
775 struct cache_time mtime;
776 char *shallow = git_path("shallow");
777 int fd;
779 mtime.sec = st.st_mtime;
780 #ifdef USE_NSEC
781 mtime.usec = st.st_mtim.usec;
782 #endif
783 if (stat(shallow, &st)) {
784 if (mtime.sec)
785 die("shallow file was removed during fetch");
786 } else if (st.st_mtime != mtime.sec
787 #ifdef USE_NSEC
788 || st.st_mtim.usec != mtime.usec
789 #endif
791 die("shallow file was changed during fetch");
793 fd = hold_lock_file_for_update(&lock, shallow, 1);
794 if (!write_shallow_commits(fd, 0)) {
795 unlink(shallow);
796 rollback_lock_file(&lock);
797 } else {
798 commit_lock_file(&lock);
802 return ref_cpy;