Add zlib
[git/pclouds.git] / builtin-fetch-pack.c
blob8f0e456f97080313e522fe6de6567fbf11733f38
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
10 static int keep_pack;
11 static int transfer_unpack_limit = -1;
12 static int fetch_unpack_limit = -1;
13 static int unpack_limit = 100;
14 static int quiet;
15 static int verbose;
16 static int fetch_all;
17 static int depth;
18 static int no_progress;
19 static const char fetch_pack_usage[] =
20 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
21 static const char *uploadpack = "git-upload-pack";
23 #define COMPLETE (1U << 0)
24 #define COMMON (1U << 1)
25 #define COMMON_REF (1U << 2)
26 #define SEEN (1U << 3)
27 #define POPPED (1U << 4)
30 * After sending this many "have"s if we do not get any new ACK , we
31 * give up traversing our history.
33 #define MAX_IN_VAIN 256
35 static struct commit_list *rev_list;
36 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
38 static void rev_list_push(struct commit *commit, int mark)
40 if (!(commit->object.flags & mark)) {
41 commit->object.flags |= mark;
43 if (!(commit->object.parsed))
44 parse_commit(commit);
46 insert_by_date(commit, &rev_list);
48 if (!(commit->object.flags & COMMON))
49 non_common_revs++;
53 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
55 struct object *o = deref_tag(parse_object(sha1), path, 0);
57 if (o && o->type == OBJ_COMMIT)
58 rev_list_push((struct commit *)o, SEEN);
60 return 0;
64 This function marks a rev and its ancestors as common.
65 In some cases, it is desirable to mark only the ancestors (for example
66 when only the server does not yet know that they are common).
69 static void mark_common(struct commit *commit,
70 int ancestors_only, int dont_parse)
72 if (commit != NULL && !(commit->object.flags & COMMON)) {
73 struct object *o = (struct object *)commit;
75 if (!ancestors_only)
76 o->flags |= COMMON;
78 if (!(o->flags & SEEN))
79 rev_list_push(commit, SEEN);
80 else {
81 struct commit_list *parents;
83 if (!ancestors_only && !(o->flags & POPPED))
84 non_common_revs--;
85 if (!o->parsed && !dont_parse)
86 parse_commit(commit);
88 for (parents = commit->parents;
89 parents;
90 parents = parents->next)
91 mark_common(parents->item, 0, dont_parse);
97 Get the next rev to send, ignoring the common.
100 static const unsigned char* get_rev(void)
102 struct commit *commit = NULL;
104 while (commit == NULL) {
105 unsigned int mark;
106 struct commit_list* parents;
108 if (rev_list == NULL || non_common_revs == 0)
109 return NULL;
111 commit = rev_list->item;
112 if (!(commit->object.parsed))
113 parse_commit(commit);
114 commit->object.flags |= POPPED;
115 if (!(commit->object.flags & COMMON))
116 non_common_revs--;
118 parents = commit->parents;
120 if (commit->object.flags & COMMON) {
121 /* do not send "have", and ignore ancestors */
122 commit = NULL;
123 mark = COMMON | SEEN;
124 } else if (commit->object.flags & COMMON_REF)
125 /* send "have", and ignore ancestors */
126 mark = COMMON | SEEN;
127 else
128 /* send "have", also for its ancestors */
129 mark = SEEN;
131 while (parents) {
132 if (!(parents->item->object.flags & SEEN))
133 rev_list_push(parents->item, mark);
134 if (mark & COMMON)
135 mark_common(parents->item, 1, 0);
136 parents = parents->next;
139 rev_list = rev_list->next;
142 return commit->object.sha1;
145 static int find_common(int fd[2], unsigned char *result_sha1,
146 struct ref *refs)
148 int fetching;
149 int count = 0, flushes = 0, retval;
150 const unsigned char *sha1;
151 unsigned in_vain = 0;
152 int got_continue = 0;
154 for_each_ref(rev_list_insert_ref, NULL);
156 fetching = 0;
157 for ( ; refs ; refs = refs->next) {
158 unsigned char *remote = refs->old_sha1;
159 struct object *o;
162 * If that object is complete (i.e. it is an ancestor of a
163 * local ref), we tell them we have it but do not have to
164 * tell them about its ancestors, which they already know
165 * about.
167 * We use lookup_object here because we are only
168 * interested in the case we *know* the object is
169 * reachable and we have already scanned it.
171 if (((o = lookup_object(remote)) != NULL) &&
172 (o->flags & COMPLETE)) {
173 continue;
176 if (!fetching)
177 packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
178 sha1_to_hex(remote),
179 (multi_ack ? " multi_ack" : ""),
180 (use_sideband == 2 ? " side-band-64k" : ""),
181 (use_sideband == 1 ? " side-band" : ""),
182 (use_thin_pack ? " thin-pack" : ""),
183 (no_progress ? " no-progress" : ""),
184 " ofs-delta");
185 else
186 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
187 fetching++;
189 if (is_repository_shallow())
190 write_shallow_commits(fd[1], 1);
191 if (depth > 0)
192 packet_write(fd[1], "deepen %d", depth);
193 packet_flush(fd[1]);
194 if (!fetching)
195 return 1;
197 if (depth > 0) {
198 char line[1024];
199 unsigned char sha1[20];
200 int len;
202 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
203 if (!prefixcmp(line, "shallow ")) {
204 if (get_sha1_hex(line + 8, sha1))
205 die("invalid shallow line: %s", line);
206 register_shallow(sha1);
207 continue;
209 if (!prefixcmp(line, "unshallow ")) {
210 if (get_sha1_hex(line + 10, sha1))
211 die("invalid unshallow line: %s", line);
212 if (!lookup_object(sha1))
213 die("object not found: %s", line);
214 /* make sure that it is parsed as shallow */
215 parse_object(sha1);
216 if (unregister_shallow(sha1))
217 die("no shallow found: %s", line);
218 continue;
220 die("expected shallow/unshallow, got %s", line);
224 flushes = 0;
225 retval = -1;
226 while ((sha1 = get_rev())) {
227 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
228 if (verbose)
229 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
230 in_vain++;
231 if (!(31 & ++count)) {
232 int ack;
234 packet_flush(fd[1]);
235 flushes++;
238 * We keep one window "ahead" of the other side, and
239 * will wait for an ACK only on the next one
241 if (count == 32)
242 continue;
244 do {
245 ack = get_ack(fd[0], result_sha1);
246 if (verbose && ack)
247 fprintf(stderr, "got ack %d %s\n", ack,
248 sha1_to_hex(result_sha1));
249 if (ack == 1) {
250 flushes = 0;
251 multi_ack = 0;
252 retval = 0;
253 goto done;
254 } else if (ack == 2) {
255 struct commit *commit =
256 lookup_commit(result_sha1);
257 mark_common(commit, 0, 1);
258 retval = 0;
259 in_vain = 0;
260 got_continue = 1;
262 } while (ack);
263 flushes--;
264 if (got_continue && MAX_IN_VAIN < in_vain) {
265 if (verbose)
266 fprintf(stderr, "giving up\n");
267 break; /* give up */
271 done:
272 packet_write(fd[1], "done\n");
273 if (verbose)
274 fprintf(stderr, "done\n");
275 if (retval != 0) {
276 multi_ack = 0;
277 flushes++;
279 while (flushes || multi_ack) {
280 int ack = get_ack(fd[0], result_sha1);
281 if (ack) {
282 if (verbose)
283 fprintf(stderr, "got ack (%d) %s\n", ack,
284 sha1_to_hex(result_sha1));
285 if (ack == 1)
286 return 0;
287 multi_ack = 1;
288 continue;
290 flushes--;
292 return retval;
295 static struct commit_list *complete;
297 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
299 struct object *o = parse_object(sha1);
301 while (o && o->type == OBJ_TAG) {
302 struct tag *t = (struct tag *) o;
303 if (!t->tagged)
304 break; /* broken repository */
305 o->flags |= COMPLETE;
306 o = parse_object(t->tagged->sha1);
308 if (o && o->type == OBJ_COMMIT) {
309 struct commit *commit = (struct commit *)o;
310 commit->object.flags |= COMPLETE;
311 insert_by_date(commit, &complete);
313 return 0;
316 static void mark_recent_complete_commits(unsigned long cutoff)
318 while (complete && cutoff <= complete->item->date) {
319 if (verbose)
320 fprintf(stderr, "Marking %s as complete\n",
321 sha1_to_hex(complete->item->object.sha1));
322 pop_most_recent_commit(&complete, COMPLETE);
326 static void filter_refs(struct ref **refs, int nr_match, char **match)
328 struct ref **return_refs;
329 struct ref *newlist = NULL;
330 struct ref **newtail = &newlist;
331 struct ref *ref, *next;
332 struct ref *fastarray[32];
334 if (nr_match && !fetch_all) {
335 if (ARRAY_SIZE(fastarray) < nr_match)
336 return_refs = xcalloc(nr_match, sizeof(struct ref *));
337 else {
338 return_refs = fastarray;
339 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
342 else
343 return_refs = NULL;
345 for (ref = *refs; ref; ref = next) {
346 next = ref->next;
347 if (!memcmp(ref->name, "refs/", 5) &&
348 check_ref_format(ref->name + 5))
349 ; /* trash */
350 else if (fetch_all &&
351 (!depth || prefixcmp(ref->name, "refs/tags/") )) {
352 *newtail = ref;
353 ref->next = NULL;
354 newtail = &ref->next;
355 continue;
357 else {
358 int order = path_match(ref->name, nr_match, match);
359 if (order) {
360 return_refs[order-1] = ref;
361 continue; /* we will link it later */
364 free(ref);
367 if (!fetch_all) {
368 int i;
369 for (i = 0; i < nr_match; i++) {
370 ref = return_refs[i];
371 if (ref) {
372 *newtail = ref;
373 ref->next = NULL;
374 newtail = &ref->next;
377 if (return_refs != fastarray)
378 free(return_refs);
380 *refs = newlist;
383 static int everything_local(struct ref **refs, int nr_match, char **match)
385 struct ref *ref;
386 int retval;
387 unsigned long cutoff = 0;
389 track_object_refs = 0;
390 save_commit_buffer = 0;
392 for (ref = *refs; ref; ref = ref->next) {
393 struct object *o;
395 o = parse_object(ref->old_sha1);
396 if (!o)
397 continue;
399 /* We already have it -- which may mean that we were
400 * in sync with the other side at some time after
401 * that (it is OK if we guess wrong here).
403 if (o->type == OBJ_COMMIT) {
404 struct commit *commit = (struct commit *)o;
405 if (!cutoff || cutoff < commit->date)
406 cutoff = commit->date;
410 if (!depth) {
411 for_each_ref(mark_complete, NULL);
412 if (cutoff)
413 mark_recent_complete_commits(cutoff);
417 * Mark all complete remote refs as common refs.
418 * Don't mark them common yet; the server has to be told so first.
420 for (ref = *refs; ref; ref = ref->next) {
421 struct object *o = deref_tag(lookup_object(ref->old_sha1),
422 NULL, 0);
424 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
425 continue;
427 if (!(o->flags & SEEN)) {
428 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
430 mark_common((struct commit *)o, 1, 1);
434 filter_refs(refs, nr_match, match);
436 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
437 const unsigned char *remote = ref->old_sha1;
438 unsigned char local[20];
439 struct object *o;
441 o = lookup_object(remote);
442 if (!o || !(o->flags & COMPLETE)) {
443 retval = 0;
444 if (!verbose)
445 continue;
446 fprintf(stderr,
447 "want %s (%s)\n", sha1_to_hex(remote),
448 ref->name);
449 continue;
452 hashcpy(ref->new_sha1, local);
453 if (!verbose)
454 continue;
455 fprintf(stderr,
456 "already have %s (%s)\n", sha1_to_hex(remote),
457 ref->name);
459 return retval;
462 static pid_t setup_sideband(int fd[2], int xd[2])
464 pid_t side_pid;
466 if (!use_sideband) {
467 fd[0] = xd[0];
468 fd[1] = xd[1];
469 return 0;
471 /* xd[] is talking with upload-pack; subprocess reads from
472 * xd[0], spits out band#2 to stderr, and feeds us band#1
473 * through our fd[0].
475 if (pipe(fd) < 0)
476 die("fetch-pack: unable to set up pipe");
477 side_pid = fork();
478 if (side_pid < 0)
479 die("fetch-pack: unable to fork off sideband demultiplexer");
480 if (!side_pid) {
481 /* subprocess */
482 close(fd[0]);
483 if (xd[0] != xd[1])
484 close(xd[1]);
485 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
486 exit(1);
487 exit(0);
489 close(xd[0]);
490 close(fd[1]);
491 fd[1] = xd[1];
492 return side_pid;
495 static int get_pack(int xd[2])
497 int status;
498 pid_t pid, side_pid;
499 int fd[2];
500 const char *argv[20];
501 char keep_arg[256];
502 char hdr_arg[256];
503 const char **av;
504 int do_keep = keep_pack;
506 side_pid = setup_sideband(fd, xd);
508 av = argv;
509 *hdr_arg = 0;
510 if (unpack_limit) {
511 struct pack_header header;
513 if (read_pack_header(fd[0], &header))
514 die("protocol error: bad pack header");
515 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
516 ntohl(header.hdr_version), ntohl(header.hdr_entries));
517 if (ntohl(header.hdr_entries) < unpack_limit)
518 do_keep = 0;
519 else
520 do_keep = 1;
523 if (do_keep) {
524 *av++ = "index-pack";
525 *av++ = "--stdin";
526 if (!quiet && !no_progress)
527 *av++ = "-v";
528 if (use_thin_pack)
529 *av++ = "--fix-thin";
530 if (keep_pack > 1 || unpack_limit) {
531 int s = sprintf(keep_arg,
532 "--keep=fetch-pack %d on ", getpid());
533 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
534 strcpy(keep_arg + s, "localhost");
535 *av++ = keep_arg;
538 else {
539 *av++ = "unpack-objects";
540 if (quiet)
541 *av++ = "-q";
543 if (*hdr_arg)
544 *av++ = hdr_arg;
545 *av++ = NULL;
547 pid = spawnv_git_cmd(argv, fd, NULL);
548 if (pid < 0)
549 die("fetch-pack: unable to fork off %s", argv[0]);
550 close(fd[1]);
551 while (waitpid(pid, &status, 0) < 0) {
552 if (errno != EINTR)
553 die("waiting for %s: %s", argv[0], strerror(errno));
555 if (WIFEXITED(status)) {
556 int code = WEXITSTATUS(status);
557 if (code)
558 die("%s died with error code %d", argv[0], code);
559 return 0;
561 if (WIFSIGNALED(status)) {
562 int sig = WTERMSIG(status);
563 die("%s died of signal %d", argv[0], sig);
565 die("%s died of unnatural causes %d", argv[0], status);
568 static int fetch_pack(int fd[2], int nr_match, char **match)
570 struct ref *ref;
571 unsigned char sha1[20];
573 get_remote_heads(fd[0], &ref, 0, NULL, 0);
574 if (is_repository_shallow() && !server_supports("shallow"))
575 die("Server does not support shallow clients");
576 if (server_supports("multi_ack")) {
577 if (verbose)
578 fprintf(stderr, "Server supports multi_ack\n");
579 multi_ack = 1;
581 #ifndef __MINGW32__
582 if (server_supports("side-band-64k")) {
583 if (verbose)
584 fprintf(stderr, "Server supports side-band-64k\n");
585 use_sideband = 2;
587 else if (server_supports("side-band")) {
588 if (verbose)
589 fprintf(stderr, "Server supports side-band\n");
590 use_sideband = 1;
592 #endif
593 if (!ref) {
594 packet_flush(fd[1]);
595 die("no matching remote head");
597 if (everything_local(&ref, nr_match, match)) {
598 packet_flush(fd[1]);
599 goto all_done;
601 if (find_common(fd, sha1, ref) < 0)
602 if (keep_pack != 1)
603 /* When cloning, it is not unusual to have
604 * no common commit.
606 fprintf(stderr, "warning: no common commits\n");
608 if (get_pack(fd))
609 die("git-fetch-pack: fetch failed.");
611 all_done:
612 while (ref) {
613 printf("%s %s\n",
614 sha1_to_hex(ref->old_sha1), ref->name);
615 ref = ref->next;
617 return 0;
620 static int remove_duplicates(int nr_heads, char **heads)
622 int src, dst;
624 for (src = dst = 0; src < nr_heads; src++) {
625 /* If heads[src] is different from any of
626 * heads[0..dst], push it in.
628 int i;
629 for (i = 0; i < dst; i++) {
630 if (!strcmp(heads[i], heads[src]))
631 break;
633 if (i < dst)
634 continue;
635 if (src != dst)
636 heads[dst] = heads[src];
637 dst++;
639 heads[dst] = 0;
640 return dst;
643 static int fetch_pack_config(const char *var, const char *value)
645 if (strcmp(var, "fetch.unpacklimit") == 0) {
646 fetch_unpack_limit = git_config_int(var, value);
647 return 0;
650 if (strcmp(var, "transfer.unpacklimit") == 0) {
651 transfer_unpack_limit = git_config_int(var, value);
652 return 0;
655 return git_default_config(var, value);
658 static struct lock_file lock;
660 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
662 int i, ret, nr_heads;
663 char *dest = NULL, **heads;
664 int fd[2];
665 pid_t pid;
666 struct stat st;
668 git_config(fetch_pack_config);
670 if (0 <= transfer_unpack_limit)
671 unpack_limit = transfer_unpack_limit;
672 else if (0 <= fetch_unpack_limit)
673 unpack_limit = fetch_unpack_limit;
675 nr_heads = 0;
676 heads = NULL;
677 for (i = 1; i < argc; i++) {
678 char *arg = argv[i];
680 if (*arg == '-') {
681 if (!prefixcmp(arg, "--upload-pack=")) {
682 uploadpack = arg + 14;
683 continue;
685 if (!prefixcmp(arg, "--exec=")) {
686 uploadpack = arg + 7;
687 continue;
689 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
690 quiet = 1;
691 continue;
693 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
694 keep_pack++;
695 unpack_limit = 0;
696 continue;
698 if (!strcmp("--thin", arg)) {
699 use_thin_pack = 1;
700 continue;
702 if (!strcmp("--all", arg)) {
703 fetch_all = 1;
704 continue;
706 if (!strcmp("-v", arg)) {
707 verbose = 1;
708 continue;
710 if (!prefixcmp(arg, "--depth=")) {
711 depth = strtol(arg + 8, NULL, 0);
712 if (stat(git_path("shallow"), &st))
713 st.st_mtime = 0;
714 continue;
716 if (!strcmp("--no-progress", arg)) {
717 no_progress = 1;
718 continue;
720 usage(fetch_pack_usage);
722 dest = arg;
723 heads = argv + i + 1;
724 nr_heads = argc - i - 1;
725 break;
727 if (!dest)
728 usage(fetch_pack_usage);
729 pid = git_connect(fd, dest, uploadpack, verbose ? CONNECT_VERBOSE : 0);
730 if (pid < 0)
731 return 1;
732 if (heads && nr_heads)
733 nr_heads = remove_duplicates(nr_heads, heads);
734 ret = fetch_pack(fd, nr_heads, heads);
735 close(fd[0]);
736 close(fd[1]);
737 ret |= finish_connect(pid);
739 if (!ret && nr_heads) {
740 /* If the heads to pull were given, we should have
741 * consumed all of them by matching the remote.
742 * Otherwise, 'git-fetch remote no-such-ref' would
743 * silently succeed without issuing an error.
745 for (i = 0; i < nr_heads; i++)
746 if (heads[i] && heads[i][0]) {
747 error("no such remote ref %s", heads[i]);
748 ret = 1;
752 if (!ret && depth > 0) {
753 struct cache_time mtime;
754 char *shallow = git_path("shallow");
755 int fd;
757 mtime.sec = st.st_mtime;
758 #ifdef USE_NSEC
759 mtime.usec = st.st_mtim.usec;
760 #endif
761 if (stat(shallow, &st)) {
762 if (mtime.sec)
763 die("shallow file was removed during fetch");
764 } else if (st.st_mtime != mtime.sec
765 #ifdef USE_NSEC
766 || st.st_mtim.usec != mtime.usec
767 #endif
769 die("shallow file was changed during fetch");
771 fd = hold_lock_file_for_update(&lock, shallow, 1);
772 if (!write_shallow_commits(fd, 0)) {
773 unlink(shallow);
774 rollback_lock_file(&lock);
775 } else {
776 close(fd);
777 commit_lock_file(&lock);
781 return !!ret;