fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "sideband.h"
   8 #include <sys/wait.h>
   9
  10 static int keep_pack;
  11 static int quiet;
  12 static int verbose;
  13 static int fetch_all;
  14 static int depth;
  15 static const char fetch_pack_usage[] =
  16 "git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [--depth=<n>] [host:]directory <refs>...";
  17 static const char *exec = "git-upload-pack";
  18
  19 #define COMPLETE        (1U << 0)
  20 #define COMMON          (1U << 1)
  21 #define COMMON_REF      (1U << 2)
  22 #define SEEN            (1U << 3)
  23 #define POPPED          (1U << 4)
  24
  25 /*
  26  * After sending this many "have"s if we do not get any new ACK , we
  27  * give up traversing our history.
  28  */
  29 #define MAX_IN_VAIN 256
  30
  31 static struct commit_list *rev_list;
  32 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
  33
  34 static void rev_list_push(struct commit *commit, int mark)
  35 {
  36         if (!(commit->object.flags & mark)) {
  37                 commit->object.flags |= mark;
  38
  39                 if (!(commit->object.parsed))
  40                         parse_commit(commit);
  41
  42                 insert_by_date(commit, &rev_list);
  43
  44                 if (!(commit->object.flags & COMMON))
  45                         non_common_revs++;
  46         }
  47 }
  48
  49 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  50 {
  51         struct object *o = deref_tag(parse_object(sha1), path, 0);
  52
  53         if (o && o->type == OBJ_COMMIT)
  54                 rev_list_push((struct commit *)o, SEEN);
  55
  56         return 0;
  57 }
  58
  59 /*
  60    This function marks a rev and its ancestors as common.
  61    In some cases, it is desirable to mark only the ancestors (for example
  62    when only the server does not yet know that they are common).
  63 */
  64
  65 static void mark_common(struct commit *commit,
  66                 int ancestors_only, int dont_parse)
  67 {
  68         if (commit != NULL && !(commit->object.flags & COMMON)) {
  69                 struct object *o = (struct object *)commit;
  70
  71                 if (!ancestors_only)
  72                         o->flags |= COMMON;
  73
  74                 if (!(o->flags & SEEN))
  75                         rev_list_push(commit, SEEN);
  76                 else {
  77                         struct commit_list *parents;
  78
  79                         if (!ancestors_only && !(o->flags & POPPED))
  80                                 non_common_revs--;
  81                         if (!o->parsed && !dont_parse)
  82                                 parse_commit(commit);
  83
  84                         for (parents = commit->parents;
  85                                         parents;
  86                                         parents = parents->next)
  87                                 mark_common(parents->item, 0, dont_parse);
  88                 }
  89         }
  90 }
  91
  92 /*
  93   Get the next rev to send, ignoring the common.
  94 */
  95
  96 static const unsigned char* get_rev(void)
  97 {
  98         struct commit *commit = NULL;
  99
 100         while (commit == NULL) {
 101                 unsigned int mark;
 102                 struct commit_list* parents;
 103
 104                 if (rev_list == NULL || non_common_revs == 0)
 105                         return NULL;
 106
 107                 commit = rev_list->item;
 108                 if (!(commit->object.parsed))
 109                         parse_commit(commit);
 110                 commit->object.flags |= POPPED;
 111                 if (!(commit->object.flags & COMMON))
 112                         non_common_revs--;
 113
 114                 parents = commit->parents;
 115
 116                 if (commit->object.flags & COMMON) {
 117                         /* do not send "have", and ignore ancestors */
 118                         commit = NULL;
 119                         mark = COMMON | SEEN;
 120                 } else if (commit->object.flags & COMMON_REF)
 121                         /* send "have", and ignore ancestors */
 122                         mark = COMMON | SEEN;
 123                 else
 124                         /* send "have", also for its ancestors */
 125                         mark = SEEN;
 126
 127                 while (parents) {
 128                         if (!(parents->item->object.flags & SEEN))
 129                                 rev_list_push(parents->item, mark);
 130                         if (mark & COMMON)
 131                                 mark_common(parents->item, 1, 0);
 132                         parents = parents->next;
 133                 }
 134
 135                 rev_list = rev_list->next;
 136         }
 137
 138         return commit->object.sha1;
 139 }
 140
 141 static int find_common(int fd[2], unsigned char *result_sha1,
 142                        struct ref *refs)
 143 {
 144         int fetching;
 145         int count = 0, flushes = 0, retval;
 146         const unsigned char *sha1;
 147         unsigned in_vain = 0;
 148         int got_continue = 0;
 149
 150         for_each_ref(rev_list_insert_ref, NULL);
 151
 152         fetching = 0;
 153         for ( ; refs ; refs = refs->next) {
 154                 unsigned char *remote = refs->old_sha1;
 155                 struct object *o;
 156
 157                 /*
 158                  * If that object is complete (i.e. it is an ancestor of a
 159                  * local ref), we tell them we have it but do not have to
 160                  * tell them about its ancestors, which they already know
 161                  * about.
 162                  *
 163                  * We use lookup_object here because we are only
 164                  * interested in the case we *know* the object is
 165                  * reachable and we have already scanned it.
 166                  */
 167                 if (((o = lookup_object(remote)) != NULL) &&
 168                                 (o->flags & COMPLETE)) {
 169                         continue;
 170                 }
 171
 172                 if (!fetching)
 173                         packet_write(fd[1], "want %s%s%s%s%s%s\n",
 174                                      sha1_to_hex(remote),
 175                                      (multi_ack ? " multi_ack" : ""),
 176                                      (use_sideband == 2 ? " side-band-64k" : ""),
 177                                      (use_sideband == 1 ? " side-band" : ""),
 178                                      (use_thin_pack ? " thin-pack" : ""),
 179                                      " ofs-delta");
 180                 else
 181                         packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
 182                 fetching++;
 183         }
 184         if (is_repository_shallow())
 185                 write_shallow_commits(fd[1], 1);
 186         if (depth > 0)
 187                 packet_write(fd[1], "deepen %d", depth);
 188         packet_flush(fd[1]);
 189         if (!fetching)
 190                 return 1;
 191
 192         if (depth > 0) {
 193                 char line[1024];
 194                 unsigned char sha1[20];
 195                 int len;
 196
 197                 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
 198                         if (!strncmp("shallow ", line, 8)) {
 199                                 if (get_sha1_hex(line + 8, sha1))
 200                                         die("invalid shallow line: %s", line);
 201                                 register_shallow(sha1);
 202                                 continue;
 203                         }
 204                         if (!strncmp("unshallow ", line, 10)) {
 205                                 if (get_sha1_hex(line + 10, sha1))
 206                                         die("invalid unshallow line: %s", line);
 207                                 if (!lookup_object(sha1))
 208                                         die("object not found: %s", line);
 209                                 /* make sure that it is parsed as shallow */
 210                                 parse_object(sha1);
 211                                 if (unregister_shallow(sha1))
 212                                         die("no shallow found: %s", line);
 213                                 continue;
 214                         }
 215                         die("expected shallow/unshallow, got %s", line);
 216                 }
 217         }
 218
 219         flushes = 0;
 220         retval = -1;
 221         while ((sha1 = get_rev())) {
 222                 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
 223                 if (verbose)
 224                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 225                 in_vain++;
 226                 if (!(31 & ++count)) {
 227                         int ack;
 228
 229                         packet_flush(fd[1]);
 230                         flushes++;
 231
 232                         /*
 233                          * We keep one window "ahead" of the other side, and
 234                          * will wait for an ACK only on the next one
 235                          */
 236                         if (count == 32)
 237                                 continue;
 238
 239                         do {
 240                                 ack = get_ack(fd[0], result_sha1);
 241                                 if (verbose && ack)
 242                                         fprintf(stderr, "got ack %d %s\n", ack,
 243                                                         sha1_to_hex(result_sha1));
 244                                 if (ack == 1) {
 245                                         flushes = 0;
 246                                         multi_ack = 0;
 247                                         retval = 0;
 248                                         goto done;
 249                                 } else if (ack == 2) {
 250                                         struct commit *commit =
 251                                                 lookup_commit(result_sha1);
 252                                         mark_common(commit, 0, 1);
 253                                         retval = 0;
 254                                         in_vain = 0;
 255                                         got_continue = 1;
 256                                 }
 257                         } while (ack);
 258                         flushes--;
 259                         if (got_continue && MAX_IN_VAIN < in_vain) {
 260                                 if (verbose)
 261                                         fprintf(stderr, "giving up\n");
 262                                 break; /* give up */
 263                         }
 264                 }
 265         }
 266 done:
 267         packet_write(fd[1], "done\n");
 268         if (verbose)
 269                 fprintf(stderr, "done\n");
 270         if (retval != 0) {
 271                 multi_ack = 0;
 272                 flushes++;
 273         }
 274         while (flushes || multi_ack) {
 275                 int ack = get_ack(fd[0], result_sha1);
 276                 if (ack) {
 277                         if (verbose)
 278                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 279                                         sha1_to_hex(result_sha1));
 280                         if (ack == 1)
 281                                 return 0;
 282                         multi_ack = 1;
 283                         continue;
 284                 }
 285                 flushes--;
 286         }
 287         return retval;
 288 }
 289
 290 static struct commit_list *complete;
 291
 292 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 293 {
 294         struct object *o = parse_object(sha1);
 295
 296         while (o && o->type == OBJ_TAG) {
 297                 struct tag *t = (struct tag *) o;
 298                 if (!t->tagged)
 299                         break; /* broken repository */
 300                 o->flags |= COMPLETE;
 301                 o = parse_object(t->tagged->sha1);
 302         }
 303         if (o && o->type == OBJ_COMMIT) {
 304                 struct commit *commit = (struct commit *)o;
 305                 commit->object.flags |= COMPLETE;
 306                 insert_by_date(commit, &complete);
 307         }
 308         return 0;
 309 }
 310
 311 static void mark_recent_complete_commits(unsigned long cutoff)
 312 {
 313         while (complete && cutoff <= complete->item->date) {
 314                 if (verbose)
 315                         fprintf(stderr, "Marking %s as complete\n",
 316                                 sha1_to_hex(complete->item->object.sha1));
 317                 pop_most_recent_commit(&complete, COMPLETE);
 318         }
 319 }
 320
 321 static void filter_refs(struct ref **refs, int nr_match, char **match)
 322 {
 323         struct ref **return_refs;
 324         struct ref *newlist = NULL;
 325         struct ref **newtail = &newlist;
 326         struct ref *ref, *next;
 327         struct ref *fastarray[32];
 328
 329         if (nr_match && !fetch_all) {
 330                 if (ARRAY_SIZE(fastarray) < nr_match)
 331                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 332                 else {
 333                         return_refs = fastarray;
 334                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 335                 }
 336         }
 337         else
 338                 return_refs = NULL;
 339
 340         for (ref = *refs; ref; ref = next) {
 341                 next = ref->next;
 342                 if (!memcmp(ref->name, "refs/", 5) &&
 343                     check_ref_format(ref->name + 5))
 344                         ; /* trash */
 345                 else if (fetch_all) {
 346                         *newtail = ref;
 347                         ref->next = NULL;
 348                         newtail = &ref->next;
 349                         continue;
 350                 }
 351                 else {
 352                         int order = path_match(ref->name, nr_match, match);
 353                         if (order) {
 354                                 return_refs[order-1] = ref;
 355                                 continue; /* we will link it later */
 356                         }
 357                 }
 358                 free(ref);
 359         }
 360
 361         if (!fetch_all) {
 362                 int i;
 363                 for (i = 0; i < nr_match; i++) {
 364                         ref = return_refs[i];
 365                         if (ref) {
 366                                 *newtail = ref;
 367                                 ref->next = NULL;
 368                                 newtail = &ref->next;
 369                         }
 370                 }
 371                 if (return_refs != fastarray)
 372                         free(return_refs);
 373         }
 374         *refs = newlist;
 375 }
 376
 377 static int everything_local(struct ref **refs, int nr_match, char **match)
 378 {
 379         struct ref *ref;
 380         int retval;
 381         unsigned long cutoff = 0;
 382
 383         track_object_refs = 0;
 384         save_commit_buffer = 0;
 385
 386         for (ref = *refs; ref; ref = ref->next) {
 387                 struct object *o;
 388
 389                 o = parse_object(ref->old_sha1);
 390                 if (!o)
 391                         continue;
 392
 393                 /* We already have it -- which may mean that we were
 394                  * in sync with the other side at some time after
 395                  * that (it is OK if we guess wrong here).
 396                  */
 397                 if (o->type == OBJ_COMMIT) {
 398                         struct commit *commit = (struct commit *)o;
 399                         if (!cutoff || cutoff < commit->date)
 400                                 cutoff = commit->date;
 401                 }
 402         }
 403
 404         if (!depth) {
 405                 for_each_ref(mark_complete, NULL);
 406                 if (cutoff)
 407                         mark_recent_complete_commits(cutoff);
 408         }
 409
 410         /*
 411          * Mark all complete remote refs as common refs.
 412          * Don't mark them common yet; the server has to be told so first.
 413          */
 414         for (ref = *refs; ref; ref = ref->next) {
 415                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 416                                              NULL, 0);
 417
 418                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 419                         continue;
 420
 421                 if (!(o->flags & SEEN)) {
 422                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 423
 424                         mark_common((struct commit *)o, 1, 1);
 425                 }
 426         }
 427
 428         filter_refs(refs, nr_match, match);
 429
 430         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 431                 const unsigned char *remote = ref->old_sha1;
 432                 unsigned char local[20];
 433                 struct object *o;
 434
 435                 o = lookup_object(remote);
 436                 if (!o || !(o->flags & COMPLETE)) {
 437                         retval = 0;
 438                         if (!verbose)
 439                                 continue;
 440                         fprintf(stderr,
 441                                 "want %s (%s)\n", sha1_to_hex(remote),
 442                                 ref->name);
 443                         continue;
 444                 }
 445
 446                 hashcpy(ref->new_sha1, local);
 447                 if (!verbose)
 448                         continue;
 449                 fprintf(stderr,
 450                         "already have %s (%s)\n", sha1_to_hex(remote),
 451                         ref->name);
 452         }
 453         return retval;
 454 }
 455
 456 static pid_t setup_sideband(int fd[2], int xd[2])
 457 {
 458         pid_t side_pid;
 459
 460         if (!use_sideband) {
 461                 fd[0] = xd[0];
 462                 fd[1] = xd[1];
 463                 return 0;
 464         }
 465         /* xd[] is talking with upload-pack; subprocess reads from
 466          * xd[0], spits out band#2 to stderr, and feeds us band#1
 467          * through our fd[0].
 468          */
 469         if (pipe(fd) < 0)
 470                 die("fetch-pack: unable to set up pipe");
 471         side_pid = fork();
 472         if (side_pid < 0)
 473                 die("fetch-pack: unable to fork off sideband demultiplexer");
 474         if (!side_pid) {
 475                 /* subprocess */
 476                 close(fd[0]);
 477                 if (xd[0] != xd[1])
 478                         close(xd[1]);
 479                 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
 480                         exit(1);
 481                 exit(0);
 482         }
 483         close(xd[0]);
 484         close(fd[1]);
 485         fd[1] = xd[1];
 486         return side_pid;
 487 }
 488
 489 static int get_pack(int xd[2], const char **argv)
 490 {
 491         int status;
 492         pid_t pid, side_pid;
 493         int fd[2];
 494
 495         side_pid = setup_sideband(fd, xd);
 496         pid = fork();
 497         if (pid < 0)
 498                 die("fetch-pack: unable to fork off %s", argv[0]);
 499         if (!pid) {
 500                 dup2(fd[0], 0);
 501                 close(fd[0]);
 502                 close(fd[1]);
 503                 execv_git_cmd(argv);
 504                 die("%s exec failed", argv[0]);
 505         }
 506         close(fd[0]);
 507         close(fd[1]);
 508         while (waitpid(pid, &status, 0) < 0) {
 509                 if (errno != EINTR)
 510                         die("waiting for %s: %s", argv[0], strerror(errno));
 511         }
 512         if (WIFEXITED(status)) {
 513                 int code = WEXITSTATUS(status);
 514                 if (code)
 515                         die("%s died with error code %d", argv[0], code);
 516                 return 0;
 517         }
 518         if (WIFSIGNALED(status)) {
 519                 int sig = WTERMSIG(status);
 520                 die("%s died of signal %d", argv[0], sig);
 521         }
 522         die("%s died of unnatural causes %d", argv[0], status);
 523 }
 524
 525 static int explode_rx_pack(int xd[2])
 526 {
 527         const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
 528         return get_pack(xd, argv);
 529 }
 530
 531 static int keep_rx_pack(int xd[2])
 532 {
 533         const char *argv[6];
 534         char keep_arg[256];
 535         int n = 0;
 536
 537         argv[n++] = "index-pack";
 538         argv[n++] = "--stdin";
 539         if (!quiet)
 540                 argv[n++] = "-v";
 541         if (use_thin_pack)
 542                 argv[n++] = "--fix-thin";
 543         if (keep_pack > 1) {
 544                 int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid());
 545                 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 546                         strcpy(keep_arg + s, "localhost");
 547                 argv[n++] = keep_arg;
 548         }
 549         argv[n] = NULL;
 550         return get_pack(xd, argv);
 551 }
 552
 553 static int fetch_pack(int fd[2], int nr_match, char **match)
 554 {
 555         struct ref *ref;
 556         unsigned char sha1[20];
 557         int status;
 558
 559         get_remote_heads(fd[0], &ref, 0, NULL, 0);
 560         if (is_repository_shallow() && !server_supports("shallow"))
 561                 die("Server does not support shallow clients");
 562         if (server_supports("multi_ack")) {
 563                 if (verbose)
 564                         fprintf(stderr, "Server supports multi_ack\n");
 565                 multi_ack = 1;
 566         }
 567         if (server_supports("side-band-64k")) {
 568                 if (verbose)
 569                         fprintf(stderr, "Server supports side-band-64k\n");
 570                 use_sideband = 2;
 571         }
 572         else if (server_supports("side-band")) {
 573                 if (verbose)
 574                         fprintf(stderr, "Server supports side-band\n");
 575                 use_sideband = 1;
 576         }
 577         if (!ref) {
 578                 packet_flush(fd[1]);
 579                 die("no matching remote head");
 580         }
 581         if (everything_local(&ref, nr_match, match)) {
 582                 packet_flush(fd[1]);
 583                 goto all_done;
 584         }
 585         if (find_common(fd, sha1, ref) < 0)
 586                 if (keep_pack != 1)
 587                         /* When cloning, it is not unusual to have
 588                          * no common commit.
 589                          */
 590                         fprintf(stderr, "warning: no common commits\n");
 591
 592         status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd);
 593         if (status)
 594                 die("git-fetch-pack: fetch failed.");
 595
 596  all_done:
 597         while (ref) {
 598                 printf("%s %s\n",
 599                        sha1_to_hex(ref->old_sha1), ref->name);
 600                 ref = ref->next;
 601         }
 602         return 0;
 603 }
 604
 605 int main(int argc, char **argv)
 606 {
 607         int i, ret, nr_heads;
 608         char *dest = NULL, **heads;
 609         int fd[2];
 610         pid_t pid;
 611         struct stat st;
 612         struct lock_file lock;
 613
 614         setup_git_directory();
 615
 616         nr_heads = 0;
 617         heads = NULL;
 618         for (i = 1; i < argc; i++) {
 619                 char *arg = argv[i];
 620
 621                 if (*arg == '-') {
 622                         if (!strncmp("--exec=", arg, 7)) {
 623                                 exec = arg + 7;
 624                                 continue;
 625                         }
 626                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 627                                 quiet = 1;
 628                                 continue;
 629                         }
 630                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 631                                 keep_pack++;
 632                                 continue;
 633                         }
 634                         if (!strcmp("--thin", arg)) {
 635                                 use_thin_pack = 1;
 636                                 continue;
 637                         }
 638                         if (!strcmp("--all", arg)) {
 639                                 fetch_all = 1;
 640                                 continue;
 641                         }
 642                         if (!strcmp("-v", arg)) {
 643                                 verbose = 1;
 644                                 continue;
 645                         }
 646                         if (!strncmp("--depth=", arg, 8)) {
 647                                 depth = strtol(arg + 8, NULL, 0);
 648                                 if (stat(git_path("shallow"), &st))
 649                                         st.st_mtime = 0;
 650                                 continue;
 651                         }
 652                         usage(fetch_pack_usage);
 653                 }
 654                 dest = arg;
 655                 heads = argv + i + 1;
 656                 nr_heads = argc - i - 1;
 657                 break;
 658         }
 659         if (!dest)
 660                 usage(fetch_pack_usage);
 661         pid = git_connect(fd, dest, exec);
 662         if (pid < 0)
 663                 return 1;
 664         ret = fetch_pack(fd, nr_heads, heads);
 665         close(fd[0]);
 666         close(fd[1]);
 667         ret |= finish_connect(pid);
 668
 669         if (!ret && nr_heads) {
 670                 /* If the heads to pull were given, we should have
 671                  * consumed all of them by matching the remote.
 672                  * Otherwise, 'git-fetch remote no-such-ref' would
 673                  * silently succeed without issuing an error.
 674                  */
 675                 for (i = 0; i < nr_heads; i++)
 676                         if (heads[i] && heads[i][0]) {
 677                                 error("no such remote ref %s", heads[i]);
 678                                 ret = 1;
 679                         }
 680         }
 681
 682         if (!ret && depth > 0) {
 683                 struct cache_time mtime;
 684                 char *shallow = git_path("shallow");
 685                 int fd;
 686
 687                 mtime.sec = st.st_mtime;
 688 #ifdef USE_NSEC
 689                 mtime.usec = st.st_mtim.usec;
 690 #endif
 691                 if (stat(shallow, &st)) {
 692                         if (mtime.sec)
 693                                 die("shallow file was removed during fetch");
 694                 } else if (st.st_mtime != mtime.sec
 695 #ifdef USE_NSEC
 696                                 || st.st_mtim.usec != mtime.usec
 697 #endif
 698                           )
 699                         die("shallow file was changed during fetch");
 700
 701                 fd = hold_lock_file_for_update(&lock, shallow, 1);
 702                 if (!write_shallow_commits(fd, 0)) {
 703                         unlink(shallow);
 704                         rollback_lock_file(&lock);
 705                 } else {
 706                         close(fd);
 707                         commit_lock_file(&lock);
 708                 }
 709         }
 710
 711         return !!ret;
 712 }