fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "sideband.h"
   8 #include <sys/wait.h>
   9
  10 static int keep_pack;
  11 static int quiet;
  12 static int verbose;
  13 static int fetch_all;
  14 static int depth;
  15 static const char fetch_pack_usage[] =
  16 "git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [--depth=<n>] [host:]directory <refs>...";
  17 static const char *exec = "git-upload-pack";
  18
  19 #define COMPLETE        (1U << 0)
  20 #define COMMON          (1U << 1)
  21 #define COMMON_REF      (1U << 2)
  22 #define SEEN            (1U << 3)
  23 #define POPPED          (1U << 4)
  24
  25 /*
  26  * After sending this many "have"s if we do not get any new ACK , we
  27  * give up traversing our history.
  28  */
  29 #define MAX_IN_VAIN 256
  30
  31 static struct commit_list *rev_list;
  32 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
  33
  34 static void rev_list_push(struct commit *commit, int mark)
  35 {
  36         if (!(commit->object.flags & mark)) {
  37                 commit->object.flags |= mark;
  38
  39                 if (!(commit->object.parsed))
  40                         parse_commit(commit);
  41
  42                 insert_by_date(commit, &rev_list);
  43
  44                 if (!(commit->object.flags & COMMON))
  45                         non_common_revs++;
  46         }
  47 }
  48
  49 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  50 {
  51         struct object *o = deref_tag(parse_object(sha1), path, 0);
  52
  53         if (o && o->type == OBJ_COMMIT)
  54                 rev_list_push((struct commit *)o, SEEN);
  55
  56         return 0;
  57 }
  58
  59 /*
  60    This function marks a rev and its ancestors as common.
  61    In some cases, it is desirable to mark only the ancestors (for example
  62    when only the server does not yet know that they are common).
  63 */
  64
  65 static void mark_common(struct commit *commit,
  66                 int ancestors_only, int dont_parse)
  67 {
  68         if (commit != NULL && !(commit->object.flags & COMMON)) {
  69                 struct object *o = (struct object *)commit;
  70
  71                 if (!ancestors_only)
  72                         o->flags |= COMMON;
  73
  74                 if (!(o->flags & SEEN))
  75                         rev_list_push(commit, SEEN);
  76                 else {
  77                         struct commit_list *parents;
  78
  79                         if (!ancestors_only && !(o->flags & POPPED))
  80                                 non_common_revs--;
  81                         if (!o->parsed && !dont_parse)
  82                                 parse_commit(commit);
  83
  84                         for (parents = commit->parents;
  85                                         parents;
  86                                         parents = parents->next)
  87                                 mark_common(parents->item, 0, dont_parse);
  88                 }
  89         }
  90 }
  91
  92 /*
  93   Get the next rev to send, ignoring the common.
  94 */
  95
  96 static const unsigned char* get_rev(void)
  97 {
  98         struct commit *commit = NULL;
  99
 100         while (commit == NULL) {
 101                 unsigned int mark;
 102                 struct commit_list* parents;
 103
 104                 if (rev_list == NULL || non_common_revs == 0)
 105                         return NULL;
 106
 107                 commit = rev_list->item;
 108                 if (!(commit->object.parsed))
 109                         parse_commit(commit);
 110                 commit->object.flags |= POPPED;
 111                 if (!(commit->object.flags & COMMON))
 112                         non_common_revs--;
 113
 114                 parents = commit->parents;
 115
 116                 if (commit->object.flags & COMMON) {
 117                         /* do not send "have", and ignore ancestors */
 118                         commit = NULL;
 119                         mark = COMMON | SEEN;
 120                 } else if (commit->object.flags & COMMON_REF)
 121                         /* send "have", and ignore ancestors */
 122                         mark = COMMON | SEEN;
 123                 else
 124                         /* send "have", also for its ancestors */
 125                         mark = SEEN;
 126
 127                 while (parents) {
 128                         if (!(parents->item->object.flags & SEEN))
 129                                 rev_list_push(parents->item, mark);
 130                         if (mark & COMMON)
 131                                 mark_common(parents->item, 1, 0);
 132                         parents = parents->next;
 133                 }
 134
 135                 rev_list = rev_list->next;
 136         }
 137
 138         return commit->object.sha1;
 139 }
 140
 141 static int find_common(int fd[2], unsigned char *result_sha1,
 142                        struct ref *refs)
 143 {
 144         int fetching;
 145         int count = 0, flushes = 0, retval;
 146         const unsigned char *sha1;
 147         unsigned in_vain = 0;
 148         int got_continue = 0;
 149
 150         for_each_ref(rev_list_insert_ref, NULL);
 151
 152         fetching = 0;
 153         for ( ; refs ; refs = refs->next) {
 154                 unsigned char *remote = refs->old_sha1;
 155                 struct object *o;
 156
 157                 /*
 158                  * If that object is complete (i.e. it is an ancestor of a
 159                  * local ref), we tell them we have it but do not have to
 160                  * tell them about its ancestors, which they already know
 161                  * about.
 162                  *
 163                  * We use lookup_object here because we are only
 164                  * interested in the case we *know* the object is
 165                  * reachable and we have already scanned it.
 166                  */
 167                 if (((o = lookup_object(remote)) != NULL) &&
 168                                 (o->flags & COMPLETE)) {
 169                         continue;
 170                 }
 171
 172                 if (!fetching)
 173                         packet_write(fd[1], "want %s%s%s%s%s%s\n",
 174                                      sha1_to_hex(remote),
 175                                      (multi_ack ? " multi_ack" : ""),
 176                                      (use_sideband == 2 ? " side-band-64k" : ""),
 177                                      (use_sideband == 1 ? " side-band" : ""),
 178                                      (use_thin_pack ? " thin-pack" : ""),
 179                                      " ofs-delta");
 180                 else
 181                         packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
 182                 fetching++;
 183         }
 184         if (is_repository_shallow())
 185                 write_shallow_commits(fd[1], 1);
 186         if (depth > 0)
 187                 packet_write(fd[1], "deepen %d", depth);
 188         packet_flush(fd[1]);
 189         if (!fetching)
 190                 return 1;
 191
 192         if (depth >  0) {
 193                 char line[1024];
 194                 unsigned char sha1[20];
 195                 int len;
 196
 197                 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
 198                         if (!strncmp("shallow ", line, 8)) {
 199                                 if (get_sha1_hex(line + 8, sha1))
 200                                         die("invalid shallow line: %s", line);
 201                                 /* no need making it shallow if we have it already */
 202                                 if (lookup_object(sha1))
 203                                         continue;
 204                                 register_shallow(sha1);
 205                         }
 206                 }
 207         }
 208
 209         flushes = 0;
 210         retval = -1;
 211         while ((sha1 = get_rev())) {
 212                 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
 213                 if (verbose)
 214                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 215                 in_vain++;
 216                 if (!(31 & ++count)) {
 217                         int ack;
 218
 219                         packet_flush(fd[1]);
 220                         flushes++;
 221
 222                         /*
 223                          * We keep one window "ahead" of the other side, and
 224                          * will wait for an ACK only on the next one
 225                          */
 226                         if (count == 32)
 227                                 continue;
 228
 229                         do {
 230                                 ack = get_ack(fd[0], result_sha1);
 231                                 if (verbose && ack)
 232                                         fprintf(stderr, "got ack %d %s\n", ack,
 233                                                         sha1_to_hex(result_sha1));
 234                                 if (ack == 1) {
 235                                         flushes = 0;
 236                                         multi_ack = 0;
 237                                         retval = 0;
 238                                         goto done;
 239                                 } else if (ack == 2) {
 240                                         struct commit *commit =
 241                                                 lookup_commit(result_sha1);
 242                                         mark_common(commit, 0, 1);
 243                                         retval = 0;
 244                                         in_vain = 0;
 245                                         got_continue = 1;
 246                                 }
 247                         } while (ack);
 248                         flushes--;
 249                         if (got_continue && MAX_IN_VAIN < in_vain) {
 250                                 if (verbose)
 251                                         fprintf(stderr, "giving up\n");
 252                                 break; /* give up */
 253                         }
 254                 }
 255         }
 256 done:
 257         packet_write(fd[1], "done\n");
 258         if (verbose)
 259                 fprintf(stderr, "done\n");
 260         if (retval != 0) {
 261                 multi_ack = 0;
 262                 flushes++;
 263         }
 264         while (flushes || multi_ack) {
 265                 int ack = get_ack(fd[0], result_sha1);
 266                 if (ack) {
 267                         if (verbose)
 268                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 269                                         sha1_to_hex(result_sha1));
 270                         if (ack == 1)
 271                                 return 0;
 272                         multi_ack = 1;
 273                         continue;
 274                 }
 275                 flushes--;
 276         }
 277         return retval;
 278 }
 279
 280 static struct commit_list *complete;
 281
 282 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 283 {
 284         struct object *o = parse_object(sha1);
 285
 286         while (o && o->type == OBJ_TAG) {
 287                 struct tag *t = (struct tag *) o;
 288                 if (!t->tagged)
 289                         break; /* broken repository */
 290                 o->flags |= COMPLETE;
 291                 o = parse_object(t->tagged->sha1);
 292         }
 293         if (o && o->type == OBJ_COMMIT) {
 294                 struct commit *commit = (struct commit *)o;
 295                 commit->object.flags |= COMPLETE;
 296                 insert_by_date(commit, &complete);
 297         }
 298         return 0;
 299 }
 300
 301 static void mark_recent_complete_commits(unsigned long cutoff)
 302 {
 303         while (complete && cutoff <= complete->item->date) {
 304                 if (verbose)
 305                         fprintf(stderr, "Marking %s as complete\n",
 306                                 sha1_to_hex(complete->item->object.sha1));
 307                 pop_most_recent_commit(&complete, COMPLETE);
 308         }
 309 }
 310
 311 static void filter_refs(struct ref **refs, int nr_match, char **match)
 312 {
 313         struct ref **return_refs;
 314         struct ref *newlist = NULL;
 315         struct ref **newtail = &newlist;
 316         struct ref *ref, *next;
 317         struct ref *fastarray[32];
 318
 319         if (nr_match && !fetch_all) {
 320                 if (ARRAY_SIZE(fastarray) < nr_match)
 321                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 322                 else {
 323                         return_refs = fastarray;
 324                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 325                 }
 326         }
 327         else
 328                 return_refs = NULL;
 329
 330         for (ref = *refs; ref; ref = next) {
 331                 next = ref->next;
 332                 if (!memcmp(ref->name, "refs/", 5) &&
 333                     check_ref_format(ref->name + 5))
 334                         ; /* trash */
 335                 else if (fetch_all) {
 336                         *newtail = ref;
 337                         ref->next = NULL;
 338                         newtail = &ref->next;
 339                         continue;
 340                 }
 341                 else {
 342                         int order = path_match(ref->name, nr_match, match);
 343                         if (order) {
 344                                 return_refs[order-1] = ref;
 345                                 continue; /* we will link it later */
 346                         }
 347                 }
 348                 free(ref);
 349         }
 350
 351         if (!fetch_all) {
 352                 int i;
 353                 for (i = 0; i < nr_match; i++) {
 354                         ref = return_refs[i];
 355                         if (ref) {
 356                                 *newtail = ref;
 357                                 ref->next = NULL;
 358                                 newtail = &ref->next;
 359                         }
 360                 }
 361                 if (return_refs != fastarray)
 362                         free(return_refs);
 363         }
 364         *refs = newlist;
 365 }
 366
 367 static int everything_local(struct ref **refs, int nr_match, char **match)
 368 {
 369         struct ref *ref;
 370         int retval;
 371         unsigned long cutoff = 0;
 372
 373         track_object_refs = 0;
 374         save_commit_buffer = 0;
 375
 376         for (ref = *refs; ref; ref = ref->next) {
 377                 struct object *o;
 378
 379                 o = parse_object(ref->old_sha1);
 380                 if (!o)
 381                         continue;
 382
 383                 /* We already have it -- which may mean that we were
 384                  * in sync with the other side at some time after
 385                  * that (it is OK if we guess wrong here).
 386                  */
 387                 if (o->type == OBJ_COMMIT) {
 388                         struct commit *commit = (struct commit *)o;
 389                         if (!cutoff || cutoff < commit->date)
 390                                 cutoff = commit->date;
 391                 }
 392         }
 393
 394         for_each_ref(mark_complete, NULL);
 395         if (cutoff)
 396                 mark_recent_complete_commits(cutoff);
 397
 398         /*
 399          * Mark all complete remote refs as common refs.
 400          * Don't mark them common yet; the server has to be told so first.
 401          */
 402         for (ref = *refs; ref; ref = ref->next) {
 403                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 404                                              NULL, 0);
 405
 406                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 407                         continue;
 408
 409                 if (!(o->flags & SEEN)) {
 410                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 411
 412                         mark_common((struct commit *)o, 1, 1);
 413                 }
 414         }
 415
 416         filter_refs(refs, nr_match, match);
 417
 418         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 419                 const unsigned char *remote = ref->old_sha1;
 420                 unsigned char local[20];
 421                 struct object *o;
 422
 423                 o = lookup_object(remote);
 424                 if (!o || !(o->flags & COMPLETE)) {
 425                         retval = 0;
 426                         if (!verbose)
 427                                 continue;
 428                         fprintf(stderr,
 429                                 "want %s (%s)\n", sha1_to_hex(remote),
 430                                 ref->name);
 431                         continue;
 432                 }
 433
 434                 hashcpy(ref->new_sha1, local);
 435                 if (!verbose)
 436                         continue;
 437                 fprintf(stderr,
 438                         "already have %s (%s)\n", sha1_to_hex(remote),
 439                         ref->name);
 440         }
 441         return retval;
 442 }
 443
 444 static pid_t setup_sideband(int fd[2], int xd[2])
 445 {
 446         pid_t side_pid;
 447
 448         if (!use_sideband) {
 449                 fd[0] = xd[0];
 450                 fd[1] = xd[1];
 451                 return 0;
 452         }
 453         /* xd[] is talking with upload-pack; subprocess reads from
 454          * xd[0], spits out band#2 to stderr, and feeds us band#1
 455          * through our fd[0].
 456          */
 457         if (pipe(fd) < 0)
 458                 die("fetch-pack: unable to set up pipe");
 459         side_pid = fork();
 460         if (side_pid < 0)
 461                 die("fetch-pack: unable to fork off sideband demultiplexer");
 462         if (!side_pid) {
 463                 /* subprocess */
 464                 close(fd[0]);
 465                 if (xd[0] != xd[1])
 466                         close(xd[1]);
 467                 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
 468                         exit(1);
 469                 exit(0);
 470         }
 471         close(xd[0]);
 472         close(fd[1]);
 473         fd[1] = xd[1];
 474         return side_pid;
 475 }
 476
 477 static int get_pack(int xd[2], const char **argv)
 478 {
 479         int status;
 480         pid_t pid, side_pid;
 481         int fd[2];
 482
 483         side_pid = setup_sideband(fd, xd);
 484         pid = fork();
 485         if (pid < 0)
 486                 die("fetch-pack: unable to fork off %s", argv[0]);
 487         if (!pid) {
 488                 dup2(fd[0], 0);
 489                 close(fd[0]);
 490                 close(fd[1]);
 491                 execv_git_cmd(argv);
 492                 die("%s exec failed", argv[0]);
 493         }
 494         close(fd[0]);
 495         close(fd[1]);
 496         while (waitpid(pid, &status, 0) < 0) {
 497                 if (errno != EINTR)
 498                         die("waiting for %s: %s", argv[0], strerror(errno));
 499         }
 500         if (WIFEXITED(status)) {
 501                 int code = WEXITSTATUS(status);
 502                 if (code)
 503                         die("%s died with error code %d", argv[0], code);
 504                 return 0;
 505         }
 506         if (WIFSIGNALED(status)) {
 507                 int sig = WTERMSIG(status);
 508                 die("%s died of signal %d", argv[0], sig);
 509         }
 510         die("%s died of unnatural causes %d", argv[0], status);
 511 }
 512
 513 static int explode_rx_pack(int xd[2])
 514 {
 515         const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
 516         return get_pack(xd, argv);
 517 }
 518
 519 static int keep_rx_pack(int xd[2])
 520 {
 521         const char *argv[6];
 522         char keep_arg[256];
 523         int n = 0;
 524
 525         argv[n++] = "index-pack";
 526         argv[n++] = "--stdin";
 527         if (!quiet)
 528                 argv[n++] = "-v";
 529         if (use_thin_pack)
 530                 argv[n++] = "--fix-thin";
 531         if (keep_pack > 1) {
 532                 int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid());
 533                 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 534                         strcpy(keep_arg + s, "localhost");
 535                 argv[n++] = keep_arg;
 536         }
 537         argv[n] = NULL;
 538         return get_pack(xd, argv);
 539 }
 540
 541 static int fetch_pack(int fd[2], int nr_match, char **match)
 542 {
 543         struct ref *ref;
 544         unsigned char sha1[20];
 545         int status;
 546
 547         get_remote_heads(fd[0], &ref, 0, NULL, 0);
 548         if (is_repository_shallow() && !server_supports("shallow"))
 549                 die("Server does not support shallow clients");
 550         if (server_supports("multi_ack")) {
 551                 if (verbose)
 552                         fprintf(stderr, "Server supports multi_ack\n");
 553                 multi_ack = 1;
 554         }
 555         if (server_supports("side-band-64k")) {
 556                 if (verbose)
 557                         fprintf(stderr, "Server supports side-band-64k\n");
 558                 use_sideband = 2;
 559         }
 560         else if (server_supports("side-band")) {
 561                 if (verbose)
 562                         fprintf(stderr, "Server supports side-band\n");
 563                 use_sideband = 1;
 564         }
 565         if (!ref) {
 566                 packet_flush(fd[1]);
 567                 die("no matching remote head");
 568         }
 569         if (everything_local(&ref, nr_match, match)) {
 570                 packet_flush(fd[1]);
 571                 goto all_done;
 572         }
 573         if (find_common(fd, sha1, ref) < 0)
 574                 if (keep_pack != 1)
 575                         /* When cloning, it is not unusual to have
 576                          * no common commit.
 577                          */
 578                         fprintf(stderr, "warning: no common commits\n");
 579
 580         status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd);
 581         if (status)
 582                 die("git-fetch-pack: fetch failed.");
 583
 584  all_done:
 585         while (ref) {
 586                 printf("%s %s\n",
 587                        sha1_to_hex(ref->old_sha1), ref->name);
 588                 ref = ref->next;
 589         }
 590         return 0;
 591 }
 592
 593 int main(int argc, char **argv)
 594 {
 595         int i, ret, nr_heads;
 596         char *dest = NULL, **heads;
 597         int fd[2];
 598         pid_t pid;
 599         struct stat st;
 600         struct lock_file lock;
 601
 602         setup_git_directory();
 603
 604         nr_heads = 0;
 605         heads = NULL;
 606         for (i = 1; i < argc; i++) {
 607                 char *arg = argv[i];
 608
 609                 if (*arg == '-') {
 610                         if (!strncmp("--exec=", arg, 7)) {
 611                                 exec = arg + 7;
 612                                 continue;
 613                         }
 614                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 615                                 quiet = 1;
 616                                 continue;
 617                         }
 618                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 619                                 keep_pack++;
 620                                 continue;
 621                         }
 622                         if (!strcmp("--thin", arg)) {
 623                                 use_thin_pack = 1;
 624                                 continue;
 625                         }
 626                         if (!strcmp("--all", arg)) {
 627                                 fetch_all = 1;
 628                                 continue;
 629                         }
 630                         if (!strcmp("-v", arg)) {
 631                                 verbose = 1;
 632                                 continue;
 633                         }
 634                         if (!strncmp("--depth=", arg, 8)) {
 635                                 depth = strtol(arg + 8, NULL, 0);
 636                                 if (stat(git_path("shallow"), &st))
 637                                         st.st_mtime = 0;
 638                                 continue;
 639                         }
 640                         usage(fetch_pack_usage);
 641                 }
 642                 dest = arg;
 643                 heads = argv + i + 1;
 644                 nr_heads = argc - i - 1;
 645                 break;
 646         }
 647         if (!dest)
 648                 usage(fetch_pack_usage);
 649         if (is_repository_shallow() && depth > 0)
 650                 die("Deepening of a shallow repository not yet supported!");
 651         pid = git_connect(fd, dest, exec);
 652         if (pid < 0)
 653                 return 1;
 654         ret = fetch_pack(fd, nr_heads, heads);
 655         close(fd[0]);
 656         close(fd[1]);
 657         ret |= finish_connect(pid);
 658
 659         if (!ret && nr_heads) {
 660                 /* If the heads to pull were given, we should have
 661                  * consumed all of them by matching the remote.
 662                  * Otherwise, 'git-fetch remote no-such-ref' would
 663                  * silently succeed without issuing an error.
 664                  */
 665                 for (i = 0; i < nr_heads; i++)
 666                         if (heads[i] && heads[i][0]) {
 667                                 error("no such remote ref %s", heads[i]);
 668                                 ret = 1;
 669                         }
 670         }
 671
 672         if (!ret && depth > 0) {
 673                 struct cache_time mtime;
 674                 char *shallow = git_path("shallow");
 675                 int fd;
 676
 677                 mtime.sec = st.st_mtime;
 678 #ifdef USE_NSEC
 679                 mtime.usec = st.st_mtim.usec;
 680 #endif
 681                 if (stat(shallow, &st)) {
 682                         if (mtime.sec)
 683                                 die("shallow file was removed during fetch");
 684                 } else if (st.st_mtime != mtime.sec
 685 #ifdef USE_NSEC
 686                                 || st.st_mtim.usec != mtime.usec
 687 #endif
 688                           )
 689                         die("shallow file was changed during fetch");
 690
 691                 fd = hold_lock_file_for_update(&lock, shallow, 1);
 692                 if (!write_shallow_commits(fd, 0)) {
 693                         unlink(lock.filename);
 694                         rollback_lock_file(&lock);
 695                 } else {
 696                         close(fd);
 697                         commit_lock_file(&lock);
 698                 }
 699         }
 700
 701         return !!ret;
 702 }