fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "sideband.h"
   8
   9 static int keep_pack;
  10 static int quiet;
  11 static int verbose;
  12 static int fetch_all;
  13 static const char fetch_pack_usage[] =
  14 "git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [host:]directory <refs>...";
  15 static const char *exec = "git-upload-pack";
  16
  17 #define COMPLETE        (1U << 0)
  18 #define COMMON          (1U << 1)
  19 #define COMMON_REF      (1U << 2)
  20 #define SEEN            (1U << 3)
  21 #define POPPED          (1U << 4)
  22
  23 /*
  24  * After sending this many "have"s if we do not get any new ACK , we
  25  * give up traversing our history.
  26  */
  27 #define MAX_IN_VAIN 256
  28
  29 static struct commit_list *rev_list;
  30 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
  31
  32 static void rev_list_push(struct commit *commit, int mark)
  33 {
  34         if (!(commit->object.flags & mark)) {
  35                 commit->object.flags |= mark;
  36
  37                 if (!(commit->object.parsed))
  38                         parse_commit(commit);
  39
  40                 insert_by_date(commit, &rev_list);
  41
  42                 if (!(commit->object.flags & COMMON))
  43                         non_common_revs++;
  44         }
  45 }
  46
  47 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  48 {
  49         struct object *o = deref_tag(parse_object(sha1), path, 0);
  50
  51         if (o && o->type == OBJ_COMMIT)
  52                 rev_list_push((struct commit *)o, SEEN);
  53
  54         return 0;
  55 }
  56
  57 /*
  58    This function marks a rev and its ancestors as common.
  59    In some cases, it is desirable to mark only the ancestors (for example
  60    when only the server does not yet know that they are common).
  61 */
  62
  63 static void mark_common(struct commit *commit,
  64                 int ancestors_only, int dont_parse)
  65 {
  66         if (commit != NULL && !(commit->object.flags & COMMON)) {
  67                 struct object *o = (struct object *)commit;
  68
  69                 if (!ancestors_only)
  70                         o->flags |= COMMON;
  71
  72                 if (!(o->flags & SEEN))
  73                         rev_list_push(commit, SEEN);
  74                 else {
  75                         struct commit_list *parents;
  76
  77                         if (!ancestors_only && !(o->flags & POPPED))
  78                                 non_common_revs--;
  79                         if (!o->parsed && !dont_parse)
  80                                 parse_commit(commit);
  81
  82                         for (parents = commit->parents;
  83                                         parents;
  84                                         parents = parents->next)
  85                                 mark_common(parents->item, 0, dont_parse);
  86                 }
  87         }
  88 }
  89
  90 /*
  91   Get the next rev to send, ignoring the common.
  92 */
  93
  94 static const unsigned char* get_rev(void)
  95 {
  96         struct commit *commit = NULL;
  97
  98         while (commit == NULL) {
  99                 unsigned int mark;
 100                 struct commit_list* parents;
 101
 102                 if (rev_list == NULL || non_common_revs == 0)
 103                         return NULL;
 104
 105                 commit = rev_list->item;
 106                 if (!(commit->object.parsed))
 107                         parse_commit(commit);
 108                 commit->object.flags |= POPPED;
 109                 if (!(commit->object.flags & COMMON))
 110                         non_common_revs--;
 111
 112                 parents = commit->parents;
 113
 114                 if (commit->object.flags & COMMON) {
 115                         /* do not send "have", and ignore ancestors */
 116                         commit = NULL;
 117                         mark = COMMON | SEEN;
 118                 } else if (commit->object.flags & COMMON_REF)
 119                         /* send "have", and ignore ancestors */
 120                         mark = COMMON | SEEN;
 121                 else
 122                         /* send "have", also for its ancestors */
 123                         mark = SEEN;
 124
 125                 while (parents) {
 126                         if (!(parents->item->object.flags & SEEN))
 127                                 rev_list_push(parents->item, mark);
 128                         if (mark & COMMON)
 129                                 mark_common(parents->item, 1, 0);
 130                         parents = parents->next;
 131                 }
 132
 133                 rev_list = rev_list->next;
 134         }
 135
 136         return commit->object.sha1;
 137 }
 138
 139 static int find_common(int fd[2], unsigned char *result_sha1,
 140                        struct ref *refs)
 141 {
 142         int fetching;
 143         int count = 0, flushes = 0, retval;
 144         const unsigned char *sha1;
 145         unsigned in_vain = 0;
 146         int got_continue = 0;
 147
 148         for_each_ref(rev_list_insert_ref, NULL);
 149
 150         fetching = 0;
 151         for ( ; refs ; refs = refs->next) {
 152                 unsigned char *remote = refs->old_sha1;
 153                 struct object *o;
 154
 155                 /*
 156                  * If that object is complete (i.e. it is an ancestor of a
 157                  * local ref), we tell them we have it but do not have to
 158                  * tell them about its ancestors, which they already know
 159                  * about.
 160                  *
 161                  * We use lookup_object here because we are only
 162                  * interested in the case we *know* the object is
 163                  * reachable and we have already scanned it.
 164                  */
 165                 if (((o = lookup_object(remote)) != NULL) &&
 166                                 (o->flags & COMPLETE)) {
 167                         continue;
 168                 }
 169
 170                 if (!fetching)
 171                         packet_write(fd[1], "want %s%s%s%s%s%s\n",
 172                                      sha1_to_hex(remote),
 173                                      (multi_ack ? " multi_ack" : ""),
 174                                      (use_sideband == 2 ? " side-band-64k" : ""),
 175                                      (use_sideband == 1 ? " side-band" : ""),
 176                                      (use_thin_pack ? " thin-pack" : ""),
 177                                      " ofs-delta");
 178                 else
 179                         packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
 180                 fetching++;
 181         }
 182         packet_flush(fd[1]);
 183         if (!fetching)
 184                 return 1;
 185
 186         flushes = 0;
 187         retval = -1;
 188         while ((sha1 = get_rev())) {
 189                 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
 190                 if (verbose)
 191                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 192                 in_vain++;
 193                 if (!(31 & ++count)) {
 194                         int ack;
 195
 196                         packet_flush(fd[1]);
 197                         flushes++;
 198
 199                         /*
 200                          * We keep one window "ahead" of the other side, and
 201                          * will wait for an ACK only on the next one
 202                          */
 203                         if (count == 32)
 204                                 continue;
 205
 206                         do {
 207                                 ack = get_ack(fd[0], result_sha1);
 208                                 if (verbose && ack)
 209                                         fprintf(stderr, "got ack %d %s\n", ack,
 210                                                         sha1_to_hex(result_sha1));
 211                                 if (ack == 1) {
 212                                         flushes = 0;
 213                                         multi_ack = 0;
 214                                         retval = 0;
 215                                         goto done;
 216                                 } else if (ack == 2) {
 217                                         struct commit *commit =
 218                                                 lookup_commit(result_sha1);
 219                                         mark_common(commit, 0, 1);
 220                                         retval = 0;
 221                                         in_vain = 0;
 222                                         got_continue = 1;
 223                                 }
 224                         } while (ack);
 225                         flushes--;
 226                         if (got_continue && MAX_IN_VAIN < in_vain) {
 227                                 if (verbose)
 228                                         fprintf(stderr, "giving up\n");
 229                                 break; /* give up */
 230                         }
 231                 }
 232         }
 233 done:
 234         packet_write(fd[1], "done\n");
 235         if (verbose)
 236                 fprintf(stderr, "done\n");
 237         if (retval != 0) {
 238                 multi_ack = 0;
 239                 flushes++;
 240         }
 241         while (flushes || multi_ack) {
 242                 int ack = get_ack(fd[0], result_sha1);
 243                 if (ack) {
 244                         if (verbose)
 245                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 246                                         sha1_to_hex(result_sha1));
 247                         if (ack == 1)
 248                                 return 0;
 249                         multi_ack = 1;
 250                         continue;
 251                 }
 252                 flushes--;
 253         }
 254         return retval;
 255 }
 256
 257 static struct commit_list *complete;
 258
 259 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 260 {
 261         struct object *o = parse_object(sha1);
 262
 263         while (o && o->type == OBJ_TAG) {
 264                 struct tag *t = (struct tag *) o;
 265                 if (!t->tagged)
 266                         break; /* broken repository */
 267                 o->flags |= COMPLETE;
 268                 o = parse_object(t->tagged->sha1);
 269         }
 270         if (o && o->type == OBJ_COMMIT) {
 271                 struct commit *commit = (struct commit *)o;
 272                 commit->object.flags |= COMPLETE;
 273                 insert_by_date(commit, &complete);
 274         }
 275         return 0;
 276 }
 277
 278 static void mark_recent_complete_commits(unsigned long cutoff)
 279 {
 280         while (complete && cutoff <= complete->item->date) {
 281                 if (verbose)
 282                         fprintf(stderr, "Marking %s as complete\n",
 283                                 sha1_to_hex(complete->item->object.sha1));
 284                 pop_most_recent_commit(&complete, COMPLETE);
 285         }
 286 }
 287
 288 static void filter_refs(struct ref **refs, int nr_match, char **match)
 289 {
 290         struct ref **return_refs;
 291         struct ref *newlist = NULL;
 292         struct ref **newtail = &newlist;
 293         struct ref *ref, *next;
 294         struct ref *fastarray[32];
 295
 296         if (nr_match && !fetch_all) {
 297                 if (ARRAY_SIZE(fastarray) < nr_match)
 298                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 299                 else {
 300                         return_refs = fastarray;
 301                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 302                 }
 303         }
 304         else
 305                 return_refs = NULL;
 306
 307         for (ref = *refs; ref; ref = next) {
 308                 next = ref->next;
 309                 if (!memcmp(ref->name, "refs/", 5) &&
 310                     check_ref_format(ref->name + 5))
 311                         ; /* trash */
 312                 else if (fetch_all) {
 313                         *newtail = ref;
 314                         ref->next = NULL;
 315                         newtail = &ref->next;
 316                         continue;
 317                 }
 318                 else {
 319                         int order = path_match(ref->name, nr_match, match);
 320                         if (order) {
 321                                 return_refs[order-1] = ref;
 322                                 continue; /* we will link it later */
 323                         }
 324                 }
 325                 free(ref);
 326         }
 327
 328         if (!fetch_all) {
 329                 int i;
 330                 for (i = 0; i < nr_match; i++) {
 331                         ref = return_refs[i];
 332                         if (ref) {
 333                                 *newtail = ref;
 334                                 ref->next = NULL;
 335                                 newtail = &ref->next;
 336                         }
 337                 }
 338                 if (return_refs != fastarray)
 339                         free(return_refs);
 340         }
 341         *refs = newlist;
 342 }
 343
 344 static int everything_local(struct ref **refs, int nr_match, char **match)
 345 {
 346         struct ref *ref;
 347         int retval;
 348         unsigned long cutoff = 0;
 349
 350         track_object_refs = 0;
 351         save_commit_buffer = 0;
 352
 353         for (ref = *refs; ref; ref = ref->next) {
 354                 struct object *o;
 355
 356                 o = parse_object(ref->old_sha1);
 357                 if (!o)
 358                         continue;
 359
 360                 /* We already have it -- which may mean that we were
 361                  * in sync with the other side at some time after
 362                  * that (it is OK if we guess wrong here).
 363                  */
 364                 if (o->type == OBJ_COMMIT) {
 365                         struct commit *commit = (struct commit *)o;
 366                         if (!cutoff || cutoff < commit->date)
 367                                 cutoff = commit->date;
 368                 }
 369         }
 370
 371         for_each_ref(mark_complete, NULL);
 372         if (cutoff)
 373                 mark_recent_complete_commits(cutoff);
 374
 375         /*
 376          * Mark all complete remote refs as common refs.
 377          * Don't mark them common yet; the server has to be told so first.
 378          */
 379         for (ref = *refs; ref; ref = ref->next) {
 380                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 381                                              NULL, 0);
 382
 383                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 384                         continue;
 385
 386                 if (!(o->flags & SEEN)) {
 387                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 388
 389                         mark_common((struct commit *)o, 1, 1);
 390                 }
 391         }
 392
 393         filter_refs(refs, nr_match, match);
 394
 395         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 396                 const unsigned char *remote = ref->old_sha1;
 397                 unsigned char local[20];
 398                 struct object *o;
 399
 400                 o = lookup_object(remote);
 401                 if (!o || !(o->flags & COMPLETE)) {
 402                         retval = 0;
 403                         if (!verbose)
 404                                 continue;
 405                         fprintf(stderr,
 406                                 "want %s (%s)\n", sha1_to_hex(remote),
 407                                 ref->name);
 408                         continue;
 409                 }
 410
 411                 hashcpy(ref->new_sha1, local);
 412                 if (!verbose)
 413                         continue;
 414                 fprintf(stderr,
 415                         "already have %s (%s)\n", sha1_to_hex(remote),
 416                         ref->name);
 417         }
 418         return retval;
 419 }
 420
 421 static pid_t setup_sideband(int fd[2], int xd[2])
 422 {
 423         pid_t side_pid;
 424
 425         if (!use_sideband) {
 426                 fd[0] = xd[0];
 427                 fd[1] = xd[1];
 428                 return 0;
 429         }
 430         /* xd[] is talking with upload-pack; subprocess reads from
 431          * xd[0], spits out band#2 to stderr, and feeds us band#1
 432          * through our fd[0].
 433          */
 434         if (pipe(fd) < 0)
 435                 die("fetch-pack: unable to set up pipe");
 436         side_pid = fork();
 437         if (side_pid < 0)
 438                 die("fetch-pack: unable to fork off sideband demultiplexer");
 439         if (!side_pid) {
 440                 /* subprocess */
 441                 close(fd[0]);
 442                 if (xd[0] != xd[1])
 443                         close(xd[1]);
 444                 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
 445                         exit(1);
 446                 exit(0);
 447         }
 448         close(xd[0]);
 449         close(fd[1]);
 450         fd[1] = xd[1];
 451         return side_pid;
 452 }
 453
 454 static int get_pack(int xd[2], const char **argv)
 455 {
 456         int status;
 457         pid_t pid, side_pid;
 458         int fd[2];
 459
 460         side_pid = setup_sideband(fd, xd);
 461         pid = fork();
 462         if (pid < 0)
 463                 die("fetch-pack: unable to fork off %s", argv[0]);
 464         if (!pid) {
 465                 dup2(fd[0], 0);
 466                 close(fd[0]);
 467                 close(fd[1]);
 468                 execv_git_cmd(argv);
 469                 die("%s exec failed", argv[0]);
 470         }
 471         close(fd[0]);
 472         close(fd[1]);
 473         while (waitpid(pid, &status, 0) < 0) {
 474                 if (errno != EINTR)
 475                         die("waiting for %s: %s", argv[0], strerror(errno));
 476         }
 477         if (WIFEXITED(status)) {
 478                 int code = WEXITSTATUS(status);
 479                 if (code)
 480                         die("%s died with error code %d", argv[0], code);
 481                 return 0;
 482         }
 483         if (WIFSIGNALED(status)) {
 484                 int sig = WTERMSIG(status);
 485                 die("%s died of signal %d", argv[0], sig);
 486         }
 487         die("%s died of unnatural causes %d", argv[0], status);
 488 }
 489
 490 static int explode_rx_pack(int xd[2])
 491 {
 492         const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
 493         return get_pack(xd, argv);
 494 }
 495
 496 static int keep_rx_pack(int xd[2])
 497 {
 498         const char *argv[6];
 499         char keep_arg[256];
 500         int n = 0;
 501
 502         argv[n++] = "index-pack";
 503         argv[n++] = "--stdin";
 504         if (!quiet)
 505                 argv[n++] = "-v";
 506         if (use_thin_pack)
 507                 argv[n++] = "--fix-thin";
 508         if (keep_pack > 1) {
 509                 int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid());
 510                 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 511                         strcpy(keep_arg + s, "localhost");
 512                 argv[n++] = keep_arg;
 513         }
 514         argv[n] = NULL;
 515         return get_pack(xd, argv);
 516 }
 517
 518 static int fetch_pack(int fd[2], int nr_match, char **match)
 519 {
 520         struct ref *ref;
 521         unsigned char sha1[20];
 522         int status;
 523
 524         get_remote_heads(fd[0], &ref, 0, NULL, 0);
 525         if (server_supports("multi_ack")) {
 526                 if (verbose)
 527                         fprintf(stderr, "Server supports multi_ack\n");
 528                 multi_ack = 1;
 529         }
 530         if (server_supports("side-band-64k")) {
 531                 if (verbose)
 532                         fprintf(stderr, "Server supports side-band-64k\n");
 533                 use_sideband = 2;
 534         }
 535         else if (server_supports("side-band")) {
 536                 if (verbose)
 537                         fprintf(stderr, "Server supports side-band\n");
 538                 use_sideband = 1;
 539         }
 540         if (!ref) {
 541                 packet_flush(fd[1]);
 542                 die("no matching remote head");
 543         }
 544         if (everything_local(&ref, nr_match, match)) {
 545                 packet_flush(fd[1]);
 546                 goto all_done;
 547         }
 548         if (find_common(fd, sha1, ref) < 0)
 549                 if (keep_pack != 1)
 550                         /* When cloning, it is not unusual to have
 551                          * no common commit.
 552                          */
 553                         fprintf(stderr, "warning: no common commits\n");
 554
 555         status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd);
 556         if (status)
 557                 die("git-fetch-pack: fetch failed.");
 558
 559  all_done:
 560         while (ref) {
 561                 printf("%s %s\n",
 562                        sha1_to_hex(ref->old_sha1), ref->name);
 563                 ref = ref->next;
 564         }
 565         return 0;
 566 }
 567
 568 static int remove_duplicates(int nr_heads, char **heads)
 569 {
 570         int src, dst;
 571
 572         for (src = dst = 0; src < nr_heads; src++) {
 573                 /* If heads[src] is different from any of
 574                  * heads[0..dst], push it in.
 575                  */
 576                 int i;
 577                 for (i = 0; i < dst; i++) {
 578                         if (!strcmp(heads[i], heads[src]))
 579                                 break;
 580                 }
 581                 if (i < dst)
 582                         continue;
 583                 if (src != dst)
 584                         heads[dst] = heads[src];
 585                 dst++;
 586         }
 587         heads[dst] = 0;
 588         return dst;
 589 }
 590
 591 int main(int argc, char **argv)
 592 {
 593         int i, ret, nr_heads;
 594         char *dest = NULL, **heads;
 595         int fd[2];
 596         pid_t pid;
 597
 598         setup_git_directory();
 599
 600         nr_heads = 0;
 601         heads = NULL;
 602         for (i = 1; i < argc; i++) {
 603                 char *arg = argv[i];
 604
 605                 if (*arg == '-') {
 606                         if (!strncmp("--exec=", arg, 7)) {
 607                                 exec = arg + 7;
 608                                 continue;
 609                         }
 610                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 611                                 quiet = 1;
 612                                 continue;
 613                         }
 614                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 615                                 keep_pack++;
 616                                 continue;
 617                         }
 618                         if (!strcmp("--thin", arg)) {
 619                                 use_thin_pack = 1;
 620                                 continue;
 621                         }
 622                         if (!strcmp("--all", arg)) {
 623                                 fetch_all = 1;
 624                                 continue;
 625                         }
 626                         if (!strcmp("-v", arg)) {
 627                                 verbose = 1;
 628                                 continue;
 629                         }
 630                         usage(fetch_pack_usage);
 631                 }
 632                 dest = arg;
 633                 heads = argv + i + 1;
 634                 nr_heads = argc - i - 1;
 635                 break;
 636         }
 637         if (!dest)
 638                 usage(fetch_pack_usage);
 639         pid = git_connect(fd, dest, exec);
 640         if (pid < 0)
 641                 return 1;
 642         if (heads && nr_heads)
 643                 nr_heads = remove_duplicates(nr_heads, heads);
 644         ret = fetch_pack(fd, nr_heads, heads);
 645         close(fd[0]);
 646         close(fd[1]);
 647         ret |= finish_connect(pid);
 648
 649         if (!ret && nr_heads) {
 650                 /* If the heads to pull were given, we should have
 651                  * consumed all of them by matching the remote.
 652                  * Otherwise, 'git-fetch remote no-such-ref' would
 653                  * silently succeed without issuing an error.
 654                  */
 655                 for (i = 0; i < nr_heads; i++)
 656                         if (heads[i] && heads[i][0]) {
 657                                 error("no such remote ref %s", heads[i]);
 658                                 ret = 1;
 659                         }
 660         }
 661
 662         return !!ret;
 663 }