fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "sideband.h"
   8 #include <sys/wait.h>
   9
  10 static int keep_pack;
  11 static int quiet;
  12 static int verbose;
  13 static int fetch_all;
  14 static const char fetch_pack_usage[] =
  15 "git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [host:]directory <refs>...";
  16 static const char *exec = "git-upload-pack";
  17
  18 #define COMPLETE        (1U << 0)
  19 #define COMMON          (1U << 1)
  20 #define COMMON_REF      (1U << 2)
  21 #define SEEN            (1U << 3)
  22 #define POPPED          (1U << 4)
  23
  24 /*
  25  * After sending this many "have"s if we do not get any new ACK , we
  26  * give up traversing our history.
  27  */
  28 #define MAX_IN_VAIN 256
  29
  30 static struct commit_list *rev_list;
  31 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
  32
  33 static void rev_list_push(struct commit *commit, int mark)
  34 {
  35         if (!(commit->object.flags & mark)) {
  36                 commit->object.flags |= mark;
  37
  38                 if (!(commit->object.parsed))
  39                         parse_commit(commit);
  40
  41                 insert_by_date(commit, &rev_list);
  42
  43                 if (!(commit->object.flags & COMMON))
  44                         non_common_revs++;
  45         }
  46 }
  47
  48 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  49 {
  50         struct object *o = deref_tag(parse_object(sha1), path, 0);
  51
  52         if (o && o->type == OBJ_COMMIT)
  53                 rev_list_push((struct commit *)o, SEEN);
  54
  55         return 0;
  56 }
  57
  58 /*
  59    This function marks a rev and its ancestors as common.
  60    In some cases, it is desirable to mark only the ancestors (for example
  61    when only the server does not yet know that they are common).
  62 */
  63
  64 static void mark_common(struct commit *commit,
  65                 int ancestors_only, int dont_parse)
  66 {
  67         if (commit != NULL && !(commit->object.flags & COMMON)) {
  68                 struct object *o = (struct object *)commit;
  69
  70                 if (!ancestors_only)
  71                         o->flags |= COMMON;
  72
  73                 if (!(o->flags & SEEN))
  74                         rev_list_push(commit, SEEN);
  75                 else {
  76                         struct commit_list *parents;
  77
  78                         if (!ancestors_only && !(o->flags & POPPED))
  79                                 non_common_revs--;
  80                         if (!o->parsed && !dont_parse)
  81                                 parse_commit(commit);
  82
  83                         for (parents = commit->parents;
  84                                         parents;
  85                                         parents = parents->next)
  86                                 mark_common(parents->item, 0, dont_parse);
  87                 }
  88         }
  89 }
  90
  91 /*
  92   Get the next rev to send, ignoring the common.
  93 */
  94
  95 static const unsigned char* get_rev(void)
  96 {
  97         struct commit *commit = NULL;
  98
  99         while (commit == NULL) {
 100                 unsigned int mark;
 101                 struct commit_list* parents;
 102
 103                 if (rev_list == NULL || non_common_revs == 0)
 104                         return NULL;
 105
 106                 commit = rev_list->item;
 107                 if (!(commit->object.parsed))
 108                         parse_commit(commit);
 109                 commit->object.flags |= POPPED;
 110                 if (!(commit->object.flags & COMMON))
 111                         non_common_revs--;
 112
 113                 parents = commit->parents;
 114
 115                 if (commit->object.flags & COMMON) {
 116                         /* do not send "have", and ignore ancestors */
 117                         commit = NULL;
 118                         mark = COMMON | SEEN;
 119                 } else if (commit->object.flags & COMMON_REF)
 120                         /* send "have", and ignore ancestors */
 121                         mark = COMMON | SEEN;
 122                 else
 123                         /* send "have", also for its ancestors */
 124                         mark = SEEN;
 125
 126                 while (parents) {
 127                         if (!(parents->item->object.flags & SEEN))
 128                                 rev_list_push(parents->item, mark);
 129                         if (mark & COMMON)
 130                                 mark_common(parents->item, 1, 0);
 131                         parents = parents->next;
 132                 }
 133
 134                 rev_list = rev_list->next;
 135         }
 136
 137         return commit->object.sha1;
 138 }
 139
 140 static int find_common(int fd[2], unsigned char *result_sha1,
 141                        struct ref *refs)
 142 {
 143         int fetching;
 144         int count = 0, flushes = 0, retval;
 145         const unsigned char *sha1;
 146         unsigned in_vain = 0;
 147         int got_continue = 0;
 148
 149         for_each_ref(rev_list_insert_ref, NULL);
 150
 151         fetching = 0;
 152         for ( ; refs ; refs = refs->next) {
 153                 unsigned char *remote = refs->old_sha1;
 154                 struct object *o;
 155
 156                 /*
 157                  * If that object is complete (i.e. it is an ancestor of a
 158                  * local ref), we tell them we have it but do not have to
 159                  * tell them about its ancestors, which they already know
 160                  * about.
 161                  *
 162                  * We use lookup_object here because we are only
 163                  * interested in the case we *know* the object is
 164                  * reachable and we have already scanned it.
 165                  */
 166                 if (((o = lookup_object(remote)) != NULL) &&
 167                                 (o->flags & COMPLETE)) {
 168                         continue;
 169                 }
 170
 171                 if (!fetching)
 172                         packet_write(fd[1], "want %s%s%s%s%s%s\n",
 173                                      sha1_to_hex(remote),
 174                                      (multi_ack ? " multi_ack" : ""),
 175                                      (use_sideband == 2 ? " side-band-64k" : ""),
 176                                      (use_sideband == 1 ? " side-band" : ""),
 177                                      (use_thin_pack ? " thin-pack" : ""),
 178                                      " ofs-delta");
 179                 else
 180                         packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
 181                 fetching++;
 182         }
 183         if (is_repository_shallow())
 184                 write_shallow_commits(fd[1], 1);
 185         packet_flush(fd[1]);
 186         if (!fetching)
 187                 return 1;
 188
 189         flushes = 0;
 190         retval = -1;
 191         while ((sha1 = get_rev())) {
 192                 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
 193                 if (verbose)
 194                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 195                 in_vain++;
 196                 if (!(31 & ++count)) {
 197                         int ack;
 198
 199                         packet_flush(fd[1]);
 200                         flushes++;
 201
 202                         /*
 203                          * We keep one window "ahead" of the other side, and
 204                          * will wait for an ACK only on the next one
 205                          */
 206                         if (count == 32)
 207                                 continue;
 208
 209                         do {
 210                                 ack = get_ack(fd[0], result_sha1);
 211                                 if (verbose && ack)
 212                                         fprintf(stderr, "got ack %d %s\n", ack,
 213                                                         sha1_to_hex(result_sha1));
 214                                 if (ack == 1) {
 215                                         flushes = 0;
 216                                         multi_ack = 0;
 217                                         retval = 0;
 218                                         goto done;
 219                                 } else if (ack == 2) {
 220                                         struct commit *commit =
 221                                                 lookup_commit(result_sha1);
 222                                         mark_common(commit, 0, 1);
 223                                         retval = 0;
 224                                         in_vain = 0;
 225                                         got_continue = 1;
 226                                 }
 227                         } while (ack);
 228                         flushes--;
 229                         if (got_continue && MAX_IN_VAIN < in_vain) {
 230                                 if (verbose)
 231                                         fprintf(stderr, "giving up\n");
 232                                 break; /* give up */
 233                         }
 234                 }
 235         }
 236 done:
 237         packet_write(fd[1], "done\n");
 238         if (verbose)
 239                 fprintf(stderr, "done\n");
 240         if (retval != 0) {
 241                 multi_ack = 0;
 242                 flushes++;
 243         }
 244         while (flushes || multi_ack) {
 245                 int ack = get_ack(fd[0], result_sha1);
 246                 if (ack) {
 247                         if (verbose)
 248                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 249                                         sha1_to_hex(result_sha1));
 250                         if (ack == 1)
 251                                 return 0;
 252                         multi_ack = 1;
 253                         continue;
 254                 }
 255                 flushes--;
 256         }
 257         return retval;
 258 }
 259
 260 static struct commit_list *complete;
 261
 262 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 263 {
 264         struct object *o = parse_object(sha1);
 265
 266         while (o && o->type == OBJ_TAG) {
 267                 struct tag *t = (struct tag *) o;
 268                 if (!t->tagged)
 269                         break; /* broken repository */
 270                 o->flags |= COMPLETE;
 271                 o = parse_object(t->tagged->sha1);
 272         }
 273         if (o && o->type == OBJ_COMMIT) {
 274                 struct commit *commit = (struct commit *)o;
 275                 commit->object.flags |= COMPLETE;
 276                 insert_by_date(commit, &complete);
 277         }
 278         return 0;
 279 }
 280
 281 static void mark_recent_complete_commits(unsigned long cutoff)
 282 {
 283         while (complete && cutoff <= complete->item->date) {
 284                 if (verbose)
 285                         fprintf(stderr, "Marking %s as complete\n",
 286                                 sha1_to_hex(complete->item->object.sha1));
 287                 pop_most_recent_commit(&complete, COMPLETE);
 288         }
 289 }
 290
 291 static void filter_refs(struct ref **refs, int nr_match, char **match)
 292 {
 293         struct ref **return_refs;
 294         struct ref *newlist = NULL;
 295         struct ref **newtail = &newlist;
 296         struct ref *ref, *next;
 297         struct ref *fastarray[32];
 298
 299         if (nr_match && !fetch_all) {
 300                 if (ARRAY_SIZE(fastarray) < nr_match)
 301                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 302                 else {
 303                         return_refs = fastarray;
 304                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 305                 }
 306         }
 307         else
 308                 return_refs = NULL;
 309
 310         for (ref = *refs; ref; ref = next) {
 311                 next = ref->next;
 312                 if (!memcmp(ref->name, "refs/", 5) &&
 313                     check_ref_format(ref->name + 5))
 314                         ; /* trash */
 315                 else if (fetch_all) {
 316                         *newtail = ref;
 317                         ref->next = NULL;
 318                         newtail = &ref->next;
 319                         continue;
 320                 }
 321                 else {
 322                         int order = path_match(ref->name, nr_match, match);
 323                         if (order) {
 324                                 return_refs[order-1] = ref;
 325                                 continue; /* we will link it later */
 326                         }
 327                 }
 328                 free(ref);
 329         }
 330
 331         if (!fetch_all) {
 332                 int i;
 333                 for (i = 0; i < nr_match; i++) {
 334                         ref = return_refs[i];
 335                         if (ref) {
 336                                 *newtail = ref;
 337                                 ref->next = NULL;
 338                                 newtail = &ref->next;
 339                         }
 340                 }
 341                 if (return_refs != fastarray)
 342                         free(return_refs);
 343         }
 344         *refs = newlist;
 345 }
 346
 347 static int everything_local(struct ref **refs, int nr_match, char **match)
 348 {
 349         struct ref *ref;
 350         int retval;
 351         unsigned long cutoff = 0;
 352
 353         track_object_refs = 0;
 354         save_commit_buffer = 0;
 355
 356         for (ref = *refs; ref; ref = ref->next) {
 357                 struct object *o;
 358
 359                 o = parse_object(ref->old_sha1);
 360                 if (!o)
 361                         continue;
 362
 363                 /* We already have it -- which may mean that we were
 364                  * in sync with the other side at some time after
 365                  * that (it is OK if we guess wrong here).
 366                  */
 367                 if (o->type == OBJ_COMMIT) {
 368                         struct commit *commit = (struct commit *)o;
 369                         if (!cutoff || cutoff < commit->date)
 370                                 cutoff = commit->date;
 371                 }
 372         }
 373
 374         for_each_ref(mark_complete, NULL);
 375         if (cutoff)
 376                 mark_recent_complete_commits(cutoff);
 377
 378         /*
 379          * Mark all complete remote refs as common refs.
 380          * Don't mark them common yet; the server has to be told so first.
 381          */
 382         for (ref = *refs; ref; ref = ref->next) {
 383                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 384                                              NULL, 0);
 385
 386                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 387                         continue;
 388
 389                 if (!(o->flags & SEEN)) {
 390                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 391
 392                         mark_common((struct commit *)o, 1, 1);
 393                 }
 394         }
 395
 396         filter_refs(refs, nr_match, match);
 397
 398         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 399                 const unsigned char *remote = ref->old_sha1;
 400                 unsigned char local[20];
 401                 struct object *o;
 402
 403                 o = lookup_object(remote);
 404                 if (!o || !(o->flags & COMPLETE)) {
 405                         retval = 0;
 406                         if (!verbose)
 407                                 continue;
 408                         fprintf(stderr,
 409                                 "want %s (%s)\n", sha1_to_hex(remote),
 410                                 ref->name);
 411                         continue;
 412                 }
 413
 414                 hashcpy(ref->new_sha1, local);
 415                 if (!verbose)
 416                         continue;
 417                 fprintf(stderr,
 418                         "already have %s (%s)\n", sha1_to_hex(remote),
 419                         ref->name);
 420         }
 421         return retval;
 422 }
 423
 424 static pid_t setup_sideband(int fd[2], int xd[2])
 425 {
 426         pid_t side_pid;
 427
 428         if (!use_sideband) {
 429                 fd[0] = xd[0];
 430                 fd[1] = xd[1];
 431                 return 0;
 432         }
 433         /* xd[] is talking with upload-pack; subprocess reads from
 434          * xd[0], spits out band#2 to stderr, and feeds us band#1
 435          * through our fd[0].
 436          */
 437         if (pipe(fd) < 0)
 438                 die("fetch-pack: unable to set up pipe");
 439         side_pid = fork();
 440         if (side_pid < 0)
 441                 die("fetch-pack: unable to fork off sideband demultiplexer");
 442         if (!side_pid) {
 443                 /* subprocess */
 444                 close(fd[0]);
 445                 if (xd[0] != xd[1])
 446                         close(xd[1]);
 447                 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
 448                         exit(1);
 449                 exit(0);
 450         }
 451         close(xd[0]);
 452         close(fd[1]);
 453         fd[1] = xd[1];
 454         return side_pid;
 455 }
 456
 457 static int get_pack(int xd[2], const char **argv)
 458 {
 459         int status;
 460         pid_t pid, side_pid;
 461         int fd[2];
 462
 463         side_pid = setup_sideband(fd, xd);
 464         pid = fork();
 465         if (pid < 0)
 466                 die("fetch-pack: unable to fork off %s", argv[0]);
 467         if (!pid) {
 468                 dup2(fd[0], 0);
 469                 close(fd[0]);
 470                 close(fd[1]);
 471                 execv_git_cmd(argv);
 472                 die("%s exec failed", argv[0]);
 473         }
 474         close(fd[0]);
 475         close(fd[1]);
 476         while (waitpid(pid, &status, 0) < 0) {
 477                 if (errno != EINTR)
 478                         die("waiting for %s: %s", argv[0], strerror(errno));
 479         }
 480         if (WIFEXITED(status)) {
 481                 int code = WEXITSTATUS(status);
 482                 if (code)
 483                         die("%s died with error code %d", argv[0], code);
 484                 return 0;
 485         }
 486         if (WIFSIGNALED(status)) {
 487                 int sig = WTERMSIG(status);
 488                 die("%s died of signal %d", argv[0], sig);
 489         }
 490         die("%s died of unnatural causes %d", argv[0], status);
 491 }
 492
 493 static int explode_rx_pack(int xd[2])
 494 {
 495         const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
 496         return get_pack(xd, argv);
 497 }
 498
 499 static int keep_rx_pack(int xd[2])
 500 {
 501         const char *argv[6];
 502         char keep_arg[256];
 503         int n = 0;
 504
 505         argv[n++] = "index-pack";
 506         argv[n++] = "--stdin";
 507         if (!quiet)
 508                 argv[n++] = "-v";
 509         if (use_thin_pack)
 510                 argv[n++] = "--fix-thin";
 511         if (keep_pack > 1) {
 512                 int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid());
 513                 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 514                         strcpy(keep_arg + s, "localhost");
 515                 argv[n++] = keep_arg;
 516         }
 517         argv[n] = NULL;
 518         return get_pack(xd, argv);
 519 }
 520
 521 static int fetch_pack(int fd[2], int nr_match, char **match)
 522 {
 523         struct ref *ref;
 524         unsigned char sha1[20];
 525         int status;
 526
 527         get_remote_heads(fd[0], &ref, 0, NULL, 0);
 528         if (is_repository_shallow() && !server_supports("shallow"))
 529                 die("Server does not support shallow clients");
 530         if (server_supports("multi_ack")) {
 531                 if (verbose)
 532                         fprintf(stderr, "Server supports multi_ack\n");
 533                 multi_ack = 1;
 534         }
 535         if (server_supports("side-band-64k")) {
 536                 if (verbose)
 537                         fprintf(stderr, "Server supports side-band-64k\n");
 538                 use_sideband = 2;
 539         }
 540         else if (server_supports("side-band")) {
 541                 if (verbose)
 542                         fprintf(stderr, "Server supports side-band\n");
 543                 use_sideband = 1;
 544         }
 545         if (!ref) {
 546                 packet_flush(fd[1]);
 547                 die("no matching remote head");
 548         }
 549         if (everything_local(&ref, nr_match, match)) {
 550                 packet_flush(fd[1]);
 551                 goto all_done;
 552         }
 553         if (find_common(fd, sha1, ref) < 0)
 554                 if (keep_pack != 1)
 555                         /* When cloning, it is not unusual to have
 556                          * no common commit.
 557                          */
 558                         fprintf(stderr, "warning: no common commits\n");
 559
 560         status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd);
 561         if (status)
 562                 die("git-fetch-pack: fetch failed.");
 563
 564  all_done:
 565         while (ref) {
 566                 printf("%s %s\n",
 567                        sha1_to_hex(ref->old_sha1), ref->name);
 568                 ref = ref->next;
 569         }
 570         return 0;
 571 }
 572
 573 int main(int argc, char **argv)
 574 {
 575         int i, ret, nr_heads;
 576         char *dest = NULL, **heads;
 577         int fd[2];
 578         pid_t pid;
 579
 580         setup_git_directory();
 581
 582         nr_heads = 0;
 583         heads = NULL;
 584         for (i = 1; i < argc; i++) {
 585                 char *arg = argv[i];
 586
 587                 if (*arg == '-') {
 588                         if (!strncmp("--exec=", arg, 7)) {
 589                                 exec = arg + 7;
 590                                 continue;
 591                         }
 592                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 593                                 quiet = 1;
 594                                 continue;
 595                         }
 596                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 597                                 keep_pack++;
 598                                 continue;
 599                         }
 600                         if (!strcmp("--thin", arg)) {
 601                                 use_thin_pack = 1;
 602                                 continue;
 603                         }
 604                         if (!strcmp("--all", arg)) {
 605                                 fetch_all = 1;
 606                                 continue;
 607                         }
 608                         if (!strcmp("-v", arg)) {
 609                                 verbose = 1;
 610                                 continue;
 611                         }
 612                         usage(fetch_pack_usage);
 613                 }
 614                 dest = arg;
 615                 heads = argv + i + 1;
 616                 nr_heads = argc - i - 1;
 617                 break;
 618         }
 619         if (!dest)
 620                 usage(fetch_pack_usage);
 621         pid = git_connect(fd, dest, exec);
 622         if (pid < 0)
 623                 return 1;
 624         ret = fetch_pack(fd, nr_heads, heads);
 625         close(fd[0]);
 626         close(fd[1]);
 627         ret |= finish_connect(pid);
 628
 629         if (!ret && nr_heads) {
 630                 /* If the heads to pull were given, we should have
 631                  * consumed all of them by matching the remote.
 632                  * Otherwise, 'git-fetch remote no-such-ref' would
 633                  * silently succeed without issuing an error.
 634                  */
 635                 for (i = 0; i < nr_heads; i++)
 636                         if (heads[i] && heads[i][0]) {
 637                                 error("no such remote ref %s", heads[i]);
 638                                 ret = 1;
 639                         }
 640         }
 641
 642         return !!ret;
 643 }