fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "sideband.h"
   8 #include <sys/wait.h>
   9
  10 static int keep_pack;
  11 static int quiet;
  12 static int verbose;
  13 static int fetch_all;
  14 static const char fetch_pack_usage[] =
  15 "git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [host:]directory <refs>...";
  16 static const char *exec = "git-upload-pack";
  17
  18 #define COMPLETE        (1U << 0)
  19 #define COMMON          (1U << 1)
  20 #define COMMON_REF      (1U << 2)
  21 #define SEEN            (1U << 3)
  22 #define POPPED          (1U << 4)
  23
  24 /*
  25  * After sending this many "have"s if we do not get any new ACK , we
  26  * give up traversing our history.
  27  */
  28 #define MAX_IN_VAIN 256
  29
  30 static struct commit_list *rev_list;
  31 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
  32
  33 static void rev_list_push(struct commit *commit, int mark)
  34 {
  35         if (!(commit->object.flags & mark)) {
  36                 commit->object.flags |= mark;
  37
  38                 if (!(commit->object.parsed))
  39                         parse_commit(commit);
  40
  41                 insert_by_date(commit, &rev_list);
  42
  43                 if (!(commit->object.flags & COMMON))
  44                         non_common_revs++;
  45         }
  46 }
  47
  48 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  49 {
  50         struct object *o = deref_tag(parse_object(sha1), path, 0);
  51
  52         if (o && o->type == OBJ_COMMIT)
  53                 rev_list_push((struct commit *)o, SEEN);
  54
  55         return 0;
  56 }
  57
  58 /*
  59    This function marks a rev and its ancestors as common.
  60    In some cases, it is desirable to mark only the ancestors (for example
  61    when only the server does not yet know that they are common).
  62 */
  63
  64 static void mark_common(struct commit *commit,
  65                 int ancestors_only, int dont_parse)
  66 {
  67         if (commit != NULL && !(commit->object.flags & COMMON)) {
  68                 struct object *o = (struct object *)commit;
  69
  70                 if (!ancestors_only)
  71                         o->flags |= COMMON;
  72
  73                 if (!(o->flags & SEEN))
  74                         rev_list_push(commit, SEEN);
  75                 else {
  76                         struct commit_list *parents;
  77
  78                         if (!ancestors_only && !(o->flags & POPPED))
  79                                 non_common_revs--;
  80                         if (!o->parsed && !dont_parse)
  81                                 parse_commit(commit);
  82
  83                         for (parents = commit->parents;
  84                                         parents;
  85                                         parents = parents->next)
  86                                 mark_common(parents->item, 0, dont_parse);
  87                 }
  88         }
  89 }
  90
  91 /*
  92   Get the next rev to send, ignoring the common.
  93 */
  94
  95 static const unsigned char* get_rev(void)
  96 {
  97         struct commit *commit = NULL;
  98
  99         while (commit == NULL) {
 100                 unsigned int mark;
 101                 struct commit_list* parents;
 102
 103                 if (rev_list == NULL || non_common_revs == 0)
 104                         return NULL;
 105
 106                 commit = rev_list->item;
 107                 if (!(commit->object.parsed))
 108                         parse_commit(commit);
 109                 commit->object.flags |= POPPED;
 110                 if (!(commit->object.flags & COMMON))
 111                         non_common_revs--;
 112
 113                 parents = commit->parents;
 114
 115                 if (commit->object.flags & COMMON) {
 116                         /* do not send "have", and ignore ancestors */
 117                         commit = NULL;
 118                         mark = COMMON | SEEN;
 119                 } else if (commit->object.flags & COMMON_REF)
 120                         /* send "have", and ignore ancestors */
 121                         mark = COMMON | SEEN;
 122                 else
 123                         /* send "have", also for its ancestors */
 124                         mark = SEEN;
 125
 126                 while (parents) {
 127                         if (!(parents->item->object.flags & SEEN))
 128                                 rev_list_push(parents->item, mark);
 129                         if (mark & COMMON)
 130                                 mark_common(parents->item, 1, 0);
 131                         parents = parents->next;
 132                 }
 133
 134                 rev_list = rev_list->next;
 135         }
 136
 137         return commit->object.sha1;
 138 }
 139
 140 static int find_common(int fd[2], unsigned char *result_sha1,
 141                        struct ref *refs)
 142 {
 143         int fetching;
 144         int count = 0, flushes = 0, retval;
 145         const unsigned char *sha1;
 146         unsigned in_vain = 0;
 147         int got_continue = 0;
 148
 149         for_each_ref(rev_list_insert_ref, NULL);
 150
 151         fetching = 0;
 152         for ( ; refs ; refs = refs->next) {
 153                 unsigned char *remote = refs->old_sha1;
 154                 struct object *o;
 155
 156                 /*
 157                  * If that object is complete (i.e. it is an ancestor of a
 158                  * local ref), we tell them we have it but do not have to
 159                  * tell them about its ancestors, which they already know
 160                  * about.
 161                  *
 162                  * We use lookup_object here because we are only
 163                  * interested in the case we *know* the object is
 164                  * reachable and we have already scanned it.
 165                  */
 166                 if (((o = lookup_object(remote)) != NULL) &&
 167                                 (o->flags & COMPLETE)) {
 168                         continue;
 169                 }
 170
 171                 if (!fetching)
 172                         packet_write(fd[1], "want %s%s%s%s%s%s\n",
 173                                      sha1_to_hex(remote),
 174                                      (multi_ack ? " multi_ack" : ""),
 175                                      (use_sideband == 2 ? " side-band-64k" : ""),
 176                                      (use_sideband == 1 ? " side-band" : ""),
 177                                      (use_thin_pack ? " thin-pack" : ""),
 178                                      " ofs-delta");
 179                 else
 180                         packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
 181                 fetching++;
 182         }
 183         packet_flush(fd[1]);
 184         if (!fetching)
 185                 return 1;
 186
 187         flushes = 0;
 188         retval = -1;
 189         while ((sha1 = get_rev())) {
 190                 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
 191                 if (verbose)
 192                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 193                 in_vain++;
 194                 if (!(31 & ++count)) {
 195                         int ack;
 196
 197                         packet_flush(fd[1]);
 198                         flushes++;
 199
 200                         /*
 201                          * We keep one window "ahead" of the other side, and
 202                          * will wait for an ACK only on the next one
 203                          */
 204                         if (count == 32)
 205                                 continue;
 206
 207                         do {
 208                                 ack = get_ack(fd[0], result_sha1);
 209                                 if (verbose && ack)
 210                                         fprintf(stderr, "got ack %d %s\n", ack,
 211                                                         sha1_to_hex(result_sha1));
 212                                 if (ack == 1) {
 213                                         flushes = 0;
 214                                         multi_ack = 0;
 215                                         retval = 0;
 216                                         goto done;
 217                                 } else if (ack == 2) {
 218                                         struct commit *commit =
 219                                                 lookup_commit(result_sha1);
 220                                         mark_common(commit, 0, 1);
 221                                         retval = 0;
 222                                         in_vain = 0;
 223                                         got_continue = 1;
 224                                 }
 225                         } while (ack);
 226                         flushes--;
 227                         if (got_continue && MAX_IN_VAIN < in_vain) {
 228                                 if (verbose)
 229                                         fprintf(stderr, "giving up\n");
 230                                 break; /* give up */
 231                         }
 232                 }
 233         }
 234 done:
 235         packet_write(fd[1], "done\n");
 236         if (verbose)
 237                 fprintf(stderr, "done\n");
 238         if (retval != 0) {
 239                 multi_ack = 0;
 240                 flushes++;
 241         }
 242         while (flushes || multi_ack) {
 243                 int ack = get_ack(fd[0], result_sha1);
 244                 if (ack) {
 245                         if (verbose)
 246                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 247                                         sha1_to_hex(result_sha1));
 248                         if (ack == 1)
 249                                 return 0;
 250                         multi_ack = 1;
 251                         continue;
 252                 }
 253                 flushes--;
 254         }
 255         return retval;
 256 }
 257
 258 static struct commit_list *complete;
 259
 260 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 261 {
 262         struct object *o = parse_object(sha1);
 263
 264         while (o && o->type == OBJ_TAG) {
 265                 struct tag *t = (struct tag *) o;
 266                 if (!t->tagged)
 267                         break; /* broken repository */
 268                 o->flags |= COMPLETE;
 269                 o = parse_object(t->tagged->sha1);
 270         }
 271         if (o && o->type == OBJ_COMMIT) {
 272                 struct commit *commit = (struct commit *)o;
 273                 commit->object.flags |= COMPLETE;
 274                 insert_by_date(commit, &complete);
 275         }
 276         return 0;
 277 }
 278
 279 static void mark_recent_complete_commits(unsigned long cutoff)
 280 {
 281         while (complete && cutoff <= complete->item->date) {
 282                 if (verbose)
 283                         fprintf(stderr, "Marking %s as complete\n",
 284                                 sha1_to_hex(complete->item->object.sha1));
 285                 pop_most_recent_commit(&complete, COMPLETE);
 286         }
 287 }
 288
 289 static void filter_refs(struct ref **refs, int nr_match, char **match)
 290 {
 291         struct ref **return_refs;
 292         struct ref *newlist = NULL;
 293         struct ref **newtail = &newlist;
 294         struct ref *ref, *next;
 295         struct ref *fastarray[32];
 296
 297         if (nr_match && !fetch_all) {
 298                 if (ARRAY_SIZE(fastarray) < nr_match)
 299                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 300                 else {
 301                         return_refs = fastarray;
 302                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 303                 }
 304         }
 305         else
 306                 return_refs = NULL;
 307
 308         for (ref = *refs; ref; ref = next) {
 309                 next = ref->next;
 310                 if (!memcmp(ref->name, "refs/", 5) &&
 311                     check_ref_format(ref->name + 5))
 312                         ; /* trash */
 313                 else if (fetch_all) {
 314                         *newtail = ref;
 315                         ref->next = NULL;
 316                         newtail = &ref->next;
 317                         continue;
 318                 }
 319                 else {
 320                         int order = path_match(ref->name, nr_match, match);
 321                         if (order) {
 322                                 return_refs[order-1] = ref;
 323                                 continue; /* we will link it later */
 324                         }
 325                 }
 326                 free(ref);
 327         }
 328
 329         if (!fetch_all) {
 330                 int i;
 331                 for (i = 0; i < nr_match; i++) {
 332                         ref = return_refs[i];
 333                         if (ref) {
 334                                 *newtail = ref;
 335                                 ref->next = NULL;
 336                                 newtail = &ref->next;
 337                         }
 338                 }
 339                 if (return_refs != fastarray)
 340                         free(return_refs);
 341         }
 342         *refs = newlist;
 343 }
 344
 345 static int everything_local(struct ref **refs, int nr_match, char **match)
 346 {
 347         struct ref *ref;
 348         int retval;
 349         unsigned long cutoff = 0;
 350
 351         track_object_refs = 0;
 352         save_commit_buffer = 0;
 353
 354         for (ref = *refs; ref; ref = ref->next) {
 355                 struct object *o;
 356
 357                 o = parse_object(ref->old_sha1);
 358                 if (!o)
 359                         continue;
 360
 361                 /* We already have it -- which may mean that we were
 362                  * in sync with the other side at some time after
 363                  * that (it is OK if we guess wrong here).
 364                  */
 365                 if (o->type == OBJ_COMMIT) {
 366                         struct commit *commit = (struct commit *)o;
 367                         if (!cutoff || cutoff < commit->date)
 368                                 cutoff = commit->date;
 369                 }
 370         }
 371
 372         for_each_ref(mark_complete, NULL);
 373         if (cutoff)
 374                 mark_recent_complete_commits(cutoff);
 375
 376         /*
 377          * Mark all complete remote refs as common refs.
 378          * Don't mark them common yet; the server has to be told so first.
 379          */
 380         for (ref = *refs; ref; ref = ref->next) {
 381                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 382                                              NULL, 0);
 383
 384                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 385                         continue;
 386
 387                 if (!(o->flags & SEEN)) {
 388                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 389
 390                         mark_common((struct commit *)o, 1, 1);
 391                 }
 392         }
 393
 394         filter_refs(refs, nr_match, match);
 395
 396         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 397                 const unsigned char *remote = ref->old_sha1;
 398                 unsigned char local[20];
 399                 struct object *o;
 400
 401                 o = lookup_object(remote);
 402                 if (!o || !(o->flags & COMPLETE)) {
 403                         retval = 0;
 404                         if (!verbose)
 405                                 continue;
 406                         fprintf(stderr,
 407                                 "want %s (%s)\n", sha1_to_hex(remote),
 408                                 ref->name);
 409                         continue;
 410                 }
 411
 412                 hashcpy(ref->new_sha1, local);
 413                 if (!verbose)
 414                         continue;
 415                 fprintf(stderr,
 416                         "already have %s (%s)\n", sha1_to_hex(remote),
 417                         ref->name);
 418         }
 419         return retval;
 420 }
 421
 422 static pid_t setup_sideband(int fd[2], int xd[2])
 423 {
 424         pid_t side_pid;
 425
 426         if (!use_sideband) {
 427                 fd[0] = xd[0];
 428                 fd[1] = xd[1];
 429                 return 0;
 430         }
 431         /* xd[] is talking with upload-pack; subprocess reads from
 432          * xd[0], spits out band#2 to stderr, and feeds us band#1
 433          * through our fd[0].
 434          */
 435         if (pipe(fd) < 0)
 436                 die("fetch-pack: unable to set up pipe");
 437         side_pid = fork();
 438         if (side_pid < 0)
 439                 die("fetch-pack: unable to fork off sideband demultiplexer");
 440         if (!side_pid) {
 441                 /* subprocess */
 442                 close(fd[0]);
 443                 if (xd[0] != xd[1])
 444                         close(xd[1]);
 445                 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
 446                         exit(1);
 447                 exit(0);
 448         }
 449         close(xd[0]);
 450         close(fd[1]);
 451         fd[1] = xd[1];
 452         return side_pid;
 453 }
 454
 455 static int get_pack(int xd[2], const char **argv)
 456 {
 457         int status;
 458         pid_t pid, side_pid;
 459         int fd[2];
 460
 461         side_pid = setup_sideband(fd, xd);
 462         pid = fork();
 463         if (pid < 0)
 464                 die("fetch-pack: unable to fork off %s", argv[0]);
 465         if (!pid) {
 466                 dup2(fd[0], 0);
 467                 close(fd[0]);
 468                 close(fd[1]);
 469                 execv_git_cmd(argv);
 470                 die("%s exec failed", argv[0]);
 471         }
 472         close(fd[0]);
 473         close(fd[1]);
 474         while (waitpid(pid, &status, 0) < 0) {
 475                 if (errno != EINTR)
 476                         die("waiting for %s: %s", argv[0], strerror(errno));
 477         }
 478         if (WIFEXITED(status)) {
 479                 int code = WEXITSTATUS(status);
 480                 if (code)
 481                         die("%s died with error code %d", argv[0], code);
 482                 return 0;
 483         }
 484         if (WIFSIGNALED(status)) {
 485                 int sig = WTERMSIG(status);
 486                 die("%s died of signal %d", argv[0], sig);
 487         }
 488         die("%s died of unnatural causes %d", argv[0], status);
 489 }
 490
 491 static int explode_rx_pack(int xd[2])
 492 {
 493         const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
 494         return get_pack(xd, argv);
 495 }
 496
 497 static int keep_rx_pack(int xd[2])
 498 {
 499         const char *argv[6];
 500         char keep_arg[256];
 501         int n = 0;
 502
 503         argv[n++] = "index-pack";
 504         argv[n++] = "--stdin";
 505         if (!quiet)
 506                 argv[n++] = "-v";
 507         if (use_thin_pack)
 508                 argv[n++] = "--fix-thin";
 509         if (keep_pack > 1) {
 510                 int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid());
 511                 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 512                         strcpy(keep_arg + s, "localhost");
 513                 argv[n++] = keep_arg;
 514         }
 515         argv[n] = NULL;
 516         return get_pack(xd, argv);
 517 }
 518
 519 static int fetch_pack(int fd[2], int nr_match, char **match)
 520 {
 521         struct ref *ref;
 522         unsigned char sha1[20];
 523         int status;
 524
 525         get_remote_heads(fd[0], &ref, 0, NULL, 0);
 526         if (server_supports("multi_ack")) {
 527                 if (verbose)
 528                         fprintf(stderr, "Server supports multi_ack\n");
 529                 multi_ack = 1;
 530         }
 531         if (server_supports("side-band-64k")) {
 532                 if (verbose)
 533                         fprintf(stderr, "Server supports side-band-64k\n");
 534                 use_sideband = 2;
 535         }
 536         else if (server_supports("side-band")) {
 537                 if (verbose)
 538                         fprintf(stderr, "Server supports side-band\n");
 539                 use_sideband = 1;
 540         }
 541         if (!ref) {
 542                 packet_flush(fd[1]);
 543                 die("no matching remote head");
 544         }
 545         if (everything_local(&ref, nr_match, match)) {
 546                 packet_flush(fd[1]);
 547                 goto all_done;
 548         }
 549         if (find_common(fd, sha1, ref) < 0)
 550                 if (keep_pack != 1)
 551                         /* When cloning, it is not unusual to have
 552                          * no common commit.
 553                          */
 554                         fprintf(stderr, "warning: no common commits\n");
 555
 556         status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd);
 557         if (status)
 558                 die("git-fetch-pack: fetch failed.");
 559
 560  all_done:
 561         while (ref) {
 562                 printf("%s %s\n",
 563                        sha1_to_hex(ref->old_sha1), ref->name);
 564                 ref = ref->next;
 565         }
 566         return 0;
 567 }
 568
 569 static int remove_duplicates(int nr_heads, char **heads)
 570 {
 571         int src, dst;
 572
 573         for (src = dst = 0; src < nr_heads; src++) {
 574                 /* If heads[src] is different from any of
 575                  * heads[0..dst], push it in.
 576                  */
 577                 int i;
 578                 for (i = 0; i < dst; i++) {
 579                         if (!strcmp(heads[i], heads[src]))
 580                                 break;
 581                 }
 582                 if (i < dst)
 583                         continue;
 584                 if (src != dst)
 585                         heads[dst] = heads[src];
 586                 dst++;
 587         }
 588         heads[dst] = 0;
 589         return dst;
 590 }
 591
 592 int main(int argc, char **argv)
 593 {
 594         int i, ret, nr_heads;
 595         char *dest = NULL, **heads;
 596         int fd[2];
 597         pid_t pid;
 598
 599         setup_git_directory();
 600
 601         nr_heads = 0;
 602         heads = NULL;
 603         for (i = 1; i < argc; i++) {
 604                 char *arg = argv[i];
 605
 606                 if (*arg == '-') {
 607                         if (!strncmp("--exec=", arg, 7)) {
 608                                 exec = arg + 7;
 609                                 continue;
 610                         }
 611                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 612                                 quiet = 1;
 613                                 continue;
 614                         }
 615                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 616                                 keep_pack++;
 617                                 continue;
 618                         }
 619                         if (!strcmp("--thin", arg)) {
 620                                 use_thin_pack = 1;
 621                                 continue;
 622                         }
 623                         if (!strcmp("--all", arg)) {
 624                                 fetch_all = 1;
 625                                 continue;
 626                         }
 627                         if (!strcmp("-v", arg)) {
 628                                 verbose = 1;
 629                                 continue;
 630                         }
 631                         usage(fetch_pack_usage);
 632                 }
 633                 dest = arg;
 634                 heads = argv + i + 1;
 635                 nr_heads = argc - i - 1;
 636                 break;
 637         }
 638         if (!dest)
 639                 usage(fetch_pack_usage);
 640         pid = git_connect(fd, dest, exec);
 641         if (pid < 0)
 642                 return 1;
 643         if (heads && nr_heads)
 644                 nr_heads = remove_duplicates(nr_heads, heads);
 645         ret = fetch_pack(fd, nr_heads, heads);
 646         close(fd[0]);
 647         close(fd[1]);
 648         ret |= finish_connect(pid);
 649
 650         if (!ret && nr_heads) {
 651                 /* If the heads to pull were given, we should have
 652                  * consumed all of them by matching the remote.
 653                  * Otherwise, 'git-fetch remote no-such-ref' would
 654                  * silently succeed without issuing an error.
 655                  */
 656                 for (i = 0; i < nr_heads; i++)
 657                         if (heads[i] && heads[i][0]) {
 658                                 error("no such remote ref %s", heads[i]);
 659                                 ret = 1;
 660                         }
 661         }
 662
 663         return !!ret;
 664 }