fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18 #include "sigchain.h"
  19
  20 static int transfer_unpack_limit = -1;
  21 static int fetch_unpack_limit = -1;
  22 static int unpack_limit = 100;
  23 static int prefer_ofs_delta = 1;
  24 static int no_done;
  25 static int fetch_fsck_objects = -1;
  26 static int transfer_fsck_objects = -1;
  27 static int agent_supported;
  28 static struct lock_file shallow_lock;
  29 static const char *alternate_shallow_file;
  30
  31 /* Remember to update object flag allocation in object.h */
  32 #define COMPLETE        (1U << 0)
  33 #define COMMON          (1U << 1)
  34 #define COMMON_REF      (1U << 2)
  35 #define SEEN            (1U << 3)
  36 #define POPPED          (1U << 4)
  37
  38 static int marked;
  39
  40 /*
  41  * After sending this many "have"s if we do not get any new ACK , we
  42  * give up traversing our history.
  43  */
  44 #define MAX_IN_VAIN 256
  45
  46 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  47 static int non_common_revs, multi_ack, use_sideband;
  48 /* Allow specifying sha1 if it is a ref tip. */
  49 #define ALLOW_TIP_SHA1  01
  50 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  51 #define ALLOW_REACHABLE_SHA1    02
  52 static unsigned int allow_unadvertised_object_request;
  53
  54 static void rev_list_push(struct commit *commit, int mark)
  55 {
  56         if (!(commit->object.flags & mark)) {
  57                 commit->object.flags |= mark;
  58
  59                 if (parse_commit(commit))
  60                         return;
  61
  62                 prio_queue_put(&rev_list, commit);
  63
  64                 if (!(commit->object.flags & COMMON))
  65                         non_common_revs++;
  66         }
  67 }
  68
  69 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
  70 {
  71         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  72
  73         if (o && o->type == OBJ_COMMIT)
  74                 rev_list_push((struct commit *)o, SEEN);
  75
  76         return 0;
  77 }
  78
  79 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
  80                                    int flag, void *cb_data)
  81 {
  82         return rev_list_insert_ref(refname, oid->hash);
  83 }
  84
  85 static int clear_marks(const char *refname, const struct object_id *oid,
  86                        int flag, void *cb_data)
  87 {
  88         struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
  89
  90         if (o && o->type == OBJ_COMMIT)
  91                 clear_commit_marks((struct commit *)o,
  92                                    COMMON | COMMON_REF | SEEN | POPPED);
  93         return 0;
  94 }
  95
  96 /*
  97    This function marks a rev and its ancestors as common.
  98    In some cases, it is desirable to mark only the ancestors (for example
  99    when only the server does not yet know that they are common).
 100 */
 101
 102 static void mark_common(struct commit *commit,
 103                 int ancestors_only, int dont_parse)
 104 {
 105         if (commit != NULL && !(commit->object.flags & COMMON)) {
 106                 struct object *o = (struct object *)commit;
 107
 108                 if (!ancestors_only)
 109                         o->flags |= COMMON;
 110
 111                 if (!(o->flags & SEEN))
 112                         rev_list_push(commit, SEEN);
 113                 else {
 114                         struct commit_list *parents;
 115
 116                         if (!ancestors_only && !(o->flags & POPPED))
 117                                 non_common_revs--;
 118                         if (!o->parsed && !dont_parse)
 119                                 if (parse_commit(commit))
 120                                         return;
 121
 122                         for (parents = commit->parents;
 123                                         parents;
 124                                         parents = parents->next)
 125                                 mark_common(parents->item, 0, dont_parse);
 126                 }
 127         }
 128 }
 129
 130 /*
 131   Get the next rev to send, ignoring the common.
 132 */
 133
 134 static const unsigned char *get_rev(void)
 135 {
 136         struct commit *commit = NULL;
 137
 138         while (commit == NULL) {
 139                 unsigned int mark;
 140                 struct commit_list *parents;
 141
 142                 if (rev_list.nr == 0 || non_common_revs == 0)
 143                         return NULL;
 144
 145                 commit = prio_queue_get(&rev_list);
 146                 parse_commit(commit);
 147                 parents = commit->parents;
 148
 149                 commit->object.flags |= POPPED;
 150                 if (!(commit->object.flags & COMMON))
 151                         non_common_revs--;
 152
 153                 if (commit->object.flags & COMMON) {
 154                         /* do not send "have", and ignore ancestors */
 155                         commit = NULL;
 156                         mark = COMMON | SEEN;
 157                 } else if (commit->object.flags & COMMON_REF)
 158                         /* send "have", and ignore ancestors */
 159                         mark = COMMON | SEEN;
 160                 else
 161                         /* send "have", also for its ancestors */
 162                         mark = SEEN;
 163
 164                 while (parents) {
 165                         if (!(parents->item->object.flags & SEEN))
 166                                 rev_list_push(parents->item, mark);
 167                         if (mark & COMMON)
 168                                 mark_common(parents->item, 1, 0);
 169                         parents = parents->next;
 170                 }
 171         }
 172
 173         return commit->object.oid.hash;
 174 }
 175
 176 enum ack_type {
 177         NAK = 0,
 178         ACK,
 179         ACK_continue,
 180         ACK_common,
 181         ACK_ready
 182 };
 183
 184 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 185 {
 186         if (args->stateless_rpc && args->depth > 0) {
 187                 /* If we sent a depth we will get back "duplicate"
 188                  * shallow and unshallow commands every time there
 189                  * is a block of have lines exchanged.
 190                  */
 191                 char *line;
 192                 while ((line = packet_read_line(fd, NULL))) {
 193                         if (starts_with(line, "shallow "))
 194                                 continue;
 195                         if (starts_with(line, "unshallow "))
 196                                 continue;
 197                         die("git fetch-pack: expected shallow list");
 198                 }
 199         }
 200 }
 201
 202 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 203 {
 204         int len;
 205         char *line = packet_read_line(fd, &len);
 206         const char *arg;
 207
 208         if (!len)
 209                 die("git fetch-pack: expected ACK/NAK, got EOF");
 210         if (!strcmp(line, "NAK"))
 211                 return NAK;
 212         if (skip_prefix(line, "ACK ", &arg)) {
 213                 if (!get_sha1_hex(arg, result_sha1)) {
 214                         arg += 40;
 215                         len -= arg - line;
 216                         if (len < 1)
 217                                 return ACK;
 218                         if (strstr(arg, "continue"))
 219                                 return ACK_continue;
 220                         if (strstr(arg, "common"))
 221                                 return ACK_common;
 222                         if (strstr(arg, "ready"))
 223                                 return ACK_ready;
 224                         return ACK;
 225                 }
 226         }
 227         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 228 }
 229
 230 static void send_request(struct fetch_pack_args *args,
 231                          int fd, struct strbuf *buf)
 232 {
 233         if (args->stateless_rpc) {
 234                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 235                 packet_flush(fd);
 236         } else
 237                 write_or_die(fd, buf->buf, buf->len);
 238 }
 239
 240 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 241 {
 242         rev_list_insert_ref(NULL, ref->old_oid.hash);
 243 }
 244
 245 #define INITIAL_FLUSH 16
 246 #define PIPESAFE_FLUSH 32
 247 #define LARGE_FLUSH 1024
 248
 249 static int next_flush(struct fetch_pack_args *args, int count)
 250 {
 251         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 252
 253         if (count < flush_limit)
 254                 count <<= 1;
 255         else
 256                 count += flush_limit;
 257         return count;
 258 }
 259
 260 static int find_common(struct fetch_pack_args *args,
 261                        int fd[2], unsigned char *result_sha1,
 262                        struct ref *refs)
 263 {
 264         int fetching;
 265         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 266         const unsigned char *sha1;
 267         unsigned in_vain = 0;
 268         int got_continue = 0;
 269         int got_ready = 0;
 270         struct strbuf req_buf = STRBUF_INIT;
 271         size_t state_len = 0;
 272
 273         if (args->stateless_rpc && multi_ack == 1)
 274                 die("--stateless-rpc requires multi_ack_detailed");
 275         if (marked)
 276                 for_each_ref(clear_marks, NULL);
 277         marked = 1;
 278
 279         for_each_ref(rev_list_insert_ref_oid, NULL);
 280         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 281
 282         fetching = 0;
 283         for ( ; refs ; refs = refs->next) {
 284                 unsigned char *remote = refs->old_oid.hash;
 285                 const char *remote_hex;
 286                 struct object *o;
 287
 288                 /*
 289                  * If that object is complete (i.e. it is an ancestor of a
 290                  * local ref), we tell them we have it but do not have to
 291                  * tell them about its ancestors, which they already know
 292                  * about.
 293                  *
 294                  * We use lookup_object here because we are only
 295                  * interested in the case we *know* the object is
 296                  * reachable and we have already scanned it.
 297                  */
 298                 if (((o = lookup_object(remote)) != NULL) &&
 299                                 (o->flags & COMPLETE)) {
 300                         continue;
 301                 }
 302
 303                 remote_hex = sha1_to_hex(remote);
 304                 if (!fetching) {
 305                         struct strbuf c = STRBUF_INIT;
 306                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 307                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 308                         if (no_done)            strbuf_addstr(&c, " no-done");
 309                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 310                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 311                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 312                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 313                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 314                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 315                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 316                                                             git_user_agent_sanitized());
 317                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 318                         strbuf_release(&c);
 319                 } else
 320                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 321                 fetching++;
 322         }
 323
 324         if (!fetching) {
 325                 strbuf_release(&req_buf);
 326                 packet_flush(fd[1]);
 327                 return 1;
 328         }
 329
 330         if (is_repository_shallow())
 331                 write_shallow_commits(&req_buf, 1, NULL);
 332         if (args->depth > 0)
 333                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 334         packet_buf_flush(&req_buf);
 335         state_len = req_buf.len;
 336
 337         if (args->depth > 0) {
 338                 char *line;
 339                 const char *arg;
 340                 unsigned char sha1[20];
 341
 342                 send_request(args, fd[1], &req_buf);
 343                 while ((line = packet_read_line(fd[0], NULL))) {
 344                         if (skip_prefix(line, "shallow ", &arg)) {
 345                                 if (get_sha1_hex(arg, sha1))
 346                                         die("invalid shallow line: %s", line);
 347                                 register_shallow(sha1);
 348                                 continue;
 349                         }
 350                         if (skip_prefix(line, "unshallow ", &arg)) {
 351                                 if (get_sha1_hex(arg, sha1))
 352                                         die("invalid unshallow line: %s", line);
 353                                 if (!lookup_object(sha1))
 354                                         die("object not found: %s", line);
 355                                 /* make sure that it is parsed as shallow */
 356                                 if (!parse_object(sha1))
 357                                         die("error in object: %s", line);
 358                                 if (unregister_shallow(sha1))
 359                                         die("no shallow found: %s", line);
 360                                 continue;
 361                         }
 362                         die("expected shallow/unshallow, got %s", line);
 363                 }
 364         } else if (!args->stateless_rpc)
 365                 send_request(args, fd[1], &req_buf);
 366
 367         if (!args->stateless_rpc) {
 368                 /* If we aren't using the stateless-rpc interface
 369                  * we don't need to retain the headers.
 370                  */
 371                 strbuf_setlen(&req_buf, 0);
 372                 state_len = 0;
 373         }
 374
 375         flushes = 0;
 376         retval = -1;
 377         while ((sha1 = get_rev())) {
 378                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 379                 if (args->verbose)
 380                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 381                 in_vain++;
 382                 if (flush_at <= ++count) {
 383                         int ack;
 384
 385                         packet_buf_flush(&req_buf);
 386                         send_request(args, fd[1], &req_buf);
 387                         strbuf_setlen(&req_buf, state_len);
 388                         flushes++;
 389                         flush_at = next_flush(args, count);
 390
 391                         /*
 392                          * We keep one window "ahead" of the other side, and
 393                          * will wait for an ACK only on the next one
 394                          */
 395                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 396                                 continue;
 397
 398                         consume_shallow_list(args, fd[0]);
 399                         do {
 400                                 ack = get_ack(fd[0], result_sha1);
 401                                 if (args->verbose && ack)
 402                                         fprintf(stderr, "got ack %d %s\n", ack,
 403                                                         sha1_to_hex(result_sha1));
 404                                 switch (ack) {
 405                                 case ACK:
 406                                         flushes = 0;
 407                                         multi_ack = 0;
 408                                         retval = 0;
 409                                         goto done;
 410                                 case ACK_common:
 411                                 case ACK_ready:
 412                                 case ACK_continue: {
 413                                         struct commit *commit =
 414                                                 lookup_commit(result_sha1);
 415                                         if (!commit)
 416                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 417                                         if (args->stateless_rpc
 418                                          && ack == ACK_common
 419                                          && !(commit->object.flags & COMMON)) {
 420                                                 /* We need to replay the have for this object
 421                                                  * on the next RPC request so the peer knows
 422                                                  * it is in common with us.
 423                                                  */
 424                                                 const char *hex = sha1_to_hex(result_sha1);
 425                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 426                                                 state_len = req_buf.len;
 427                                         }
 428                                         mark_common(commit, 0, 1);
 429                                         retval = 0;
 430                                         in_vain = 0;
 431                                         got_continue = 1;
 432                                         if (ack == ACK_ready) {
 433                                                 clear_prio_queue(&rev_list);
 434                                                 got_ready = 1;
 435                                         }
 436                                         break;
 437                                         }
 438                                 }
 439                         } while (ack);
 440                         flushes--;
 441                         if (got_continue && MAX_IN_VAIN < in_vain) {
 442                                 if (args->verbose)
 443                                         fprintf(stderr, "giving up\n");
 444                                 break; /* give up */
 445                         }
 446                 }
 447         }
 448 done:
 449         if (!got_ready || !no_done) {
 450                 packet_buf_write(&req_buf, "done\n");
 451                 send_request(args, fd[1], &req_buf);
 452         }
 453         if (args->verbose)
 454                 fprintf(stderr, "done\n");
 455         if (retval != 0) {
 456                 multi_ack = 0;
 457                 flushes++;
 458         }
 459         strbuf_release(&req_buf);
 460
 461         if (!got_ready || !no_done)
 462                 consume_shallow_list(args, fd[0]);
 463         while (flushes || multi_ack) {
 464                 int ack = get_ack(fd[0], result_sha1);
 465                 if (ack) {
 466                         if (args->verbose)
 467                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 468                                         sha1_to_hex(result_sha1));
 469                         if (ack == ACK)
 470                                 return 0;
 471                         multi_ack = 1;
 472                         continue;
 473                 }
 474                 flushes--;
 475         }
 476         /* it is no error to fetch into a completely empty repo */
 477         return count ? retval : 0;
 478 }
 479
 480 static struct commit_list *complete;
 481
 482 static int mark_complete(const unsigned char *sha1)
 483 {
 484         struct object *o = parse_object(sha1);
 485
 486         while (o && o->type == OBJ_TAG) {
 487                 struct tag *t = (struct tag *) o;
 488                 if (!t->tagged)
 489                         break; /* broken repository */
 490                 o->flags |= COMPLETE;
 491                 o = parse_object(t->tagged->oid.hash);
 492         }
 493         if (o && o->type == OBJ_COMMIT) {
 494                 struct commit *commit = (struct commit *)o;
 495                 if (!(commit->object.flags & COMPLETE)) {
 496                         commit->object.flags |= COMPLETE;
 497                         commit_list_insert(commit, &complete);
 498                 }
 499         }
 500         return 0;
 501 }
 502
 503 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 504                              int flag, void *cb_data)
 505 {
 506         return mark_complete(oid->hash);
 507 }
 508
 509 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 510                                          unsigned long cutoff)
 511 {
 512         while (complete && cutoff <= complete->item->date) {
 513                 if (args->verbose)
 514                         fprintf(stderr, "Marking %s as complete\n",
 515                                 oid_to_hex(&complete->item->object.oid));
 516                 pop_most_recent_commit(&complete, COMPLETE);
 517         }
 518 }
 519
 520 static void filter_refs(struct fetch_pack_args *args,
 521                         struct ref **refs,
 522                         struct ref **sought, int nr_sought)
 523 {
 524         struct ref *newlist = NULL;
 525         struct ref **newtail = &newlist;
 526         struct ref *ref, *next;
 527         int i;
 528
 529         i = 0;
 530         for (ref = *refs; ref; ref = next) {
 531                 int keep = 0;
 532                 next = ref->next;
 533
 534                 if (starts_with(ref->name, "refs/") &&
 535                     check_refname_format(ref->name, 0))
 536                         ; /* trash */
 537                 else {
 538                         while (i < nr_sought) {
 539                                 int cmp = strcmp(ref->name, sought[i]->name);
 540                                 if (cmp < 0)
 541                                         break; /* definitely do not have it */
 542                                 else if (cmp == 0) {
 543                                         keep = 1; /* definitely have it */
 544                                         sought[i]->matched = 1;
 545                                 }
 546                                 i++;
 547                         }
 548                 }
 549
 550                 if (!keep && args->fetch_all &&
 551                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 552                         keep = 1;
 553
 554                 if (keep) {
 555                         *newtail = ref;
 556                         ref->next = NULL;
 557                         newtail = &ref->next;
 558                 } else {
 559                         free(ref);
 560                 }
 561         }
 562
 563         /* Append unmatched requests to the list */
 564         if ((allow_unadvertised_object_request &
 565             (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
 566                 for (i = 0; i < nr_sought; i++) {
 567                         unsigned char sha1[20];
 568
 569                         ref = sought[i];
 570                         if (ref->matched)
 571                                 continue;
 572                         if (get_sha1_hex(ref->name, sha1) ||
 573                             ref->name[40] != '\0' ||
 574                             hashcmp(sha1, ref->old_oid.hash))
 575                                 continue;
 576
 577                         ref->matched = 1;
 578                         *newtail = copy_ref(ref);
 579                         newtail = &(*newtail)->next;
 580                 }
 581         }
 582         *refs = newlist;
 583 }
 584
 585 static void mark_alternate_complete(const struct ref *ref, void *unused)
 586 {
 587         mark_complete(ref->old_oid.hash);
 588 }
 589
 590 static int everything_local(struct fetch_pack_args *args,
 591                             struct ref **refs,
 592                             struct ref **sought, int nr_sought)
 593 {
 594         struct ref *ref;
 595         int retval;
 596         unsigned long cutoff = 0;
 597
 598         save_commit_buffer = 0;
 599
 600         for (ref = *refs; ref; ref = ref->next) {
 601                 struct object *o;
 602
 603                 if (!has_object_file(&ref->old_oid))
 604                         continue;
 605
 606                 o = parse_object(ref->old_oid.hash);
 607                 if (!o)
 608                         continue;
 609
 610                 /* We already have it -- which may mean that we were
 611                  * in sync with the other side at some time after
 612                  * that (it is OK if we guess wrong here).
 613                  */
 614                 if (o->type == OBJ_COMMIT) {
 615                         struct commit *commit = (struct commit *)o;
 616                         if (!cutoff || cutoff < commit->date)
 617                                 cutoff = commit->date;
 618                 }
 619         }
 620
 621         if (!args->depth) {
 622                 for_each_ref(mark_complete_oid, NULL);
 623                 for_each_alternate_ref(mark_alternate_complete, NULL);
 624                 commit_list_sort_by_date(&complete);
 625                 if (cutoff)
 626                         mark_recent_complete_commits(args, cutoff);
 627         }
 628
 629         /*
 630          * Mark all complete remote refs as common refs.
 631          * Don't mark them common yet; the server has to be told so first.
 632          */
 633         for (ref = *refs; ref; ref = ref->next) {
 634                 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 635                                              NULL, 0);
 636
 637                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 638                         continue;
 639
 640                 if (!(o->flags & SEEN)) {
 641                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 642
 643                         mark_common((struct commit *)o, 1, 1);
 644                 }
 645         }
 646
 647         filter_refs(args, refs, sought, nr_sought);
 648
 649         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 650                 const unsigned char *remote = ref->old_oid.hash;
 651                 struct object *o;
 652
 653                 o = lookup_object(remote);
 654                 if (!o || !(o->flags & COMPLETE)) {
 655                         retval = 0;
 656                         if (!args->verbose)
 657                                 continue;
 658                         fprintf(stderr,
 659                                 "want %s (%s)\n", sha1_to_hex(remote),
 660                                 ref->name);
 661                         continue;
 662                 }
 663                 if (!args->verbose)
 664                         continue;
 665                 fprintf(stderr,
 666                         "already have %s (%s)\n", sha1_to_hex(remote),
 667                         ref->name);
 668         }
 669         return retval;
 670 }
 671
 672 static int sideband_demux(int in, int out, void *data)
 673 {
 674         int *xd = data;
 675         int ret;
 676
 677         sigchain_push(SIGPIPE, SIG_IGN);
 678         ret = recv_sideband("fetch-pack", xd[0], out);
 679         close(out);
 680         sigchain_pop(SIGPIPE);
 681         return ret;
 682 }
 683
 684 static int get_pack(struct fetch_pack_args *args,
 685                     int xd[2], char **pack_lockfile)
 686 {
 687         struct async demux;
 688         int do_keep = args->keep_pack;
 689         const char *cmd_name;
 690         struct pack_header header;
 691         int pass_header = 0;
 692         struct child_process cmd = CHILD_PROCESS_INIT;
 693         int ret;
 694
 695         memset(&demux, 0, sizeof(demux));
 696         if (use_sideband) {
 697                 /* xd[] is talking with upload-pack; subprocess reads from
 698                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 699                  * through demux->out.
 700                  */
 701                 demux.proc = sideband_demux;
 702                 demux.data = xd;
 703                 demux.out = -1;
 704                 if (start_async(&demux))
 705                         die("fetch-pack: unable to fork off sideband"
 706                             " demultiplexer");
 707         }
 708         else
 709                 demux.out = xd[0];
 710
 711         if (!args->keep_pack && unpack_limit) {
 712
 713                 if (read_pack_header(demux.out, &header))
 714                         die("protocol error: bad pack header");
 715                 pass_header = 1;
 716                 if (ntohl(header.hdr_entries) < unpack_limit)
 717                         do_keep = 0;
 718                 else
 719                         do_keep = 1;
 720         }
 721
 722         if (alternate_shallow_file) {
 723                 argv_array_push(&cmd.args, "--shallow-file");
 724                 argv_array_push(&cmd.args, alternate_shallow_file);
 725         }
 726
 727         if (do_keep) {
 728                 if (pack_lockfile)
 729                         cmd.out = -1;
 730                 cmd_name = "index-pack";
 731                 argv_array_push(&cmd.args, cmd_name);
 732                 argv_array_push(&cmd.args, "--stdin");
 733                 if (!args->quiet && !args->no_progress)
 734                         argv_array_push(&cmd.args, "-v");
 735                 if (args->use_thin_pack)
 736                         argv_array_push(&cmd.args, "--fix-thin");
 737                 if (args->lock_pack || unpack_limit) {
 738                         char hostname[256];
 739                         if (gethostname(hostname, sizeof(hostname)))
 740                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 741                         argv_array_pushf(&cmd.args,
 742                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 743                                         (uintmax_t)getpid(), hostname);
 744                 }
 745                 if (args->check_self_contained_and_connected)
 746                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 747         }
 748         else {
 749                 cmd_name = "unpack-objects";
 750                 argv_array_push(&cmd.args, cmd_name);
 751                 if (args->quiet || args->no_progress)
 752                         argv_array_push(&cmd.args, "-q");
 753                 args->check_self_contained_and_connected = 0;
 754         }
 755
 756         if (pass_header)
 757                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 758                                  ntohl(header.hdr_version),
 759                                  ntohl(header.hdr_entries));
 760         if (fetch_fsck_objects >= 0
 761             ? fetch_fsck_objects
 762             : transfer_fsck_objects >= 0
 763             ? transfer_fsck_objects
 764             : 0)
 765                 argv_array_push(&cmd.args, "--strict");
 766
 767         cmd.in = demux.out;
 768         cmd.git_cmd = 1;
 769         if (start_command(&cmd))
 770                 die("fetch-pack: unable to fork off %s", cmd_name);
 771         if (do_keep && pack_lockfile) {
 772                 *pack_lockfile = index_pack_lockfile(cmd.out);
 773                 close(cmd.out);
 774         }
 775
 776         if (!use_sideband)
 777                 /* Closed by start_command() */
 778                 xd[0] = -1;
 779
 780         ret = finish_command(&cmd);
 781         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 782                 args->self_contained_and_connected =
 783                         args->check_self_contained_and_connected &&
 784                         ret == 0;
 785         else
 786                 die("%s failed", cmd_name);
 787         if (use_sideband && finish_async(&demux))
 788                 die("error in sideband demultiplexer");
 789         return 0;
 790 }
 791
 792 static int cmp_ref_by_name(const void *a_, const void *b_)
 793 {
 794         const struct ref *a = *((const struct ref **)a_);
 795         const struct ref *b = *((const struct ref **)b_);
 796         return strcmp(a->name, b->name);
 797 }
 798
 799 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 800                                  int fd[2],
 801                                  const struct ref *orig_ref,
 802                                  struct ref **sought, int nr_sought,
 803                                  struct shallow_info *si,
 804                                  char **pack_lockfile)
 805 {
 806         struct ref *ref = copy_ref_list(orig_ref);
 807         unsigned char sha1[20];
 808         const char *agent_feature;
 809         int agent_len;
 810
 811         sort_ref_list(&ref, ref_compare_name);
 812         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 813
 814         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 815                 die("Server does not support shallow clients");
 816         if (server_supports("multi_ack_detailed")) {
 817                 if (args->verbose)
 818                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 819                 multi_ack = 2;
 820                 if (server_supports("no-done")) {
 821                         if (args->verbose)
 822                                 fprintf(stderr, "Server supports no-done\n");
 823                         if (args->stateless_rpc)
 824                                 no_done = 1;
 825                 }
 826         }
 827         else if (server_supports("multi_ack")) {
 828                 if (args->verbose)
 829                         fprintf(stderr, "Server supports multi_ack\n");
 830                 multi_ack = 1;
 831         }
 832         if (server_supports("side-band-64k")) {
 833                 if (args->verbose)
 834                         fprintf(stderr, "Server supports side-band-64k\n");
 835                 use_sideband = 2;
 836         }
 837         else if (server_supports("side-band")) {
 838                 if (args->verbose)
 839                         fprintf(stderr, "Server supports side-band\n");
 840                 use_sideband = 1;
 841         }
 842         if (server_supports("allow-tip-sha1-in-want")) {
 843                 if (args->verbose)
 844                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 845                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 846         }
 847         if (server_supports("allow-reachable-sha1-in-want")) {
 848                 if (args->verbose)
 849                         fprintf(stderr, "Server supports allow-reachable-sha1-in-want\n");
 850                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 851         }
 852         if (!server_supports("thin-pack"))
 853                 args->use_thin_pack = 0;
 854         if (!server_supports("no-progress"))
 855                 args->no_progress = 0;
 856         if (!server_supports("include-tag"))
 857                 args->include_tag = 0;
 858         if (server_supports("ofs-delta")) {
 859                 if (args->verbose)
 860                         fprintf(stderr, "Server supports ofs-delta\n");
 861         } else
 862                 prefer_ofs_delta = 0;
 863
 864         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 865                 agent_supported = 1;
 866                 if (args->verbose && agent_len)
 867                         fprintf(stderr, "Server version is %.*s\n",
 868                                 agent_len, agent_feature);
 869         }
 870
 871         if (everything_local(args, &ref, sought, nr_sought)) {
 872                 packet_flush(fd[1]);
 873                 goto all_done;
 874         }
 875         if (find_common(args, fd, sha1, ref) < 0)
 876                 if (!args->keep_pack)
 877                         /* When cloning, it is not unusual to have
 878                          * no common commit.
 879                          */
 880                         warning("no common commits");
 881
 882         if (args->stateless_rpc)
 883                 packet_flush(fd[1]);
 884         if (args->depth > 0)
 885                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 886                                         NULL);
 887         else if (si->nr_ours || si->nr_theirs)
 888                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 889         else
 890                 alternate_shallow_file = NULL;
 891         if (get_pack(args, fd, pack_lockfile))
 892                 die("git fetch-pack: fetch failed.");
 893
 894  all_done:
 895         return ref;
 896 }
 897
 898 static void fetch_pack_config(void)
 899 {
 900         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 901         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 902         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 903         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 904         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 905
 906         git_config(git_default_config, NULL);
 907 }
 908
 909 static void fetch_pack_setup(void)
 910 {
 911         static int did_setup;
 912         if (did_setup)
 913                 return;
 914         fetch_pack_config();
 915         if (0 <= transfer_unpack_limit)
 916                 unpack_limit = transfer_unpack_limit;
 917         else if (0 <= fetch_unpack_limit)
 918                 unpack_limit = fetch_unpack_limit;
 919         did_setup = 1;
 920 }
 921
 922 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 923 {
 924         struct string_list names = STRING_LIST_INIT_NODUP;
 925         int src, dst;
 926
 927         for (src = dst = 0; src < nr; src++) {
 928                 struct string_list_item *item;
 929                 item = string_list_insert(&names, ref[src]->name);
 930                 if (item->util)
 931                         continue; /* already have it */
 932                 item->util = ref[src];
 933                 if (src != dst)
 934                         ref[dst] = ref[src];
 935                 dst++;
 936         }
 937         for (src = dst; src < nr; src++)
 938                 ref[src] = NULL;
 939         string_list_clear(&names, 0);
 940         return dst;
 941 }
 942
 943 static void update_shallow(struct fetch_pack_args *args,
 944                            struct ref **sought, int nr_sought,
 945                            struct shallow_info *si)
 946 {
 947         struct sha1_array ref = SHA1_ARRAY_INIT;
 948         int *status;
 949         int i;
 950
 951         if (args->depth > 0 && alternate_shallow_file) {
 952                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 953                         unlink_or_warn(git_path_shallow());
 954                         rollback_lock_file(&shallow_lock);
 955                 } else
 956                         commit_lock_file(&shallow_lock);
 957                 return;
 958         }
 959
 960         if (!si->shallow || !si->shallow->nr)
 961                 return;
 962
 963         if (args->cloning) {
 964                 /*
 965                  * remote is shallow, but this is a clone, there are
 966                  * no objects in repo to worry about. Accept any
 967                  * shallow points that exist in the pack (iow in repo
 968                  * after get_pack() and reprepare_packed_git())
 969                  */
 970                 struct sha1_array extra = SHA1_ARRAY_INIT;
 971                 unsigned char (*sha1)[20] = si->shallow->sha1;
 972                 for (i = 0; i < si->shallow->nr; i++)
 973                         if (has_sha1_file(sha1[i]))
 974                                 sha1_array_append(&extra, sha1[i]);
 975                 if (extra.nr) {
 976                         setup_alternate_shallow(&shallow_lock,
 977                                                 &alternate_shallow_file,
 978                                                 &extra);
 979                         commit_lock_file(&shallow_lock);
 980                 }
 981                 sha1_array_clear(&extra);
 982                 return;
 983         }
 984
 985         if (!si->nr_ours && !si->nr_theirs)
 986                 return;
 987
 988         remove_nonexistent_theirs_shallow(si);
 989         if (!si->nr_ours && !si->nr_theirs)
 990                 return;
 991         for (i = 0; i < nr_sought; i++)
 992                 sha1_array_append(&ref, sought[i]->old_oid.hash);
 993         si->ref = &ref;
 994
 995         if (args->update_shallow) {
 996                 /*
 997                  * remote is also shallow, .git/shallow may be updated
 998                  * so all refs can be accepted. Make sure we only add
 999                  * shallow roots that are actually reachable from new
1000                  * refs.
1001                  */
1002                 struct sha1_array extra = SHA1_ARRAY_INIT;
1003                 unsigned char (*sha1)[20] = si->shallow->sha1;
1004                 assign_shallow_commits_to_refs(si, NULL, NULL);
1005                 if (!si->nr_ours && !si->nr_theirs) {
1006                         sha1_array_clear(&ref);
1007                         return;
1008                 }
1009                 for (i = 0; i < si->nr_ours; i++)
1010                         sha1_array_append(&extra, sha1[si->ours[i]]);
1011                 for (i = 0; i < si->nr_theirs; i++)
1012                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1013                 setup_alternate_shallow(&shallow_lock,
1014                                         &alternate_shallow_file,
1015                                         &extra);
1016                 commit_lock_file(&shallow_lock);
1017                 sha1_array_clear(&extra);
1018                 sha1_array_clear(&ref);
1019                 return;
1020         }
1021
1022         /*
1023          * remote is also shallow, check what ref is safe to update
1024          * without updating .git/shallow
1025          */
1026         status = xcalloc(nr_sought, sizeof(*status));
1027         assign_shallow_commits_to_refs(si, NULL, status);
1028         if (si->nr_ours || si->nr_theirs) {
1029                 for (i = 0; i < nr_sought; i++)
1030                         if (status[i])
1031                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1032         }
1033         free(status);
1034         sha1_array_clear(&ref);
1035 }
1036
1037 struct ref *fetch_pack(struct fetch_pack_args *args,
1038                        int fd[], struct child_process *conn,
1039                        const struct ref *ref,
1040                        const char *dest,
1041                        struct ref **sought, int nr_sought,
1042                        struct sha1_array *shallow,
1043                        char **pack_lockfile)
1044 {
1045         struct ref *ref_cpy;
1046         struct shallow_info si;
1047
1048         fetch_pack_setup();
1049         if (nr_sought)
1050                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1051
1052         if (!ref) {
1053                 packet_flush(fd[1]);
1054                 die("no matching remote head");
1055         }
1056         prepare_shallow_info(&si, shallow);
1057         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1058                                 &si, pack_lockfile);
1059         reprepare_packed_git();
1060         update_shallow(args, sought, nr_sought, &si);
1061         clear_shallow_info(&si);
1062         return ref_cpy;
1063 }