fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int fetch_fsck_objects = -1;
  25 static int transfer_fsck_objects = -1;
  26 static int agent_supported;
  27 static struct lock_file shallow_lock;
  28 static const char *alternate_shallow_file;
  29
  30 /* Remember to update object flag allocation in object.h */
  31 #define COMPLETE        (1U << 0)
  32 #define COMMON          (1U << 1)
  33 #define COMMON_REF      (1U << 2)
  34 #define SEEN            (1U << 3)
  35 #define POPPED          (1U << 4)
  36
  37 static int marked;
  38
  39 /*
  40  * After sending this many "have"s if we do not get any new ACK , we
  41  * give up traversing our history.
  42  */
  43 #define MAX_IN_VAIN 256
  44
  45 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  46 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  47
  48 static void rev_list_push(struct commit *commit, int mark)
  49 {
  50         if (!(commit->object.flags & mark)) {
  51                 commit->object.flags |= mark;
  52
  53                 if (parse_commit(commit))
  54                         return;
  55
  56                 prio_queue_put(&rev_list, commit);
  57
  58                 if (!(commit->object.flags & COMMON))
  59                         non_common_revs++;
  60         }
  61 }
  62
  63 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  64 {
  65         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  66
  67         if (o && o->type == OBJ_COMMIT)
  68                 rev_list_push((struct commit *)o, SEEN);
  69
  70         return 0;
  71 }
  72
  73 static int clear_marks(const char *refname, const struct object_id *oid,
  74                        int flag, void *cb_data)
  75 {
  76         struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
  77
  78         if (o && o->type == OBJ_COMMIT)
  79                 clear_commit_marks((struct commit *)o,
  80                                    COMMON | COMMON_REF | SEEN | POPPED);
  81         return 0;
  82 }
  83
  84 /*
  85    This function marks a rev and its ancestors as common.
  86    In some cases, it is desirable to mark only the ancestors (for example
  87    when only the server does not yet know that they are common).
  88 */
  89
  90 static void mark_common(struct commit *commit,
  91                 int ancestors_only, int dont_parse)
  92 {
  93         if (commit != NULL && !(commit->object.flags & COMMON)) {
  94                 struct object *o = (struct object *)commit;
  95
  96                 if (!ancestors_only)
  97                         o->flags |= COMMON;
  98
  99                 if (!(o->flags & SEEN))
 100                         rev_list_push(commit, SEEN);
 101                 else {
 102                         struct commit_list *parents;
 103
 104                         if (!ancestors_only && !(o->flags & POPPED))
 105                                 non_common_revs--;
 106                         if (!o->parsed && !dont_parse)
 107                                 if (parse_commit(commit))
 108                                         return;
 109
 110                         for (parents = commit->parents;
 111                                         parents;
 112                                         parents = parents->next)
 113                                 mark_common(parents->item, 0, dont_parse);
 114                 }
 115         }
 116 }
 117
 118 /*
 119   Get the next rev to send, ignoring the common.
 120 */
 121
 122 static const unsigned char *get_rev(void)
 123 {
 124         struct commit *commit = NULL;
 125
 126         while (commit == NULL) {
 127                 unsigned int mark;
 128                 struct commit_list *parents;
 129
 130                 if (rev_list.nr == 0 || non_common_revs == 0)
 131                         return NULL;
 132
 133                 commit = prio_queue_get(&rev_list);
 134                 parse_commit(commit);
 135                 parents = commit->parents;
 136
 137                 commit->object.flags |= POPPED;
 138                 if (!(commit->object.flags & COMMON))
 139                         non_common_revs--;
 140
 141                 if (commit->object.flags & COMMON) {
 142                         /* do not send "have", and ignore ancestors */
 143                         commit = NULL;
 144                         mark = COMMON | SEEN;
 145                 } else if (commit->object.flags & COMMON_REF)
 146                         /* send "have", and ignore ancestors */
 147                         mark = COMMON | SEEN;
 148                 else
 149                         /* send "have", also for its ancestors */
 150                         mark = SEEN;
 151
 152                 while (parents) {
 153                         if (!(parents->item->object.flags & SEEN))
 154                                 rev_list_push(parents->item, mark);
 155                         if (mark & COMMON)
 156                                 mark_common(parents->item, 1, 0);
 157                         parents = parents->next;
 158                 }
 159         }
 160
 161         return commit->object.sha1;
 162 }
 163
 164 enum ack_type {
 165         NAK = 0,
 166         ACK,
 167         ACK_continue,
 168         ACK_common,
 169         ACK_ready
 170 };
 171
 172 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 173 {
 174         if (args->stateless_rpc && args->depth > 0) {
 175                 /* If we sent a depth we will get back "duplicate"
 176                  * shallow and unshallow commands every time there
 177                  * is a block of have lines exchanged.
 178                  */
 179                 char *line;
 180                 while ((line = packet_read_line(fd, NULL))) {
 181                         if (starts_with(line, "shallow "))
 182                                 continue;
 183                         if (starts_with(line, "unshallow "))
 184                                 continue;
 185                         die("git fetch-pack: expected shallow list");
 186                 }
 187         }
 188 }
 189
 190 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 191 {
 192         int len;
 193         char *line = packet_read_line(fd, &len);
 194         const char *arg;
 195
 196         if (!len)
 197                 die("git fetch-pack: expected ACK/NAK, got EOF");
 198         if (!strcmp(line, "NAK"))
 199                 return NAK;
 200         if (skip_prefix(line, "ACK ", &arg)) {
 201                 if (!get_sha1_hex(arg, result_sha1)) {
 202                         arg += 40;
 203                         len -= arg - line;
 204                         if (len < 1)
 205                                 return ACK;
 206                         if (strstr(arg, "continue"))
 207                                 return ACK_continue;
 208                         if (strstr(arg, "common"))
 209                                 return ACK_common;
 210                         if (strstr(arg, "ready"))
 211                                 return ACK_ready;
 212                         return ACK;
 213                 }
 214         }
 215         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 216 }
 217
 218 static void send_request(struct fetch_pack_args *args,
 219                          int fd, struct strbuf *buf)
 220 {
 221         if (args->stateless_rpc) {
 222                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 223                 packet_flush(fd);
 224         } else
 225                 write_or_die(fd, buf->buf, buf->len);
 226 }
 227
 228 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 229 {
 230         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 231 }
 232
 233 #define INITIAL_FLUSH 16
 234 #define PIPESAFE_FLUSH 32
 235 #define LARGE_FLUSH 1024
 236
 237 static int next_flush(struct fetch_pack_args *args, int count)
 238 {
 239         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 240
 241         if (count < flush_limit)
 242                 count <<= 1;
 243         else
 244                 count += flush_limit;
 245         return count;
 246 }
 247
 248 static int find_common(struct fetch_pack_args *args,
 249                        int fd[2], unsigned char *result_sha1,
 250                        struct ref *refs)
 251 {
 252         int fetching;
 253         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 254         const unsigned char *sha1;
 255         unsigned in_vain = 0;
 256         int got_continue = 0;
 257         int got_ready = 0;
 258         struct strbuf req_buf = STRBUF_INIT;
 259         size_t state_len = 0;
 260         struct each_ref_fn_sha1_adapter wrapped_rev_list_insert_ref =
 261                 {rev_list_insert_ref, NULL};
 262
 263         if (args->stateless_rpc && multi_ack == 1)
 264                 die("--stateless-rpc requires multi_ack_detailed");
 265         if (marked)
 266                 for_each_ref(clear_marks, NULL);
 267         marked = 1;
 268
 269         for_each_ref(each_ref_fn_adapter, &wrapped_rev_list_insert_ref);
 270         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 271
 272         fetching = 0;
 273         for ( ; refs ; refs = refs->next) {
 274                 unsigned char *remote = refs->old_sha1;
 275                 const char *remote_hex;
 276                 struct object *o;
 277
 278                 /*
 279                  * If that object is complete (i.e. it is an ancestor of a
 280                  * local ref), we tell them we have it but do not have to
 281                  * tell them about its ancestors, which they already know
 282                  * about.
 283                  *
 284                  * We use lookup_object here because we are only
 285                  * interested in the case we *know* the object is
 286                  * reachable and we have already scanned it.
 287                  */
 288                 if (((o = lookup_object(remote)) != NULL) &&
 289                                 (o->flags & COMPLETE)) {
 290                         continue;
 291                 }
 292
 293                 remote_hex = sha1_to_hex(remote);
 294                 if (!fetching) {
 295                         struct strbuf c = STRBUF_INIT;
 296                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 297                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 298                         if (no_done)            strbuf_addstr(&c, " no-done");
 299                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 300                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 301                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 302                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 303                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 304                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 305                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 306                                                             git_user_agent_sanitized());
 307                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 308                         strbuf_release(&c);
 309                 } else
 310                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 311                 fetching++;
 312         }
 313
 314         if (!fetching) {
 315                 strbuf_release(&req_buf);
 316                 packet_flush(fd[1]);
 317                 return 1;
 318         }
 319
 320         if (is_repository_shallow())
 321                 write_shallow_commits(&req_buf, 1, NULL);
 322         if (args->depth > 0)
 323                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 324         packet_buf_flush(&req_buf);
 325         state_len = req_buf.len;
 326
 327         if (args->depth > 0) {
 328                 char *line;
 329                 const char *arg;
 330                 unsigned char sha1[20];
 331
 332                 send_request(args, fd[1], &req_buf);
 333                 while ((line = packet_read_line(fd[0], NULL))) {
 334                         if (skip_prefix(line, "shallow ", &arg)) {
 335                                 if (get_sha1_hex(arg, sha1))
 336                                         die("invalid shallow line: %s", line);
 337                                 register_shallow(sha1);
 338                                 continue;
 339                         }
 340                         if (skip_prefix(line, "unshallow ", &arg)) {
 341                                 if (get_sha1_hex(arg, sha1))
 342                                         die("invalid unshallow line: %s", line);
 343                                 if (!lookup_object(sha1))
 344                                         die("object not found: %s", line);
 345                                 /* make sure that it is parsed as shallow */
 346                                 if (!parse_object(sha1))
 347                                         die("error in object: %s", line);
 348                                 if (unregister_shallow(sha1))
 349                                         die("no shallow found: %s", line);
 350                                 continue;
 351                         }
 352                         die("expected shallow/unshallow, got %s", line);
 353                 }
 354         } else if (!args->stateless_rpc)
 355                 send_request(args, fd[1], &req_buf);
 356
 357         if (!args->stateless_rpc) {
 358                 /* If we aren't using the stateless-rpc interface
 359                  * we don't need to retain the headers.
 360                  */
 361                 strbuf_setlen(&req_buf, 0);
 362                 state_len = 0;
 363         }
 364
 365         flushes = 0;
 366         retval = -1;
 367         while ((sha1 = get_rev())) {
 368                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 369                 if (args->verbose)
 370                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 371                 in_vain++;
 372                 if (flush_at <= ++count) {
 373                         int ack;
 374
 375                         packet_buf_flush(&req_buf);
 376                         send_request(args, fd[1], &req_buf);
 377                         strbuf_setlen(&req_buf, state_len);
 378                         flushes++;
 379                         flush_at = next_flush(args, count);
 380
 381                         /*
 382                          * We keep one window "ahead" of the other side, and
 383                          * will wait for an ACK only on the next one
 384                          */
 385                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 386                                 continue;
 387
 388                         consume_shallow_list(args, fd[0]);
 389                         do {
 390                                 ack = get_ack(fd[0], result_sha1);
 391                                 if (args->verbose && ack)
 392                                         fprintf(stderr, "got ack %d %s\n", ack,
 393                                                         sha1_to_hex(result_sha1));
 394                                 switch (ack) {
 395                                 case ACK:
 396                                         flushes = 0;
 397                                         multi_ack = 0;
 398                                         retval = 0;
 399                                         goto done;
 400                                 case ACK_common:
 401                                 case ACK_ready:
 402                                 case ACK_continue: {
 403                                         struct commit *commit =
 404                                                 lookup_commit(result_sha1);
 405                                         if (!commit)
 406                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 407                                         if (args->stateless_rpc
 408                                          && ack == ACK_common
 409                                          && !(commit->object.flags & COMMON)) {
 410                                                 /* We need to replay the have for this object
 411                                                  * on the next RPC request so the peer knows
 412                                                  * it is in common with us.
 413                                                  */
 414                                                 const char *hex = sha1_to_hex(result_sha1);
 415                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 416                                                 state_len = req_buf.len;
 417                                         }
 418                                         mark_common(commit, 0, 1);
 419                                         retval = 0;
 420                                         in_vain = 0;
 421                                         got_continue = 1;
 422                                         if (ack == ACK_ready) {
 423                                                 clear_prio_queue(&rev_list);
 424                                                 got_ready = 1;
 425                                         }
 426                                         break;
 427                                         }
 428                                 }
 429                         } while (ack);
 430                         flushes--;
 431                         if (got_continue && MAX_IN_VAIN < in_vain) {
 432                                 if (args->verbose)
 433                                         fprintf(stderr, "giving up\n");
 434                                 break; /* give up */
 435                         }
 436                 }
 437         }
 438 done:
 439         if (!got_ready || !no_done) {
 440                 packet_buf_write(&req_buf, "done\n");
 441                 send_request(args, fd[1], &req_buf);
 442         }
 443         if (args->verbose)
 444                 fprintf(stderr, "done\n");
 445         if (retval != 0) {
 446                 multi_ack = 0;
 447                 flushes++;
 448         }
 449         strbuf_release(&req_buf);
 450
 451         if (!got_ready || !no_done)
 452                 consume_shallow_list(args, fd[0]);
 453         while (flushes || multi_ack) {
 454                 int ack = get_ack(fd[0], result_sha1);
 455                 if (ack) {
 456                         if (args->verbose)
 457                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 458                                         sha1_to_hex(result_sha1));
 459                         if (ack == ACK)
 460                                 return 0;
 461                         multi_ack = 1;
 462                         continue;
 463                 }
 464                 flushes--;
 465         }
 466         /* it is no error to fetch into a completely empty repo */
 467         return count ? retval : 0;
 468 }
 469
 470 static struct commit_list *complete;
 471
 472 static int mark_complete(const unsigned char *sha1)
 473 {
 474         struct object *o = parse_object(sha1);
 475
 476         while (o && o->type == OBJ_TAG) {
 477                 struct tag *t = (struct tag *) o;
 478                 if (!t->tagged)
 479                         break; /* broken repository */
 480                 o->flags |= COMPLETE;
 481                 o = parse_object(t->tagged->sha1);
 482         }
 483         if (o && o->type == OBJ_COMMIT) {
 484                 struct commit *commit = (struct commit *)o;
 485                 if (!(commit->object.flags & COMPLETE)) {
 486                         commit->object.flags |= COMPLETE;
 487                         commit_list_insert(commit, &complete);
 488                 }
 489         }
 490         return 0;
 491 }
 492
 493 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 494                              int flag, void *cb_data)
 495 {
 496         return mark_complete(oid->hash);
 497 }
 498
 499 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 500                                          unsigned long cutoff)
 501 {
 502         while (complete && cutoff <= complete->item->date) {
 503                 if (args->verbose)
 504                         fprintf(stderr, "Marking %s as complete\n",
 505                                 sha1_to_hex(complete->item->object.sha1));
 506                 pop_most_recent_commit(&complete, COMPLETE);
 507         }
 508 }
 509
 510 static void filter_refs(struct fetch_pack_args *args,
 511                         struct ref **refs,
 512                         struct ref **sought, int nr_sought)
 513 {
 514         struct ref *newlist = NULL;
 515         struct ref **newtail = &newlist;
 516         struct ref *ref, *next;
 517         int i;
 518
 519         i = 0;
 520         for (ref = *refs; ref; ref = next) {
 521                 int keep = 0;
 522                 next = ref->next;
 523
 524                 if (starts_with(ref->name, "refs/") &&
 525                     check_refname_format(ref->name, 0))
 526                         ; /* trash */
 527                 else {
 528                         while (i < nr_sought) {
 529                                 int cmp = strcmp(ref->name, sought[i]->name);
 530                                 if (cmp < 0)
 531                                         break; /* definitely do not have it */
 532                                 else if (cmp == 0) {
 533                                         keep = 1; /* definitely have it */
 534                                         sought[i]->matched = 1;
 535                                 }
 536                                 i++;
 537                         }
 538                 }
 539
 540                 if (!keep && args->fetch_all &&
 541                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 542                         keep = 1;
 543
 544                 if (keep) {
 545                         *newtail = ref;
 546                         ref->next = NULL;
 547                         newtail = &ref->next;
 548                 } else {
 549                         free(ref);
 550                 }
 551         }
 552
 553         /* Append unmatched requests to the list */
 554         if (allow_tip_sha1_in_want) {
 555                 for (i = 0; i < nr_sought; i++) {
 556                         unsigned char sha1[20];
 557
 558                         ref = sought[i];
 559                         if (ref->matched)
 560                                 continue;
 561                         if (get_sha1_hex(ref->name, sha1) ||
 562                             ref->name[40] != '\0' ||
 563                             hashcmp(sha1, ref->old_sha1))
 564                                 continue;
 565
 566                         ref->matched = 1;
 567                         *newtail = copy_ref(ref);
 568                         newtail = &(*newtail)->next;
 569                 }
 570         }
 571         *refs = newlist;
 572 }
 573
 574 static void mark_alternate_complete(const struct ref *ref, void *unused)
 575 {
 576         mark_complete(ref->old_sha1);
 577 }
 578
 579 static int everything_local(struct fetch_pack_args *args,
 580                             struct ref **refs,
 581                             struct ref **sought, int nr_sought)
 582 {
 583         struct ref *ref;
 584         int retval;
 585         unsigned long cutoff = 0;
 586
 587         save_commit_buffer = 0;
 588
 589         for (ref = *refs; ref; ref = ref->next) {
 590                 struct object *o;
 591
 592                 if (!has_sha1_file(ref->old_sha1))
 593                         continue;
 594
 595                 o = parse_object(ref->old_sha1);
 596                 if (!o)
 597                         continue;
 598
 599                 /* We already have it -- which may mean that we were
 600                  * in sync with the other side at some time after
 601                  * that (it is OK if we guess wrong here).
 602                  */
 603                 if (o->type == OBJ_COMMIT) {
 604                         struct commit *commit = (struct commit *)o;
 605                         if (!cutoff || cutoff < commit->date)
 606                                 cutoff = commit->date;
 607                 }
 608         }
 609
 610         if (!args->depth) {
 611                 for_each_ref(mark_complete_oid, NULL);
 612                 for_each_alternate_ref(mark_alternate_complete, NULL);
 613                 commit_list_sort_by_date(&complete);
 614                 if (cutoff)
 615                         mark_recent_complete_commits(args, cutoff);
 616         }
 617
 618         /*
 619          * Mark all complete remote refs as common refs.
 620          * Don't mark them common yet; the server has to be told so first.
 621          */
 622         for (ref = *refs; ref; ref = ref->next) {
 623                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 624                                              NULL, 0);
 625
 626                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 627                         continue;
 628
 629                 if (!(o->flags & SEEN)) {
 630                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 631
 632                         mark_common((struct commit *)o, 1, 1);
 633                 }
 634         }
 635
 636         filter_refs(args, refs, sought, nr_sought);
 637
 638         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 639                 const unsigned char *remote = ref->old_sha1;
 640                 struct object *o;
 641
 642                 o = lookup_object(remote);
 643                 if (!o || !(o->flags & COMPLETE)) {
 644                         retval = 0;
 645                         if (!args->verbose)
 646                                 continue;
 647                         fprintf(stderr,
 648                                 "want %s (%s)\n", sha1_to_hex(remote),
 649                                 ref->name);
 650                         continue;
 651                 }
 652                 if (!args->verbose)
 653                         continue;
 654                 fprintf(stderr,
 655                         "already have %s (%s)\n", sha1_to_hex(remote),
 656                         ref->name);
 657         }
 658         return retval;
 659 }
 660
 661 static int sideband_demux(int in, int out, void *data)
 662 {
 663         int *xd = data;
 664
 665         int ret = recv_sideband("fetch-pack", xd[0], out);
 666         close(out);
 667         return ret;
 668 }
 669
 670 static int get_pack(struct fetch_pack_args *args,
 671                     int xd[2], char **pack_lockfile)
 672 {
 673         struct async demux;
 674         const char *argv[22];
 675         char keep_arg[256];
 676         char hdr_arg[256];
 677         const char **av, *cmd_name;
 678         int do_keep = args->keep_pack;
 679         struct child_process cmd = CHILD_PROCESS_INIT;
 680         int ret;
 681
 682         memset(&demux, 0, sizeof(demux));
 683         if (use_sideband) {
 684                 /* xd[] is talking with upload-pack; subprocess reads from
 685                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 686                  * through demux->out.
 687                  */
 688                 demux.proc = sideband_demux;
 689                 demux.data = xd;
 690                 demux.out = -1;
 691                 if (start_async(&demux))
 692                         die("fetch-pack: unable to fork off sideband"
 693                             " demultiplexer");
 694         }
 695         else
 696                 demux.out = xd[0];
 697
 698         cmd.argv = argv;
 699         av = argv;
 700         *hdr_arg = 0;
 701         if (!args->keep_pack && unpack_limit) {
 702                 struct pack_header header;
 703
 704                 if (read_pack_header(demux.out, &header))
 705                         die("protocol error: bad pack header");
 706                 snprintf(hdr_arg, sizeof(hdr_arg),
 707                          "--pack_header=%"PRIu32",%"PRIu32,
 708                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 709                 if (ntohl(header.hdr_entries) < unpack_limit)
 710                         do_keep = 0;
 711                 else
 712                         do_keep = 1;
 713         }
 714
 715         if (alternate_shallow_file) {
 716                 *av++ = "--shallow-file";
 717                 *av++ = alternate_shallow_file;
 718         }
 719
 720         if (do_keep) {
 721                 if (pack_lockfile)
 722                         cmd.out = -1;
 723                 *av++ = cmd_name = "index-pack";
 724                 *av++ = "--stdin";
 725                 if (!args->quiet && !args->no_progress)
 726                         *av++ = "-v";
 727                 if (args->use_thin_pack)
 728                         *av++ = "--fix-thin";
 729                 if (args->lock_pack || unpack_limit) {
 730                         int s = sprintf(keep_arg,
 731                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 732                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 733                                 strcpy(keep_arg + s, "localhost");
 734                         *av++ = keep_arg;
 735                 }
 736                 if (args->check_self_contained_and_connected)
 737                         *av++ = "--check-self-contained-and-connected";
 738         }
 739         else {
 740                 *av++ = cmd_name = "unpack-objects";
 741                 if (args->quiet || args->no_progress)
 742                         *av++ = "-q";
 743                 args->check_self_contained_and_connected = 0;
 744         }
 745         if (*hdr_arg)
 746                 *av++ = hdr_arg;
 747         if (fetch_fsck_objects >= 0
 748             ? fetch_fsck_objects
 749             : transfer_fsck_objects >= 0
 750             ? transfer_fsck_objects
 751             : 0)
 752                 *av++ = "--strict";
 753         *av++ = NULL;
 754
 755         cmd.in = demux.out;
 756         cmd.git_cmd = 1;
 757         if (start_command(&cmd))
 758                 die("fetch-pack: unable to fork off %s", cmd_name);
 759         if (do_keep && pack_lockfile) {
 760                 *pack_lockfile = index_pack_lockfile(cmd.out);
 761                 close(cmd.out);
 762         }
 763
 764         if (!use_sideband)
 765                 /* Closed by start_command() */
 766                 xd[0] = -1;
 767
 768         ret = finish_command(&cmd);
 769         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 770                 args->self_contained_and_connected =
 771                         args->check_self_contained_and_connected &&
 772                         ret == 0;
 773         else
 774                 die("%s failed", cmd_name);
 775         if (use_sideband && finish_async(&demux))
 776                 die("error in sideband demultiplexer");
 777         return 0;
 778 }
 779
 780 static int cmp_ref_by_name(const void *a_, const void *b_)
 781 {
 782         const struct ref *a = *((const struct ref **)a_);
 783         const struct ref *b = *((const struct ref **)b_);
 784         return strcmp(a->name, b->name);
 785 }
 786
 787 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 788                                  int fd[2],
 789                                  const struct ref *orig_ref,
 790                                  struct ref **sought, int nr_sought,
 791                                  struct shallow_info *si,
 792                                  char **pack_lockfile)
 793 {
 794         struct ref *ref = copy_ref_list(orig_ref);
 795         unsigned char sha1[20];
 796         const char *agent_feature;
 797         int agent_len;
 798
 799         sort_ref_list(&ref, ref_compare_name);
 800         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 801
 802         if (is_repository_shallow() && !server_supports("shallow"))
 803                 die("Server does not support shallow clients");
 804         if (server_supports("multi_ack_detailed")) {
 805                 if (args->verbose)
 806                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 807                 multi_ack = 2;
 808                 if (server_supports("no-done")) {
 809                         if (args->verbose)
 810                                 fprintf(stderr, "Server supports no-done\n");
 811                         if (args->stateless_rpc)
 812                                 no_done = 1;
 813                 }
 814         }
 815         else if (server_supports("multi_ack")) {
 816                 if (args->verbose)
 817                         fprintf(stderr, "Server supports multi_ack\n");
 818                 multi_ack = 1;
 819         }
 820         if (server_supports("side-band-64k")) {
 821                 if (args->verbose)
 822                         fprintf(stderr, "Server supports side-band-64k\n");
 823                 use_sideband = 2;
 824         }
 825         else if (server_supports("side-band")) {
 826                 if (args->verbose)
 827                         fprintf(stderr, "Server supports side-band\n");
 828                 use_sideband = 1;
 829         }
 830         if (server_supports("allow-tip-sha1-in-want")) {
 831                 if (args->verbose)
 832                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 833                 allow_tip_sha1_in_want = 1;
 834         }
 835         if (!server_supports("thin-pack"))
 836                 args->use_thin_pack = 0;
 837         if (!server_supports("no-progress"))
 838                 args->no_progress = 0;
 839         if (!server_supports("include-tag"))
 840                 args->include_tag = 0;
 841         if (server_supports("ofs-delta")) {
 842                 if (args->verbose)
 843                         fprintf(stderr, "Server supports ofs-delta\n");
 844         } else
 845                 prefer_ofs_delta = 0;
 846
 847         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 848                 agent_supported = 1;
 849                 if (args->verbose && agent_len)
 850                         fprintf(stderr, "Server version is %.*s\n",
 851                                 agent_len, agent_feature);
 852         }
 853
 854         if (everything_local(args, &ref, sought, nr_sought)) {
 855                 packet_flush(fd[1]);
 856                 goto all_done;
 857         }
 858         if (find_common(args, fd, sha1, ref) < 0)
 859                 if (!args->keep_pack)
 860                         /* When cloning, it is not unusual to have
 861                          * no common commit.
 862                          */
 863                         warning("no common commits");
 864
 865         if (args->stateless_rpc)
 866                 packet_flush(fd[1]);
 867         if (args->depth > 0)
 868                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 869                                         NULL);
 870         else if (si->nr_ours || si->nr_theirs)
 871                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 872         else
 873                 alternate_shallow_file = NULL;
 874         if (get_pack(args, fd, pack_lockfile))
 875                 die("git fetch-pack: fetch failed.");
 876
 877  all_done:
 878         return ref;
 879 }
 880
 881 static void fetch_pack_config(void)
 882 {
 883         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 884         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 885         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 886         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 887         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 888
 889         git_config(git_default_config, NULL);
 890 }
 891
 892 static void fetch_pack_setup(void)
 893 {
 894         static int did_setup;
 895         if (did_setup)
 896                 return;
 897         fetch_pack_config();
 898         if (0 <= transfer_unpack_limit)
 899                 unpack_limit = transfer_unpack_limit;
 900         else if (0 <= fetch_unpack_limit)
 901                 unpack_limit = fetch_unpack_limit;
 902         did_setup = 1;
 903 }
 904
 905 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 906 {
 907         struct string_list names = STRING_LIST_INIT_NODUP;
 908         int src, dst;
 909
 910         for (src = dst = 0; src < nr; src++) {
 911                 struct string_list_item *item;
 912                 item = string_list_insert(&names, ref[src]->name);
 913                 if (item->util)
 914                         continue; /* already have it */
 915                 item->util = ref[src];
 916                 if (src != dst)
 917                         ref[dst] = ref[src];
 918                 dst++;
 919         }
 920         for (src = dst; src < nr; src++)
 921                 ref[src] = NULL;
 922         string_list_clear(&names, 0);
 923         return dst;
 924 }
 925
 926 static void update_shallow(struct fetch_pack_args *args,
 927                            struct ref **sought, int nr_sought,
 928                            struct shallow_info *si)
 929 {
 930         struct sha1_array ref = SHA1_ARRAY_INIT;
 931         int *status;
 932         int i;
 933
 934         if (args->depth > 0 && alternate_shallow_file) {
 935                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 936                         unlink_or_warn(git_path("shallow"));
 937                         rollback_lock_file(&shallow_lock);
 938                 } else
 939                         commit_lock_file(&shallow_lock);
 940                 return;
 941         }
 942
 943         if (!si->shallow || !si->shallow->nr)
 944                 return;
 945
 946         if (args->cloning) {
 947                 /*
 948                  * remote is shallow, but this is a clone, there are
 949                  * no objects in repo to worry about. Accept any
 950                  * shallow points that exist in the pack (iow in repo
 951                  * after get_pack() and reprepare_packed_git())
 952                  */
 953                 struct sha1_array extra = SHA1_ARRAY_INIT;
 954                 unsigned char (*sha1)[20] = si->shallow->sha1;
 955                 for (i = 0; i < si->shallow->nr; i++)
 956                         if (has_sha1_file(sha1[i]))
 957                                 sha1_array_append(&extra, sha1[i]);
 958                 if (extra.nr) {
 959                         setup_alternate_shallow(&shallow_lock,
 960                                                 &alternate_shallow_file,
 961                                                 &extra);
 962                         commit_lock_file(&shallow_lock);
 963                 }
 964                 sha1_array_clear(&extra);
 965                 return;
 966         }
 967
 968         if (!si->nr_ours && !si->nr_theirs)
 969                 return;
 970
 971         remove_nonexistent_theirs_shallow(si);
 972         if (!si->nr_ours && !si->nr_theirs)
 973                 return;
 974         for (i = 0; i < nr_sought; i++)
 975                 sha1_array_append(&ref, sought[i]->old_sha1);
 976         si->ref = &ref;
 977
 978         if (args->update_shallow) {
 979                 /*
 980                  * remote is also shallow, .git/shallow may be updated
 981                  * so all refs can be accepted. Make sure we only add
 982                  * shallow roots that are actually reachable from new
 983                  * refs.
 984                  */
 985                 struct sha1_array extra = SHA1_ARRAY_INIT;
 986                 unsigned char (*sha1)[20] = si->shallow->sha1;
 987                 assign_shallow_commits_to_refs(si, NULL, NULL);
 988                 if (!si->nr_ours && !si->nr_theirs) {
 989                         sha1_array_clear(&ref);
 990                         return;
 991                 }
 992                 for (i = 0; i < si->nr_ours; i++)
 993                         sha1_array_append(&extra, sha1[si->ours[i]]);
 994                 for (i = 0; i < si->nr_theirs; i++)
 995                         sha1_array_append(&extra, sha1[si->theirs[i]]);
 996                 setup_alternate_shallow(&shallow_lock,
 997                                         &alternate_shallow_file,
 998                                         &extra);
 999                 commit_lock_file(&shallow_lock);
1000                 sha1_array_clear(&extra);
1001                 sha1_array_clear(&ref);
1002                 return;
1003         }
1004
1005         /*
1006          * remote is also shallow, check what ref is safe to update
1007          * without updating .git/shallow
1008          */
1009         status = xcalloc(nr_sought, sizeof(*status));
1010         assign_shallow_commits_to_refs(si, NULL, status);
1011         if (si->nr_ours || si->nr_theirs) {
1012                 for (i = 0; i < nr_sought; i++)
1013                         if (status[i])
1014                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1015         }
1016         free(status);
1017         sha1_array_clear(&ref);
1018 }
1019
1020 struct ref *fetch_pack(struct fetch_pack_args *args,
1021                        int fd[], struct child_process *conn,
1022                        const struct ref *ref,
1023                        const char *dest,
1024                        struct ref **sought, int nr_sought,
1025                        struct sha1_array *shallow,
1026                        char **pack_lockfile)
1027 {
1028         struct ref *ref_cpy;
1029         struct shallow_info si;
1030
1031         fetch_pack_setup();
1032         if (nr_sought)
1033                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1034
1035         if (!ref) {
1036                 packet_flush(fd[1]);
1037                 die("no matching remote head");
1038         }
1039         prepare_shallow_info(&si, shallow);
1040         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1041                                 &si, pack_lockfile);
1042         reprepare_packed_git();
1043         update_shallow(args, sought, nr_sought, &si);
1044         clear_shallow_info(&si);
1045         return ref_cpy;
1046 }