fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int fetch_fsck_objects = -1;
  25 static int transfer_fsck_objects = -1;
  26 static int agent_supported;
  27 static struct lock_file shallow_lock;
  28 static const char *alternate_shallow_file;
  29
  30 /* Remember to update object flag allocation in object.h */
  31 #define COMPLETE        (1U << 0)
  32 #define COMMON          (1U << 1)
  33 #define COMMON_REF      (1U << 2)
  34 #define SEEN            (1U << 3)
  35 #define POPPED          (1U << 4)
  36
  37 static int marked;
  38
  39 /*
  40  * After sending this many "have"s if we do not get any new ACK , we
  41  * give up traversing our history.
  42  */
  43 #define MAX_IN_VAIN 256
  44
  45 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  46 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  47
  48 static void rev_list_push(struct commit *commit, int mark)
  49 {
  50         if (!(commit->object.flags & mark)) {
  51                 commit->object.flags |= mark;
  52
  53                 if (parse_commit(commit))
  54                         return;
  55
  56                 prio_queue_put(&rev_list, commit);
  57
  58                 if (!(commit->object.flags & COMMON))
  59                         non_common_revs++;
  60         }
  61 }
  62
  63 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  64 {
  65         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  66
  67         if (o && o->type == OBJ_COMMIT)
  68                 rev_list_push((struct commit *)o, SEEN);
  69
  70         return 0;
  71 }
  72
  73 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  74 {
  75         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  76
  77         if (o && o->type == OBJ_COMMIT)
  78                 clear_commit_marks((struct commit *)o,
  79                                    COMMON | COMMON_REF | SEEN | POPPED);
  80         return 0;
  81 }
  82
  83 /*
  84    This function marks a rev and its ancestors as common.
  85    In some cases, it is desirable to mark only the ancestors (for example
  86    when only the server does not yet know that they are common).
  87 */
  88
  89 static void mark_common(struct commit *commit,
  90                 int ancestors_only, int dont_parse)
  91 {
  92         if (commit != NULL && !(commit->object.flags & COMMON)) {
  93                 struct object *o = (struct object *)commit;
  94
  95                 if (!ancestors_only)
  96                         o->flags |= COMMON;
  97
  98                 if (!(o->flags & SEEN))
  99                         rev_list_push(commit, SEEN);
 100                 else {
 101                         struct commit_list *parents;
 102
 103                         if (!ancestors_only && !(o->flags & POPPED))
 104                                 non_common_revs--;
 105                         if (!o->parsed && !dont_parse)
 106                                 if (parse_commit(commit))
 107                                         return;
 108
 109                         for (parents = commit->parents;
 110                                         parents;
 111                                         parents = parents->next)
 112                                 mark_common(parents->item, 0, dont_parse);
 113                 }
 114         }
 115 }
 116
 117 /*
 118   Get the next rev to send, ignoring the common.
 119 */
 120
 121 static const unsigned char *get_rev(void)
 122 {
 123         struct commit *commit = NULL;
 124
 125         while (commit == NULL) {
 126                 unsigned int mark;
 127                 struct commit_list *parents;
 128
 129                 if (rev_list.nr == 0 || non_common_revs == 0)
 130                         return NULL;
 131
 132                 commit = prio_queue_get(&rev_list);
 133                 parse_commit(commit);
 134                 parents = commit->parents;
 135
 136                 commit->object.flags |= POPPED;
 137                 if (!(commit->object.flags & COMMON))
 138                         non_common_revs--;
 139
 140                 if (commit->object.flags & COMMON) {
 141                         /* do not send "have", and ignore ancestors */
 142                         commit = NULL;
 143                         mark = COMMON | SEEN;
 144                 } else if (commit->object.flags & COMMON_REF)
 145                         /* send "have", and ignore ancestors */
 146                         mark = COMMON | SEEN;
 147                 else
 148                         /* send "have", also for its ancestors */
 149                         mark = SEEN;
 150
 151                 while (parents) {
 152                         if (!(parents->item->object.flags & SEEN))
 153                                 rev_list_push(parents->item, mark);
 154                         if (mark & COMMON)
 155                                 mark_common(parents->item, 1, 0);
 156                         parents = parents->next;
 157                 }
 158         }
 159
 160         return commit->object.sha1;
 161 }
 162
 163 enum ack_type {
 164         NAK = 0,
 165         ACK,
 166         ACK_continue,
 167         ACK_common,
 168         ACK_ready
 169 };
 170
 171 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 172 {
 173         if (args->stateless_rpc && args->depth > 0) {
 174                 /* If we sent a depth we will get back "duplicate"
 175                  * shallow and unshallow commands every time there
 176                  * is a block of have lines exchanged.
 177                  */
 178                 char *line;
 179                 while ((line = packet_read_line(fd, NULL))) {
 180                         if (starts_with(line, "shallow "))
 181                                 continue;
 182                         if (starts_with(line, "unshallow "))
 183                                 continue;
 184                         die("git fetch-pack: expected shallow list");
 185                 }
 186         }
 187 }
 188
 189 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 190 {
 191         int len;
 192         char *line = packet_read_line(fd, &len);
 193         const char *arg;
 194
 195         if (!len)
 196                 die("git fetch-pack: expected ACK/NAK, got EOF");
 197         if (!strcmp(line, "NAK"))
 198                 return NAK;
 199         if (skip_prefix(line, "ACK ", &arg)) {
 200                 if (!get_sha1_hex(arg, result_sha1)) {
 201                         arg += 40;
 202                         len -= arg - line;
 203                         if (len < 1)
 204                                 return ACK;
 205                         if (strstr(arg, "continue"))
 206                                 return ACK_continue;
 207                         if (strstr(arg, "common"))
 208                                 return ACK_common;
 209                         if (strstr(arg, "ready"))
 210                                 return ACK_ready;
 211                         return ACK;
 212                 }
 213         }
 214         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 215 }
 216
 217 static void send_request(struct fetch_pack_args *args,
 218                          int fd, struct strbuf *buf)
 219 {
 220         if (args->stateless_rpc) {
 221                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 222                 packet_flush(fd);
 223         } else
 224                 write_or_die(fd, buf->buf, buf->len);
 225 }
 226
 227 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 228 {
 229         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 230 }
 231
 232 #define INITIAL_FLUSH 16
 233 #define PIPESAFE_FLUSH 32
 234 #define LARGE_FLUSH 1024
 235
 236 static int next_flush(struct fetch_pack_args *args, int count)
 237 {
 238         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 239
 240         if (count < flush_limit)
 241                 count <<= 1;
 242         else
 243                 count += flush_limit;
 244         return count;
 245 }
 246
 247 static int find_common(struct fetch_pack_args *args,
 248                        int fd[2], unsigned char *result_sha1,
 249                        struct ref *refs)
 250 {
 251         int fetching;
 252         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 253         const unsigned char *sha1;
 254         unsigned in_vain = 0;
 255         int got_continue = 0;
 256         int got_ready = 0;
 257         struct strbuf req_buf = STRBUF_INIT;
 258         size_t state_len = 0;
 259
 260         if (args->stateless_rpc && multi_ack == 1)
 261                 die("--stateless-rpc requires multi_ack_detailed");
 262         if (marked)
 263                 for_each_ref(clear_marks, NULL);
 264         marked = 1;
 265
 266         for_each_ref(rev_list_insert_ref, NULL);
 267         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 268
 269         fetching = 0;
 270         for ( ; refs ; refs = refs->next) {
 271                 unsigned char *remote = refs->old_sha1;
 272                 const char *remote_hex;
 273                 struct object *o;
 274
 275                 /*
 276                  * If that object is complete (i.e. it is an ancestor of a
 277                  * local ref), we tell them we have it but do not have to
 278                  * tell them about its ancestors, which they already know
 279                  * about.
 280                  *
 281                  * We use lookup_object here because we are only
 282                  * interested in the case we *know* the object is
 283                  * reachable and we have already scanned it.
 284                  */
 285                 if (((o = lookup_object(remote)) != NULL) &&
 286                                 (o->flags & COMPLETE)) {
 287                         continue;
 288                 }
 289
 290                 remote_hex = sha1_to_hex(remote);
 291                 if (!fetching) {
 292                         struct strbuf c = STRBUF_INIT;
 293                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 294                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 295                         if (no_done)            strbuf_addstr(&c, " no-done");
 296                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 297                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 298                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 299                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 300                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 301                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 302                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 303                                                             git_user_agent_sanitized());
 304                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 305                         strbuf_release(&c);
 306                 } else
 307                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 308                 fetching++;
 309         }
 310
 311         if (!fetching) {
 312                 strbuf_release(&req_buf);
 313                 packet_flush(fd[1]);
 314                 return 1;
 315         }
 316
 317         if (is_repository_shallow())
 318                 write_shallow_commits(&req_buf, 1, NULL);
 319         if (args->depth > 0)
 320                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 321         packet_buf_flush(&req_buf);
 322         state_len = req_buf.len;
 323
 324         if (args->depth > 0) {
 325                 char *line;
 326                 const char *arg;
 327                 unsigned char sha1[20];
 328
 329                 send_request(args, fd[1], &req_buf);
 330                 while ((line = packet_read_line(fd[0], NULL))) {
 331                         if (skip_prefix(line, "shallow ", &arg)) {
 332                                 if (get_sha1_hex(arg, sha1))
 333                                         die("invalid shallow line: %s", line);
 334                                 register_shallow(sha1);
 335                                 continue;
 336                         }
 337                         if (skip_prefix(line, "unshallow ", &arg)) {
 338                                 if (get_sha1_hex(arg, sha1))
 339                                         die("invalid unshallow line: %s", line);
 340                                 if (!lookup_object(sha1))
 341                                         die("object not found: %s", line);
 342                                 /* make sure that it is parsed as shallow */
 343                                 if (!parse_object(sha1))
 344                                         die("error in object: %s", line);
 345                                 if (unregister_shallow(sha1))
 346                                         die("no shallow found: %s", line);
 347                                 continue;
 348                         }
 349                         die("expected shallow/unshallow, got %s", line);
 350                 }
 351         } else if (!args->stateless_rpc)
 352                 send_request(args, fd[1], &req_buf);
 353
 354         if (!args->stateless_rpc) {
 355                 /* If we aren't using the stateless-rpc interface
 356                  * we don't need to retain the headers.
 357                  */
 358                 strbuf_setlen(&req_buf, 0);
 359                 state_len = 0;
 360         }
 361
 362         flushes = 0;
 363         retval = -1;
 364         while ((sha1 = get_rev())) {
 365                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 366                 if (args->verbose)
 367                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 368                 in_vain++;
 369                 if (flush_at <= ++count) {
 370                         int ack;
 371
 372                         packet_buf_flush(&req_buf);
 373                         send_request(args, fd[1], &req_buf);
 374                         strbuf_setlen(&req_buf, state_len);
 375                         flushes++;
 376                         flush_at = next_flush(args, count);
 377
 378                         /*
 379                          * We keep one window "ahead" of the other side, and
 380                          * will wait for an ACK only on the next one
 381                          */
 382                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 383                                 continue;
 384
 385                         consume_shallow_list(args, fd[0]);
 386                         do {
 387                                 ack = get_ack(fd[0], result_sha1);
 388                                 if (args->verbose && ack)
 389                                         fprintf(stderr, "got ack %d %s\n", ack,
 390                                                         sha1_to_hex(result_sha1));
 391                                 switch (ack) {
 392                                 case ACK:
 393                                         flushes = 0;
 394                                         multi_ack = 0;
 395                                         retval = 0;
 396                                         goto done;
 397                                 case ACK_common:
 398                                 case ACK_ready:
 399                                 case ACK_continue: {
 400                                         struct commit *commit =
 401                                                 lookup_commit(result_sha1);
 402                                         if (!commit)
 403                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 404                                         if (args->stateless_rpc
 405                                          && ack == ACK_common
 406                                          && !(commit->object.flags & COMMON)) {
 407                                                 /* We need to replay the have for this object
 408                                                  * on the next RPC request so the peer knows
 409                                                  * it is in common with us.
 410                                                  */
 411                                                 const char *hex = sha1_to_hex(result_sha1);
 412                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 413                                                 state_len = req_buf.len;
 414                                         }
 415                                         mark_common(commit, 0, 1);
 416                                         retval = 0;
 417                                         in_vain = 0;
 418                                         got_continue = 1;
 419                                         if (ack == ACK_ready) {
 420                                                 clear_prio_queue(&rev_list);
 421                                                 got_ready = 1;
 422                                         }
 423                                         break;
 424                                         }
 425                                 }
 426                         } while (ack);
 427                         flushes--;
 428                         if (got_continue && MAX_IN_VAIN < in_vain) {
 429                                 if (args->verbose)
 430                                         fprintf(stderr, "giving up\n");
 431                                 break; /* give up */
 432                         }
 433                 }
 434         }
 435 done:
 436         if (!got_ready || !no_done) {
 437                 packet_buf_write(&req_buf, "done\n");
 438                 send_request(args, fd[1], &req_buf);
 439         }
 440         if (args->verbose)
 441                 fprintf(stderr, "done\n");
 442         if (retval != 0) {
 443                 multi_ack = 0;
 444                 flushes++;
 445         }
 446         strbuf_release(&req_buf);
 447
 448         if (!got_ready || !no_done)
 449                 consume_shallow_list(args, fd[0]);
 450         while (flushes || multi_ack) {
 451                 int ack = get_ack(fd[0], result_sha1);
 452                 if (ack) {
 453                         if (args->verbose)
 454                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 455                                         sha1_to_hex(result_sha1));
 456                         if (ack == ACK)
 457                                 return 0;
 458                         multi_ack = 1;
 459                         continue;
 460                 }
 461                 flushes--;
 462         }
 463         /* it is no error to fetch into a completely empty repo */
 464         return count ? retval : 0;
 465 }
 466
 467 static struct commit_list *complete;
 468
 469 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 470 {
 471         struct object *o = parse_object(sha1);
 472
 473         while (o && o->type == OBJ_TAG) {
 474                 struct tag *t = (struct tag *) o;
 475                 if (!t->tagged)
 476                         break; /* broken repository */
 477                 o->flags |= COMPLETE;
 478                 o = parse_object(t->tagged->sha1);
 479         }
 480         if (o && o->type == OBJ_COMMIT) {
 481                 struct commit *commit = (struct commit *)o;
 482                 if (!(commit->object.flags & COMPLETE)) {
 483                         commit->object.flags |= COMPLETE;
 484                         commit_list_insert(commit, &complete);
 485                 }
 486         }
 487         return 0;
 488 }
 489
 490 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 491                                          unsigned long cutoff)
 492 {
 493         while (complete && cutoff <= complete->item->date) {
 494                 if (args->verbose)
 495                         fprintf(stderr, "Marking %s as complete\n",
 496                                 sha1_to_hex(complete->item->object.sha1));
 497                 pop_most_recent_commit(&complete, COMPLETE);
 498         }
 499 }
 500
 501 static void filter_refs(struct fetch_pack_args *args,
 502                         struct ref **refs,
 503                         struct ref **sought, int nr_sought)
 504 {
 505         struct ref *newlist = NULL;
 506         struct ref **newtail = &newlist;
 507         struct ref *ref, *next;
 508         int i;
 509
 510         i = 0;
 511         for (ref = *refs; ref; ref = next) {
 512                 int keep = 0;
 513                 next = ref->next;
 514
 515                 if (starts_with(ref->name, "refs/") &&
 516                     check_refname_format(ref->name, 0))
 517                         ; /* trash */
 518                 else {
 519                         while (i < nr_sought) {
 520                                 int cmp = strcmp(ref->name, sought[i]->name);
 521                                 if (cmp < 0)
 522                                         break; /* definitely do not have it */
 523                                 else if (cmp == 0) {
 524                                         keep = 1; /* definitely have it */
 525                                         sought[i]->matched = 1;
 526                                 }
 527                                 i++;
 528                         }
 529                 }
 530
 531                 if (!keep && args->fetch_all &&
 532                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 533                         keep = 1;
 534
 535                 if (keep) {
 536                         *newtail = ref;
 537                         ref->next = NULL;
 538                         newtail = &ref->next;
 539                 } else {
 540                         free(ref);
 541                 }
 542         }
 543
 544         /* Append unmatched requests to the list */
 545         if (allow_tip_sha1_in_want) {
 546                 for (i = 0; i < nr_sought; i++) {
 547                         unsigned char sha1[20];
 548
 549                         ref = sought[i];
 550                         if (ref->matched)
 551                                 continue;
 552                         if (get_sha1_hex(ref->name, sha1) ||
 553                             ref->name[40] != '\0' ||
 554                             hashcmp(sha1, ref->old_sha1))
 555                                 continue;
 556
 557                         ref->matched = 1;
 558                         *newtail = ref;
 559                         ref->next = NULL;
 560                         newtail = &ref->next;
 561                 }
 562         }
 563         *refs = newlist;
 564 }
 565
 566 static void mark_alternate_complete(const struct ref *ref, void *unused)
 567 {
 568         mark_complete(NULL, ref->old_sha1, 0, NULL);
 569 }
 570
 571 static int everything_local(struct fetch_pack_args *args,
 572                             struct ref **refs,
 573                             struct ref **sought, int nr_sought)
 574 {
 575         struct ref *ref;
 576         int retval;
 577         unsigned long cutoff = 0;
 578
 579         save_commit_buffer = 0;
 580
 581         for (ref = *refs; ref; ref = ref->next) {
 582                 struct object *o;
 583
 584                 if (!has_sha1_file(ref->old_sha1))
 585                         continue;
 586
 587                 o = parse_object(ref->old_sha1);
 588                 if (!o)
 589                         continue;
 590
 591                 /* We already have it -- which may mean that we were
 592                  * in sync with the other side at some time after
 593                  * that (it is OK if we guess wrong here).
 594                  */
 595                 if (o->type == OBJ_COMMIT) {
 596                         struct commit *commit = (struct commit *)o;
 597                         if (!cutoff || cutoff < commit->date)
 598                                 cutoff = commit->date;
 599                 }
 600         }
 601
 602         if (!args->depth) {
 603                 for_each_ref(mark_complete, NULL);
 604                 for_each_alternate_ref(mark_alternate_complete, NULL);
 605                 commit_list_sort_by_date(&complete);
 606                 if (cutoff)
 607                         mark_recent_complete_commits(args, cutoff);
 608         }
 609
 610         /*
 611          * Mark all complete remote refs as common refs.
 612          * Don't mark them common yet; the server has to be told so first.
 613          */
 614         for (ref = *refs; ref; ref = ref->next) {
 615                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 616                                              NULL, 0);
 617
 618                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 619                         continue;
 620
 621                 if (!(o->flags & SEEN)) {
 622                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 623
 624                         mark_common((struct commit *)o, 1, 1);
 625                 }
 626         }
 627
 628         filter_refs(args, refs, sought, nr_sought);
 629
 630         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 631                 const unsigned char *remote = ref->old_sha1;
 632                 unsigned char local[20];
 633                 struct object *o;
 634
 635                 o = lookup_object(remote);
 636                 if (!o || !(o->flags & COMPLETE)) {
 637                         retval = 0;
 638                         if (!args->verbose)
 639                                 continue;
 640                         fprintf(stderr,
 641                                 "want %s (%s)\n", sha1_to_hex(remote),
 642                                 ref->name);
 643                         continue;
 644                 }
 645
 646                 hashcpy(ref->new_sha1, local);
 647                 if (!args->verbose)
 648                         continue;
 649                 fprintf(stderr,
 650                         "already have %s (%s)\n", sha1_to_hex(remote),
 651                         ref->name);
 652         }
 653         return retval;
 654 }
 655
 656 static int sideband_demux(int in, int out, void *data)
 657 {
 658         int *xd = data;
 659
 660         int ret = recv_sideband("fetch-pack", xd[0], out);
 661         close(out);
 662         return ret;
 663 }
 664
 665 static int get_pack(struct fetch_pack_args *args,
 666                     int xd[2], char **pack_lockfile)
 667 {
 668         struct async demux;
 669         const char *argv[22];
 670         char keep_arg[256];
 671         char hdr_arg[256];
 672         const char **av, *cmd_name;
 673         int do_keep = args->keep_pack;
 674         struct child_process cmd = CHILD_PROCESS_INIT;
 675         int ret;
 676
 677         memset(&demux, 0, sizeof(demux));
 678         if (use_sideband) {
 679                 /* xd[] is talking with upload-pack; subprocess reads from
 680                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 681                  * through demux->out.
 682                  */
 683                 demux.proc = sideband_demux;
 684                 demux.data = xd;
 685                 demux.out = -1;
 686                 if (start_async(&demux))
 687                         die("fetch-pack: unable to fork off sideband"
 688                             " demultiplexer");
 689         }
 690         else
 691                 demux.out = xd[0];
 692
 693         cmd.argv = argv;
 694         av = argv;
 695         *hdr_arg = 0;
 696         if (!args->keep_pack && unpack_limit) {
 697                 struct pack_header header;
 698
 699                 if (read_pack_header(demux.out, &header))
 700                         die("protocol error: bad pack header");
 701                 snprintf(hdr_arg, sizeof(hdr_arg),
 702                          "--pack_header=%"PRIu32",%"PRIu32,
 703                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 704                 if (ntohl(header.hdr_entries) < unpack_limit)
 705                         do_keep = 0;
 706                 else
 707                         do_keep = 1;
 708         }
 709
 710         if (alternate_shallow_file) {
 711                 *av++ = "--shallow-file";
 712                 *av++ = alternate_shallow_file;
 713         }
 714
 715         if (do_keep) {
 716                 if (pack_lockfile)
 717                         cmd.out = -1;
 718                 *av++ = cmd_name = "index-pack";
 719                 *av++ = "--stdin";
 720                 if (!args->quiet && !args->no_progress)
 721                         *av++ = "-v";
 722                 if (args->use_thin_pack)
 723                         *av++ = "--fix-thin";
 724                 if (args->lock_pack || unpack_limit) {
 725                         int s = sprintf(keep_arg,
 726                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 727                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 728                                 strcpy(keep_arg + s, "localhost");
 729                         *av++ = keep_arg;
 730                 }
 731                 if (args->check_self_contained_and_connected)
 732                         *av++ = "--check-self-contained-and-connected";
 733         }
 734         else {
 735                 *av++ = cmd_name = "unpack-objects";
 736                 if (args->quiet || args->no_progress)
 737                         *av++ = "-q";
 738                 args->check_self_contained_and_connected = 0;
 739         }
 740         if (*hdr_arg)
 741                 *av++ = hdr_arg;
 742         if (fetch_fsck_objects >= 0
 743             ? fetch_fsck_objects
 744             : transfer_fsck_objects >= 0
 745             ? transfer_fsck_objects
 746             : 0)
 747                 *av++ = "--strict";
 748         *av++ = NULL;
 749
 750         cmd.in = demux.out;
 751         cmd.git_cmd = 1;
 752         if (start_command(&cmd))
 753                 die("fetch-pack: unable to fork off %s", cmd_name);
 754         if (do_keep && pack_lockfile) {
 755                 *pack_lockfile = index_pack_lockfile(cmd.out);
 756                 close(cmd.out);
 757         }
 758
 759         if (!use_sideband)
 760                 /* Closed by start_command() */
 761                 xd[0] = -1;
 762
 763         ret = finish_command(&cmd);
 764         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 765                 args->self_contained_and_connected =
 766                         args->check_self_contained_and_connected &&
 767                         ret == 0;
 768         else
 769                 die("%s failed", cmd_name);
 770         if (use_sideband && finish_async(&demux))
 771                 die("error in sideband demultiplexer");
 772         return 0;
 773 }
 774
 775 static int cmp_ref_by_name(const void *a_, const void *b_)
 776 {
 777         const struct ref *a = *((const struct ref **)a_);
 778         const struct ref *b = *((const struct ref **)b_);
 779         return strcmp(a->name, b->name);
 780 }
 781
 782 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 783                                  int fd[2],
 784                                  const struct ref *orig_ref,
 785                                  struct ref **sought, int nr_sought,
 786                                  struct shallow_info *si,
 787                                  char **pack_lockfile)
 788 {
 789         struct ref *ref = copy_ref_list(orig_ref);
 790         unsigned char sha1[20];
 791         const char *agent_feature;
 792         int agent_len;
 793
 794         sort_ref_list(&ref, ref_compare_name);
 795         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 796
 797         if (is_repository_shallow() && !server_supports("shallow"))
 798                 die("Server does not support shallow clients");
 799         if (server_supports("multi_ack_detailed")) {
 800                 if (args->verbose)
 801                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 802                 multi_ack = 2;
 803                 if (server_supports("no-done")) {
 804                         if (args->verbose)
 805                                 fprintf(stderr, "Server supports no-done\n");
 806                         if (args->stateless_rpc)
 807                                 no_done = 1;
 808                 }
 809         }
 810         else if (server_supports("multi_ack")) {
 811                 if (args->verbose)
 812                         fprintf(stderr, "Server supports multi_ack\n");
 813                 multi_ack = 1;
 814         }
 815         if (server_supports("side-band-64k")) {
 816                 if (args->verbose)
 817                         fprintf(stderr, "Server supports side-band-64k\n");
 818                 use_sideband = 2;
 819         }
 820         else if (server_supports("side-band")) {
 821                 if (args->verbose)
 822                         fprintf(stderr, "Server supports side-band\n");
 823                 use_sideband = 1;
 824         }
 825         if (server_supports("allow-tip-sha1-in-want")) {
 826                 if (args->verbose)
 827                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 828                 allow_tip_sha1_in_want = 1;
 829         }
 830         if (!server_supports("thin-pack"))
 831                 args->use_thin_pack = 0;
 832         if (!server_supports("no-progress"))
 833                 args->no_progress = 0;
 834         if (!server_supports("include-tag"))
 835                 args->include_tag = 0;
 836         if (server_supports("ofs-delta")) {
 837                 if (args->verbose)
 838                         fprintf(stderr, "Server supports ofs-delta\n");
 839         } else
 840                 prefer_ofs_delta = 0;
 841
 842         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 843                 agent_supported = 1;
 844                 if (args->verbose && agent_len)
 845                         fprintf(stderr, "Server version is %.*s\n",
 846                                 agent_len, agent_feature);
 847         }
 848
 849         if (everything_local(args, &ref, sought, nr_sought)) {
 850                 packet_flush(fd[1]);
 851                 goto all_done;
 852         }
 853         if (find_common(args, fd, sha1, ref) < 0)
 854                 if (!args->keep_pack)
 855                         /* When cloning, it is not unusual to have
 856                          * no common commit.
 857                          */
 858                         warning("no common commits");
 859
 860         if (args->stateless_rpc)
 861                 packet_flush(fd[1]);
 862         if (args->depth > 0)
 863                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 864                                         NULL);
 865         else if (si->nr_ours || si->nr_theirs)
 866                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 867         else
 868                 alternate_shallow_file = NULL;
 869         if (get_pack(args, fd, pack_lockfile))
 870                 die("git fetch-pack: fetch failed.");
 871
 872  all_done:
 873         return ref;
 874 }
 875
 876 static void fetch_pack_config(void)
 877 {
 878         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 879         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 880         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 881         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 882         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 883
 884         git_config(git_default_config, NULL);
 885 }
 886
 887 static void fetch_pack_setup(void)
 888 {
 889         static int did_setup;
 890         if (did_setup)
 891                 return;
 892         fetch_pack_config();
 893         if (0 <= transfer_unpack_limit)
 894                 unpack_limit = transfer_unpack_limit;
 895         else if (0 <= fetch_unpack_limit)
 896                 unpack_limit = fetch_unpack_limit;
 897         did_setup = 1;
 898 }
 899
 900 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 901 {
 902         struct string_list names = STRING_LIST_INIT_NODUP;
 903         int src, dst;
 904
 905         for (src = dst = 0; src < nr; src++) {
 906                 struct string_list_item *item;
 907                 item = string_list_insert(&names, ref[src]->name);
 908                 if (item->util)
 909                         continue; /* already have it */
 910                 item->util = ref[src];
 911                 if (src != dst)
 912                         ref[dst] = ref[src];
 913                 dst++;
 914         }
 915         for (src = dst; src < nr; src++)
 916                 ref[src] = NULL;
 917         string_list_clear(&names, 0);
 918         return dst;
 919 }
 920
 921 static void update_shallow(struct fetch_pack_args *args,
 922                            struct ref **sought, int nr_sought,
 923                            struct shallow_info *si)
 924 {
 925         struct sha1_array ref = SHA1_ARRAY_INIT;
 926         int *status;
 927         int i;
 928
 929         if (args->depth > 0 && alternate_shallow_file) {
 930                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 931                         unlink_or_warn(git_path("shallow"));
 932                         rollback_lock_file(&shallow_lock);
 933                 } else
 934                         commit_lock_file(&shallow_lock);
 935                 return;
 936         }
 937
 938         if (!si->shallow || !si->shallow->nr)
 939                 return;
 940
 941         if (args->cloning) {
 942                 /*
 943                  * remote is shallow, but this is a clone, there are
 944                  * no objects in repo to worry about. Accept any
 945                  * shallow points that exist in the pack (iow in repo
 946                  * after get_pack() and reprepare_packed_git())
 947                  */
 948                 struct sha1_array extra = SHA1_ARRAY_INIT;
 949                 unsigned char (*sha1)[20] = si->shallow->sha1;
 950                 for (i = 0; i < si->shallow->nr; i++)
 951                         if (has_sha1_file(sha1[i]))
 952                                 sha1_array_append(&extra, sha1[i]);
 953                 if (extra.nr) {
 954                         setup_alternate_shallow(&shallow_lock,
 955                                                 &alternate_shallow_file,
 956                                                 &extra);
 957                         commit_lock_file(&shallow_lock);
 958                 }
 959                 sha1_array_clear(&extra);
 960                 return;
 961         }
 962
 963         if (!si->nr_ours && !si->nr_theirs)
 964                 return;
 965
 966         remove_nonexistent_theirs_shallow(si);
 967         if (!si->nr_ours && !si->nr_theirs)
 968                 return;
 969         for (i = 0; i < nr_sought; i++)
 970                 sha1_array_append(&ref, sought[i]->old_sha1);
 971         si->ref = &ref;
 972
 973         if (args->update_shallow) {
 974                 /*
 975                  * remote is also shallow, .git/shallow may be updated
 976                  * so all refs can be accepted. Make sure we only add
 977                  * shallow roots that are actually reachable from new
 978                  * refs.
 979                  */
 980                 struct sha1_array extra = SHA1_ARRAY_INIT;
 981                 unsigned char (*sha1)[20] = si->shallow->sha1;
 982                 assign_shallow_commits_to_refs(si, NULL, NULL);
 983                 if (!si->nr_ours && !si->nr_theirs) {
 984                         sha1_array_clear(&ref);
 985                         return;
 986                 }
 987                 for (i = 0; i < si->nr_ours; i++)
 988                         sha1_array_append(&extra, sha1[si->ours[i]]);
 989                 for (i = 0; i < si->nr_theirs; i++)
 990                         sha1_array_append(&extra, sha1[si->theirs[i]]);
 991                 setup_alternate_shallow(&shallow_lock,
 992                                         &alternate_shallow_file,
 993                                         &extra);
 994                 commit_lock_file(&shallow_lock);
 995                 sha1_array_clear(&extra);
 996                 sha1_array_clear(&ref);
 997                 return;
 998         }
 999
1000         /*
1001          * remote is also shallow, check what ref is safe to update
1002          * without updating .git/shallow
1003          */
1004         status = xcalloc(nr_sought, sizeof(*status));
1005         assign_shallow_commits_to_refs(si, NULL, status);
1006         if (si->nr_ours || si->nr_theirs) {
1007                 for (i = 0; i < nr_sought; i++)
1008                         if (status[i])
1009                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1010         }
1011         free(status);
1012         sha1_array_clear(&ref);
1013 }
1014
1015 struct ref *fetch_pack(struct fetch_pack_args *args,
1016                        int fd[], struct child_process *conn,
1017                        const struct ref *ref,
1018                        const char *dest,
1019                        struct ref **sought, int nr_sought,
1020                        struct sha1_array *shallow,
1021                        char **pack_lockfile)
1022 {
1023         struct ref *ref_cpy;
1024         struct shallow_info si;
1025
1026         fetch_pack_setup();
1027         if (nr_sought)
1028                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1029
1030         if (!ref) {
1031                 packet_flush(fd[1]);
1032                 die("no matching remote head");
1033         }
1034         prepare_shallow_info(&si, shallow);
1035         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1036                                 &si, pack_lockfile);
1037         reprepare_packed_git();
1038         update_shallow(args, sought, nr_sought, &si);
1039         clear_shallow_info(&si);
1040         return ref_cpy;
1041 }