fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int fetch_fsck_objects = -1;
  25 static int transfer_fsck_objects = -1;
  26 static int agent_supported;
  27 static struct lock_file shallow_lock;
  28 static const char *alternate_shallow_file;
  29
  30 /* Remember to update object flag allocation in object.h */
  31 #define COMPLETE        (1U << 0)
  32 #define COMMON          (1U << 1)
  33 #define COMMON_REF      (1U << 2)
  34 #define SEEN            (1U << 3)
  35 #define POPPED          (1U << 4)
  36
  37 static int marked;
  38
  39 /*
  40  * After sending this many "have"s if we do not get any new ACK , we
  41  * give up traversing our history.
  42  */
  43 #define MAX_IN_VAIN 256
  44
  45 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  46 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  47
  48 static void rev_list_push(struct commit *commit, int mark)
  49 {
  50         if (!(commit->object.flags & mark)) {
  51                 commit->object.flags |= mark;
  52
  53                 if (parse_commit(commit))
  54                         return;
  55
  56                 prio_queue_put(&rev_list, commit);
  57
  58                 if (!(commit->object.flags & COMMON))
  59                         non_common_revs++;
  60         }
  61 }
  62
  63 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  64 {
  65         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  66
  67         if (o && o->type == OBJ_COMMIT)
  68                 rev_list_push((struct commit *)o, SEEN);
  69
  70         return 0;
  71 }
  72
  73 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  74 {
  75         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  76
  77         if (o && o->type == OBJ_COMMIT)
  78                 clear_commit_marks((struct commit *)o,
  79                                    COMMON | COMMON_REF | SEEN | POPPED);
  80         return 0;
  81 }
  82
  83 /*
  84    This function marks a rev and its ancestors as common.
  85    In some cases, it is desirable to mark only the ancestors (for example
  86    when only the server does not yet know that they are common).
  87 */
  88
  89 static void mark_common(struct commit *commit,
  90                 int ancestors_only, int dont_parse)
  91 {
  92         if (commit != NULL && !(commit->object.flags & COMMON)) {
  93                 struct object *o = (struct object *)commit;
  94
  95                 if (!ancestors_only)
  96                         o->flags |= COMMON;
  97
  98                 if (!(o->flags & SEEN))
  99                         rev_list_push(commit, SEEN);
 100                 else {
 101                         struct commit_list *parents;
 102
 103                         if (!ancestors_only && !(o->flags & POPPED))
 104                                 non_common_revs--;
 105                         if (!o->parsed && !dont_parse)
 106                                 if (parse_commit(commit))
 107                                         return;
 108
 109                         for (parents = commit->parents;
 110                                         parents;
 111                                         parents = parents->next)
 112                                 mark_common(parents->item, 0, dont_parse);
 113                 }
 114         }
 115 }
 116
 117 /*
 118   Get the next rev to send, ignoring the common.
 119 */
 120
 121 static const unsigned char *get_rev(void)
 122 {
 123         struct commit *commit = NULL;
 124
 125         while (commit == NULL) {
 126                 unsigned int mark;
 127                 struct commit_list *parents;
 128
 129                 if (rev_list.nr == 0 || non_common_revs == 0)
 130                         return NULL;
 131
 132                 commit = prio_queue_get(&rev_list);
 133                 parse_commit(commit);
 134                 parents = commit->parents;
 135
 136                 commit->object.flags |= POPPED;
 137                 if (!(commit->object.flags & COMMON))
 138                         non_common_revs--;
 139
 140                 if (commit->object.flags & COMMON) {
 141                         /* do not send "have", and ignore ancestors */
 142                         commit = NULL;
 143                         mark = COMMON | SEEN;
 144                 } else if (commit->object.flags & COMMON_REF)
 145                         /* send "have", and ignore ancestors */
 146                         mark = COMMON | SEEN;
 147                 else
 148                         /* send "have", also for its ancestors */
 149                         mark = SEEN;
 150
 151                 while (parents) {
 152                         if (!(parents->item->object.flags & SEEN))
 153                                 rev_list_push(parents->item, mark);
 154                         if (mark & COMMON)
 155                                 mark_common(parents->item, 1, 0);
 156                         parents = parents->next;
 157                 }
 158         }
 159
 160         return commit->object.sha1;
 161 }
 162
 163 enum ack_type {
 164         NAK = 0,
 165         ACK,
 166         ACK_continue,
 167         ACK_common,
 168         ACK_ready
 169 };
 170
 171 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 172 {
 173         if (args->stateless_rpc && args->depth > 0) {
 174                 /* If we sent a depth we will get back "duplicate"
 175                  * shallow and unshallow commands every time there
 176                  * is a block of have lines exchanged.
 177                  */
 178                 char *line;
 179                 while ((line = packet_read_line(fd, NULL))) {
 180                         if (starts_with(line, "shallow "))
 181                                 continue;
 182                         if (starts_with(line, "unshallow "))
 183                                 continue;
 184                         die("git fetch-pack: expected shallow list");
 185                 }
 186         }
 187 }
 188
 189 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 190 {
 191         int len;
 192         char *line = packet_read_line(fd, &len);
 193         const char *arg;
 194
 195         if (!len)
 196                 die("git fetch-pack: expected ACK/NAK, got EOF");
 197         if (!strcmp(line, "NAK"))
 198                 return NAK;
 199         if (skip_prefix(line, "ACK ", &arg)) {
 200                 if (!get_sha1_hex(arg, result_sha1)) {
 201                         arg += 40;
 202                         len -= arg - line;
 203                         if (len < 1)
 204                                 return ACK;
 205                         if (strstr(arg, "continue"))
 206                                 return ACK_continue;
 207                         if (strstr(arg, "common"))
 208                                 return ACK_common;
 209                         if (strstr(arg, "ready"))
 210                                 return ACK_ready;
 211                         return ACK;
 212                 }
 213         }
 214         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 215 }
 216
 217 static void send_request(struct fetch_pack_args *args,
 218                          int fd, struct strbuf *buf)
 219 {
 220         if (args->stateless_rpc) {
 221                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 222                 packet_flush(fd);
 223         } else
 224                 write_or_die(fd, buf->buf, buf->len);
 225 }
 226
 227 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 228 {
 229         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 230 }
 231
 232 #define INITIAL_FLUSH 16
 233 #define PIPESAFE_FLUSH 32
 234 #define LARGE_FLUSH 1024
 235
 236 static int next_flush(struct fetch_pack_args *args, int count)
 237 {
 238         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 239
 240         if (count < flush_limit)
 241                 count <<= 1;
 242         else
 243                 count += flush_limit;
 244         return count;
 245 }
 246
 247 static int find_common(struct fetch_pack_args *args,
 248                        int fd[2], unsigned char *result_sha1,
 249                        struct ref *refs)
 250 {
 251         int fetching;
 252         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 253         const unsigned char *sha1;
 254         unsigned in_vain = 0;
 255         int got_continue = 0;
 256         int got_ready = 0;
 257         struct strbuf req_buf = STRBUF_INIT;
 258         size_t state_len = 0;
 259
 260         if (args->stateless_rpc && multi_ack == 1)
 261                 die("--stateless-rpc requires multi_ack_detailed");
 262         if (marked)
 263                 for_each_ref(clear_marks, NULL);
 264         marked = 1;
 265
 266         for_each_ref(rev_list_insert_ref, NULL);
 267         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 268
 269         fetching = 0;
 270         for ( ; refs ; refs = refs->next) {
 271                 unsigned char *remote = refs->old_sha1;
 272                 const char *remote_hex;
 273                 struct object *o;
 274
 275                 /*
 276                  * If that object is complete (i.e. it is an ancestor of a
 277                  * local ref), we tell them we have it but do not have to
 278                  * tell them about its ancestors, which they already know
 279                  * about.
 280                  *
 281                  * We use lookup_object here because we are only
 282                  * interested in the case we *know* the object is
 283                  * reachable and we have already scanned it.
 284                  */
 285                 if (((o = lookup_object(remote)) != NULL) &&
 286                                 (o->flags & COMPLETE)) {
 287                         continue;
 288                 }
 289
 290                 remote_hex = sha1_to_hex(remote);
 291                 if (!fetching) {
 292                         struct strbuf c = STRBUF_INIT;
 293                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 294                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 295                         if (no_done)            strbuf_addstr(&c, " no-done");
 296                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 297                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 298                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 299                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 300                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 301                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 302                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 303                                                             git_user_agent_sanitized());
 304                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 305                         strbuf_release(&c);
 306                 } else
 307                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 308                 fetching++;
 309         }
 310
 311         if (!fetching) {
 312                 strbuf_release(&req_buf);
 313                 packet_flush(fd[1]);
 314                 return 1;
 315         }
 316
 317         if (is_repository_shallow())
 318                 write_shallow_commits(&req_buf, 1, NULL);
 319         if (args->depth > 0)
 320                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 321         packet_buf_flush(&req_buf);
 322         state_len = req_buf.len;
 323
 324         if (args->depth > 0) {
 325                 char *line;
 326                 const char *arg;
 327                 unsigned char sha1[20];
 328
 329                 send_request(args, fd[1], &req_buf);
 330                 while ((line = packet_read_line(fd[0], NULL))) {
 331                         if (skip_prefix(line, "shallow ", &arg)) {
 332                                 if (get_sha1_hex(arg, sha1))
 333                                         die("invalid shallow line: %s", line);
 334                                 register_shallow(sha1);
 335                                 continue;
 336                         }
 337                         if (skip_prefix(line, "unshallow ", &arg)) {
 338                                 if (get_sha1_hex(arg, sha1))
 339                                         die("invalid unshallow line: %s", line);
 340                                 if (!lookup_object(sha1))
 341                                         die("object not found: %s", line);
 342                                 /* make sure that it is parsed as shallow */
 343                                 if (!parse_object(sha1))
 344                                         die("error in object: %s", line);
 345                                 if (unregister_shallow(sha1))
 346                                         die("no shallow found: %s", line);
 347                                 continue;
 348                         }
 349                         die("expected shallow/unshallow, got %s", line);
 350                 }
 351         } else if (!args->stateless_rpc)
 352                 send_request(args, fd[1], &req_buf);
 353
 354         if (!args->stateless_rpc) {
 355                 /* If we aren't using the stateless-rpc interface
 356                  * we don't need to retain the headers.
 357                  */
 358                 strbuf_setlen(&req_buf, 0);
 359                 state_len = 0;
 360         }
 361
 362         flushes = 0;
 363         retval = -1;
 364         while ((sha1 = get_rev())) {
 365                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 366                 if (args->verbose)
 367                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 368                 in_vain++;
 369                 if (flush_at <= ++count) {
 370                         int ack;
 371
 372                         packet_buf_flush(&req_buf);
 373                         send_request(args, fd[1], &req_buf);
 374                         strbuf_setlen(&req_buf, state_len);
 375                         flushes++;
 376                         flush_at = next_flush(args, count);
 377
 378                         /*
 379                          * We keep one window "ahead" of the other side, and
 380                          * will wait for an ACK only on the next one
 381                          */
 382                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 383                                 continue;
 384
 385                         consume_shallow_list(args, fd[0]);
 386                         do {
 387                                 ack = get_ack(fd[0], result_sha1);
 388                                 if (args->verbose && ack)
 389                                         fprintf(stderr, "got ack %d %s\n", ack,
 390                                                         sha1_to_hex(result_sha1));
 391                                 switch (ack) {
 392                                 case ACK:
 393                                         flushes = 0;
 394                                         multi_ack = 0;
 395                                         retval = 0;
 396                                         goto done;
 397                                 case ACK_common:
 398                                 case ACK_ready:
 399                                 case ACK_continue: {
 400                                         struct commit *commit =
 401                                                 lookup_commit(result_sha1);
 402                                         if (!commit)
 403                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 404                                         if (args->stateless_rpc
 405                                          && ack == ACK_common
 406                                          && !(commit->object.flags & COMMON)) {
 407                                                 /* We need to replay the have for this object
 408                                                  * on the next RPC request so the peer knows
 409                                                  * it is in common with us.
 410                                                  */
 411                                                 const char *hex = sha1_to_hex(result_sha1);
 412                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 413                                                 state_len = req_buf.len;
 414                                         }
 415                                         mark_common(commit, 0, 1);
 416                                         retval = 0;
 417                                         in_vain = 0;
 418                                         got_continue = 1;
 419                                         if (ack == ACK_ready) {
 420                                                 clear_prio_queue(&rev_list);
 421                                                 got_ready = 1;
 422                                         }
 423                                         break;
 424                                         }
 425                                 }
 426                         } while (ack);
 427                         flushes--;
 428                         if (got_continue && MAX_IN_VAIN < in_vain) {
 429                                 if (args->verbose)
 430                                         fprintf(stderr, "giving up\n");
 431                                 break; /* give up */
 432                         }
 433                 }
 434         }
 435 done:
 436         if (!got_ready || !no_done) {
 437                 packet_buf_write(&req_buf, "done\n");
 438                 send_request(args, fd[1], &req_buf);
 439         }
 440         if (args->verbose)
 441                 fprintf(stderr, "done\n");
 442         if (retval != 0) {
 443                 multi_ack = 0;
 444                 flushes++;
 445         }
 446         strbuf_release(&req_buf);
 447
 448         if (!got_ready || !no_done)
 449                 consume_shallow_list(args, fd[0]);
 450         while (flushes || multi_ack) {
 451                 int ack = get_ack(fd[0], result_sha1);
 452                 if (ack) {
 453                         if (args->verbose)
 454                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 455                                         sha1_to_hex(result_sha1));
 456                         if (ack == ACK)
 457                                 return 0;
 458                         multi_ack = 1;
 459                         continue;
 460                 }
 461                 flushes--;
 462         }
 463         /* it is no error to fetch into a completely empty repo */
 464         return count ? retval : 0;
 465 }
 466
 467 static struct commit_list *complete;
 468
 469 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 470 {
 471         struct object *o = parse_object(sha1);
 472
 473         while (o && o->type == OBJ_TAG) {
 474                 struct tag *t = (struct tag *) o;
 475                 if (!t->tagged)
 476                         break; /* broken repository */
 477                 o->flags |= COMPLETE;
 478                 o = parse_object(t->tagged->sha1);
 479         }
 480         if (o && o->type == OBJ_COMMIT) {
 481                 struct commit *commit = (struct commit *)o;
 482                 if (!(commit->object.flags & COMPLETE)) {
 483                         commit->object.flags |= COMPLETE;
 484                         commit_list_insert(commit, &complete);
 485                 }
 486         }
 487         return 0;
 488 }
 489
 490 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 491                                          unsigned long cutoff)
 492 {
 493         while (complete && cutoff <= complete->item->date) {
 494                 if (args->verbose)
 495                         fprintf(stderr, "Marking %s as complete\n",
 496                                 sha1_to_hex(complete->item->object.sha1));
 497                 pop_most_recent_commit(&complete, COMPLETE);
 498         }
 499 }
 500
 501 static void filter_refs(struct fetch_pack_args *args,
 502                         struct ref **refs,
 503                         struct ref **sought, int nr_sought)
 504 {
 505         struct ref *newlist = NULL;
 506         struct ref **newtail = &newlist;
 507         struct ref *ref, *next;
 508         int i;
 509
 510         i = 0;
 511         for (ref = *refs; ref; ref = next) {
 512                 int keep = 0;
 513                 next = ref->next;
 514
 515                 if (starts_with(ref->name, "refs/") &&
 516                     check_refname_format(ref->name, 0))
 517                         ; /* trash */
 518                 else {
 519                         while (i < nr_sought) {
 520                                 int cmp = strcmp(ref->name, sought[i]->name);
 521                                 if (cmp < 0)
 522                                         break; /* definitely do not have it */
 523                                 else if (cmp == 0) {
 524                                         keep = 1; /* definitely have it */
 525                                         sought[i]->matched = 1;
 526                                 }
 527                                 i++;
 528                         }
 529                 }
 530
 531                 if (!keep && args->fetch_all &&
 532                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 533                         keep = 1;
 534
 535                 if (keep) {
 536                         *newtail = ref;
 537                         ref->next = NULL;
 538                         newtail = &ref->next;
 539                 } else {
 540                         free(ref);
 541                 }
 542         }
 543
 544         /* Append unmatched requests to the list */
 545         if (allow_tip_sha1_in_want) {
 546                 for (i = 0; i < nr_sought; i++) {
 547                         ref = sought[i];
 548                         if (ref->matched)
 549                                 continue;
 550                         if (get_sha1_hex(ref->name, ref->old_sha1))
 551                                 continue;
 552
 553                         ref->matched = 1;
 554                         *newtail = ref;
 555                         ref->next = NULL;
 556                         newtail = &ref->next;
 557                 }
 558         }
 559         *refs = newlist;
 560 }
 561
 562 static void mark_alternate_complete(const struct ref *ref, void *unused)
 563 {
 564         mark_complete(NULL, ref->old_sha1, 0, NULL);
 565 }
 566
 567 static int everything_local(struct fetch_pack_args *args,
 568                             struct ref **refs,
 569                             struct ref **sought, int nr_sought)
 570 {
 571         struct ref *ref;
 572         int retval;
 573         unsigned long cutoff = 0;
 574
 575         save_commit_buffer = 0;
 576
 577         for (ref = *refs; ref; ref = ref->next) {
 578                 struct object *o;
 579
 580                 if (!has_sha1_file(ref->old_sha1))
 581                         continue;
 582
 583                 o = parse_object(ref->old_sha1);
 584                 if (!o)
 585                         continue;
 586
 587                 /* We already have it -- which may mean that we were
 588                  * in sync with the other side at some time after
 589                  * that (it is OK if we guess wrong here).
 590                  */
 591                 if (o->type == OBJ_COMMIT) {
 592                         struct commit *commit = (struct commit *)o;
 593                         if (!cutoff || cutoff < commit->date)
 594                                 cutoff = commit->date;
 595                 }
 596         }
 597
 598         if (!args->depth) {
 599                 for_each_ref(mark_complete, NULL);
 600                 for_each_alternate_ref(mark_alternate_complete, NULL);
 601                 commit_list_sort_by_date(&complete);
 602                 if (cutoff)
 603                         mark_recent_complete_commits(args, cutoff);
 604         }
 605
 606         /*
 607          * Mark all complete remote refs as common refs.
 608          * Don't mark them common yet; the server has to be told so first.
 609          */
 610         for (ref = *refs; ref; ref = ref->next) {
 611                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 612                                              NULL, 0);
 613
 614                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 615                         continue;
 616
 617                 if (!(o->flags & SEEN)) {
 618                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 619
 620                         mark_common((struct commit *)o, 1, 1);
 621                 }
 622         }
 623
 624         filter_refs(args, refs, sought, nr_sought);
 625
 626         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 627                 const unsigned char *remote = ref->old_sha1;
 628                 unsigned char local[20];
 629                 struct object *o;
 630
 631                 o = lookup_object(remote);
 632                 if (!o || !(o->flags & COMPLETE)) {
 633                         retval = 0;
 634                         if (!args->verbose)
 635                                 continue;
 636                         fprintf(stderr,
 637                                 "want %s (%s)\n", sha1_to_hex(remote),
 638                                 ref->name);
 639                         continue;
 640                 }
 641
 642                 hashcpy(ref->new_sha1, local);
 643                 if (!args->verbose)
 644                         continue;
 645                 fprintf(stderr,
 646                         "already have %s (%s)\n", sha1_to_hex(remote),
 647                         ref->name);
 648         }
 649         return retval;
 650 }
 651
 652 static int sideband_demux(int in, int out, void *data)
 653 {
 654         int *xd = data;
 655
 656         int ret = recv_sideband("fetch-pack", xd[0], out);
 657         close(out);
 658         return ret;
 659 }
 660
 661 static int get_pack(struct fetch_pack_args *args,
 662                     int xd[2], char **pack_lockfile)
 663 {
 664         struct async demux;
 665         const char *argv[22];
 666         char keep_arg[256];
 667         char hdr_arg[256];
 668         const char **av, *cmd_name;
 669         int do_keep = args->keep_pack;
 670         struct child_process cmd = CHILD_PROCESS_INIT;
 671         int ret;
 672
 673         memset(&demux, 0, sizeof(demux));
 674         if (use_sideband) {
 675                 /* xd[] is talking with upload-pack; subprocess reads from
 676                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 677                  * through demux->out.
 678                  */
 679                 demux.proc = sideband_demux;
 680                 demux.data = xd;
 681                 demux.out = -1;
 682                 if (start_async(&demux))
 683                         die("fetch-pack: unable to fork off sideband"
 684                             " demultiplexer");
 685         }
 686         else
 687                 demux.out = xd[0];
 688
 689         cmd.argv = argv;
 690         av = argv;
 691         *hdr_arg = 0;
 692         if (!args->keep_pack && unpack_limit) {
 693                 struct pack_header header;
 694
 695                 if (read_pack_header(demux.out, &header))
 696                         die("protocol error: bad pack header");
 697                 snprintf(hdr_arg, sizeof(hdr_arg),
 698                          "--pack_header=%"PRIu32",%"PRIu32,
 699                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 700                 if (ntohl(header.hdr_entries) < unpack_limit)
 701                         do_keep = 0;
 702                 else
 703                         do_keep = 1;
 704         }
 705
 706         if (alternate_shallow_file) {
 707                 *av++ = "--shallow-file";
 708                 *av++ = alternate_shallow_file;
 709         }
 710
 711         if (do_keep) {
 712                 if (pack_lockfile)
 713                         cmd.out = -1;
 714                 *av++ = cmd_name = "index-pack";
 715                 *av++ = "--stdin";
 716                 if (!args->quiet && !args->no_progress)
 717                         *av++ = "-v";
 718                 if (args->use_thin_pack)
 719                         *av++ = "--fix-thin";
 720                 if (args->lock_pack || unpack_limit) {
 721                         int s = sprintf(keep_arg,
 722                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 723                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 724                                 strcpy(keep_arg + s, "localhost");
 725                         *av++ = keep_arg;
 726                 }
 727                 if (args->check_self_contained_and_connected)
 728                         *av++ = "--check-self-contained-and-connected";
 729         }
 730         else {
 731                 *av++ = cmd_name = "unpack-objects";
 732                 if (args->quiet || args->no_progress)
 733                         *av++ = "-q";
 734                 args->check_self_contained_and_connected = 0;
 735         }
 736         if (*hdr_arg)
 737                 *av++ = hdr_arg;
 738         if (fetch_fsck_objects >= 0
 739             ? fetch_fsck_objects
 740             : transfer_fsck_objects >= 0
 741             ? transfer_fsck_objects
 742             : 0)
 743                 *av++ = "--strict";
 744         *av++ = NULL;
 745
 746         cmd.in = demux.out;
 747         cmd.git_cmd = 1;
 748         if (start_command(&cmd))
 749                 die("fetch-pack: unable to fork off %s", cmd_name);
 750         if (do_keep && pack_lockfile) {
 751                 *pack_lockfile = index_pack_lockfile(cmd.out);
 752                 close(cmd.out);
 753         }
 754
 755         if (!use_sideband)
 756                 /* Closed by start_command() */
 757                 xd[0] = -1;
 758
 759         ret = finish_command(&cmd);
 760         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 761                 args->self_contained_and_connected =
 762                         args->check_self_contained_and_connected &&
 763                         ret == 0;
 764         else
 765                 die("%s failed", cmd_name);
 766         if (use_sideband && finish_async(&demux))
 767                 die("error in sideband demultiplexer");
 768         return 0;
 769 }
 770
 771 static int cmp_ref_by_name(const void *a_, const void *b_)
 772 {
 773         const struct ref *a = *((const struct ref **)a_);
 774         const struct ref *b = *((const struct ref **)b_);
 775         return strcmp(a->name, b->name);
 776 }
 777
 778 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 779                                  int fd[2],
 780                                  const struct ref *orig_ref,
 781                                  struct ref **sought, int nr_sought,
 782                                  struct shallow_info *si,
 783                                  char **pack_lockfile)
 784 {
 785         struct ref *ref = copy_ref_list(orig_ref);
 786         unsigned char sha1[20];
 787         const char *agent_feature;
 788         int agent_len;
 789
 790         sort_ref_list(&ref, ref_compare_name);
 791         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 792
 793         if (is_repository_shallow() && !server_supports("shallow"))
 794                 die("Server does not support shallow clients");
 795         if (server_supports("multi_ack_detailed")) {
 796                 if (args->verbose)
 797                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 798                 multi_ack = 2;
 799                 if (server_supports("no-done")) {
 800                         if (args->verbose)
 801                                 fprintf(stderr, "Server supports no-done\n");
 802                         if (args->stateless_rpc)
 803                                 no_done = 1;
 804                 }
 805         }
 806         else if (server_supports("multi_ack")) {
 807                 if (args->verbose)
 808                         fprintf(stderr, "Server supports multi_ack\n");
 809                 multi_ack = 1;
 810         }
 811         if (server_supports("side-band-64k")) {
 812                 if (args->verbose)
 813                         fprintf(stderr, "Server supports side-band-64k\n");
 814                 use_sideband = 2;
 815         }
 816         else if (server_supports("side-band")) {
 817                 if (args->verbose)
 818                         fprintf(stderr, "Server supports side-band\n");
 819                 use_sideband = 1;
 820         }
 821         if (server_supports("allow-tip-sha1-in-want")) {
 822                 if (args->verbose)
 823                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 824                 allow_tip_sha1_in_want = 1;
 825         }
 826         if (!server_supports("thin-pack"))
 827                 args->use_thin_pack = 0;
 828         if (!server_supports("no-progress"))
 829                 args->no_progress = 0;
 830         if (!server_supports("include-tag"))
 831                 args->include_tag = 0;
 832         if (server_supports("ofs-delta")) {
 833                 if (args->verbose)
 834                         fprintf(stderr, "Server supports ofs-delta\n");
 835         } else
 836                 prefer_ofs_delta = 0;
 837
 838         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 839                 agent_supported = 1;
 840                 if (args->verbose && agent_len)
 841                         fprintf(stderr, "Server version is %.*s\n",
 842                                 agent_len, agent_feature);
 843         }
 844
 845         if (everything_local(args, &ref, sought, nr_sought)) {
 846                 packet_flush(fd[1]);
 847                 goto all_done;
 848         }
 849         if (find_common(args, fd, sha1, ref) < 0)
 850                 if (!args->keep_pack)
 851                         /* When cloning, it is not unusual to have
 852                          * no common commit.
 853                          */
 854                         warning("no common commits");
 855
 856         if (args->stateless_rpc)
 857                 packet_flush(fd[1]);
 858         if (args->depth > 0)
 859                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 860                                         NULL);
 861         else if (si->nr_ours || si->nr_theirs)
 862                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 863         else
 864                 alternate_shallow_file = NULL;
 865         if (get_pack(args, fd, pack_lockfile))
 866                 die("git fetch-pack: fetch failed.");
 867
 868  all_done:
 869         return ref;
 870 }
 871
 872 static void fetch_pack_config(void)
 873 {
 874         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 875         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 876         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 877         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 878         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 879
 880         git_config(git_default_config, NULL);
 881 }
 882
 883 static void fetch_pack_setup(void)
 884 {
 885         static int did_setup;
 886         if (did_setup)
 887                 return;
 888         fetch_pack_config();
 889         if (0 <= transfer_unpack_limit)
 890                 unpack_limit = transfer_unpack_limit;
 891         else if (0 <= fetch_unpack_limit)
 892                 unpack_limit = fetch_unpack_limit;
 893         did_setup = 1;
 894 }
 895
 896 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 897 {
 898         struct string_list names = STRING_LIST_INIT_NODUP;
 899         int src, dst;
 900
 901         for (src = dst = 0; src < nr; src++) {
 902                 struct string_list_item *item;
 903                 item = string_list_insert(&names, ref[src]->name);
 904                 if (item->util)
 905                         continue; /* already have it */
 906                 item->util = ref[src];
 907                 if (src != dst)
 908                         ref[dst] = ref[src];
 909                 dst++;
 910         }
 911         for (src = dst; src < nr; src++)
 912                 ref[src] = NULL;
 913         string_list_clear(&names, 0);
 914         return dst;
 915 }
 916
 917 static void update_shallow(struct fetch_pack_args *args,
 918                            struct ref **sought, int nr_sought,
 919                            struct shallow_info *si)
 920 {
 921         struct sha1_array ref = SHA1_ARRAY_INIT;
 922         int *status;
 923         int i;
 924
 925         if (args->depth > 0 && alternate_shallow_file) {
 926                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 927                         unlink_or_warn(git_path("shallow"));
 928                         rollback_lock_file(&shallow_lock);
 929                 } else
 930                         commit_lock_file(&shallow_lock);
 931                 return;
 932         }
 933
 934         if (!si->shallow || !si->shallow->nr)
 935                 return;
 936
 937         if (args->cloning) {
 938                 /*
 939                  * remote is shallow, but this is a clone, there are
 940                  * no objects in repo to worry about. Accept any
 941                  * shallow points that exist in the pack (iow in repo
 942                  * after get_pack() and reprepare_packed_git())
 943                  */
 944                 struct sha1_array extra = SHA1_ARRAY_INIT;
 945                 unsigned char (*sha1)[20] = si->shallow->sha1;
 946                 for (i = 0; i < si->shallow->nr; i++)
 947                         if (has_sha1_file(sha1[i]))
 948                                 sha1_array_append(&extra, sha1[i]);
 949                 if (extra.nr) {
 950                         setup_alternate_shallow(&shallow_lock,
 951                                                 &alternate_shallow_file,
 952                                                 &extra);
 953                         commit_lock_file(&shallow_lock);
 954                 }
 955                 sha1_array_clear(&extra);
 956                 return;
 957         }
 958
 959         if (!si->nr_ours && !si->nr_theirs)
 960                 return;
 961
 962         remove_nonexistent_theirs_shallow(si);
 963         if (!si->nr_ours && !si->nr_theirs)
 964                 return;
 965         for (i = 0; i < nr_sought; i++)
 966                 sha1_array_append(&ref, sought[i]->old_sha1);
 967         si->ref = &ref;
 968
 969         if (args->update_shallow) {
 970                 /*
 971                  * remote is also shallow, .git/shallow may be updated
 972                  * so all refs can be accepted. Make sure we only add
 973                  * shallow roots that are actually reachable from new
 974                  * refs.
 975                  */
 976                 struct sha1_array extra = SHA1_ARRAY_INIT;
 977                 unsigned char (*sha1)[20] = si->shallow->sha1;
 978                 assign_shallow_commits_to_refs(si, NULL, NULL);
 979                 if (!si->nr_ours && !si->nr_theirs) {
 980                         sha1_array_clear(&ref);
 981                         return;
 982                 }
 983                 for (i = 0; i < si->nr_ours; i++)
 984                         sha1_array_append(&extra, sha1[si->ours[i]]);
 985                 for (i = 0; i < si->nr_theirs; i++)
 986                         sha1_array_append(&extra, sha1[si->theirs[i]]);
 987                 setup_alternate_shallow(&shallow_lock,
 988                                         &alternate_shallow_file,
 989                                         &extra);
 990                 commit_lock_file(&shallow_lock);
 991                 sha1_array_clear(&extra);
 992                 sha1_array_clear(&ref);
 993                 return;
 994         }
 995
 996         /*
 997          * remote is also shallow, check what ref is safe to update
 998          * without updating .git/shallow
 999          */
1000         status = xcalloc(nr_sought, sizeof(*status));
1001         assign_shallow_commits_to_refs(si, NULL, status);
1002         if (si->nr_ours || si->nr_theirs) {
1003                 for (i = 0; i < nr_sought; i++)
1004                         if (status[i])
1005                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1006         }
1007         free(status);
1008         sha1_array_clear(&ref);
1009 }
1010
1011 struct ref *fetch_pack(struct fetch_pack_args *args,
1012                        int fd[], struct child_process *conn,
1013                        const struct ref *ref,
1014                        const char *dest,
1015                        struct ref **sought, int nr_sought,
1016                        struct sha1_array *shallow,
1017                        char **pack_lockfile)
1018 {
1019         struct ref *ref_cpy;
1020         struct shallow_info si;
1021
1022         fetch_pack_setup();
1023         if (nr_sought)
1024                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1025
1026         if (!ref) {
1027                 packet_flush(fd[1]);
1028                 die("no matching remote head");
1029         }
1030         prepare_shallow_info(&si, shallow);
1031         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1032                                 &si, pack_lockfile);
1033         reprepare_packed_git();
1034         update_shallow(args, sought, nr_sought, &si);
1035         clear_shallow_info(&si);
1036         return ref_cpy;
1037 }