fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "connect.h"
  13 #include "transport.h"
  14 #include "version.h"
  15 #include "prio-queue.h"
  16 #include "sha1-array.h"
  17
  18 static int transfer_unpack_limit = -1;
  19 static int fetch_unpack_limit = -1;
  20 static int unpack_limit = 100;
  21 static int prefer_ofs_delta = 1;
  22 static int no_done;
  23 static int fetch_fsck_objects = -1;
  24 static int transfer_fsck_objects = -1;
  25 static int agent_supported;
  26 static struct lock_file shallow_lock;
  27 static const char *alternate_shallow_file;
  28
  29 /* Remember to update object flag allocation in object.h */
  30 #define COMPLETE        (1U << 0)
  31 #define COMMON          (1U << 1)
  32 #define COMMON_REF      (1U << 2)
  33 #define SEEN            (1U << 3)
  34 #define POPPED          (1U << 4)
  35
  36 static int marked;
  37
  38 /*
  39  * After sending this many "have"s if we do not get any new ACK , we
  40  * give up traversing our history.
  41  */
  42 #define MAX_IN_VAIN 256
  43
  44 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  45 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  46
  47 static void rev_list_push(struct commit *commit, int mark)
  48 {
  49         if (!(commit->object.flags & mark)) {
  50                 commit->object.flags |= mark;
  51
  52                 if (parse_commit(commit))
  53                         return;
  54
  55                 prio_queue_put(&rev_list, commit);
  56
  57                 if (!(commit->object.flags & COMMON))
  58                         non_common_revs++;
  59         }
  60 }
  61
  62 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  63 {
  64         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  65
  66         if (o && o->type == OBJ_COMMIT)
  67                 rev_list_push((struct commit *)o, SEEN);
  68
  69         return 0;
  70 }
  71
  72 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  73 {
  74         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  75
  76         if (o && o->type == OBJ_COMMIT)
  77                 clear_commit_marks((struct commit *)o,
  78                                    COMMON | COMMON_REF | SEEN | POPPED);
  79         return 0;
  80 }
  81
  82 /*
  83    This function marks a rev and its ancestors as common.
  84    In some cases, it is desirable to mark only the ancestors (for example
  85    when only the server does not yet know that they are common).
  86 */
  87
  88 static void mark_common(struct commit *commit,
  89                 int ancestors_only, int dont_parse)
  90 {
  91         if (commit != NULL && !(commit->object.flags & COMMON)) {
  92                 struct object *o = (struct object *)commit;
  93
  94                 if (!ancestors_only)
  95                         o->flags |= COMMON;
  96
  97                 if (!(o->flags & SEEN))
  98                         rev_list_push(commit, SEEN);
  99                 else {
 100                         struct commit_list *parents;
 101
 102                         if (!ancestors_only && !(o->flags & POPPED))
 103                                 non_common_revs--;
 104                         if (!o->parsed && !dont_parse)
 105                                 if (parse_commit(commit))
 106                                         return;
 107
 108                         for (parents = commit->parents;
 109                                         parents;
 110                                         parents = parents->next)
 111                                 mark_common(parents->item, 0, dont_parse);
 112                 }
 113         }
 114 }
 115
 116 /*
 117   Get the next rev to send, ignoring the common.
 118 */
 119
 120 static const unsigned char *get_rev(void)
 121 {
 122         struct commit *commit = NULL;
 123
 124         while (commit == NULL) {
 125                 unsigned int mark;
 126                 struct commit_list *parents;
 127
 128                 if (rev_list.nr == 0 || non_common_revs == 0)
 129                         return NULL;
 130
 131                 commit = prio_queue_get(&rev_list);
 132                 parse_commit(commit);
 133                 parents = commit->parents;
 134
 135                 commit->object.flags |= POPPED;
 136                 if (!(commit->object.flags & COMMON))
 137                         non_common_revs--;
 138
 139                 if (commit->object.flags & COMMON) {
 140                         /* do not send "have", and ignore ancestors */
 141                         commit = NULL;
 142                         mark = COMMON | SEEN;
 143                 } else if (commit->object.flags & COMMON_REF)
 144                         /* send "have", and ignore ancestors */
 145                         mark = COMMON | SEEN;
 146                 else
 147                         /* send "have", also for its ancestors */
 148                         mark = SEEN;
 149
 150                 while (parents) {
 151                         if (!(parents->item->object.flags & SEEN))
 152                                 rev_list_push(parents->item, mark);
 153                         if (mark & COMMON)
 154                                 mark_common(parents->item, 1, 0);
 155                         parents = parents->next;
 156                 }
 157         }
 158
 159         return commit->object.sha1;
 160 }
 161
 162 enum ack_type {
 163         NAK = 0,
 164         ACK,
 165         ACK_continue,
 166         ACK_common,
 167         ACK_ready
 168 };
 169
 170 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 171 {
 172         if (args->stateless_rpc && args->depth > 0) {
 173                 /* If we sent a depth we will get back "duplicate"
 174                  * shallow and unshallow commands every time there
 175                  * is a block of have lines exchanged.
 176                  */
 177                 char *line;
 178                 while ((line = packet_read_line(fd, NULL))) {
 179                         if (starts_with(line, "shallow "))
 180                                 continue;
 181                         if (starts_with(line, "unshallow "))
 182                                 continue;
 183                         die("git fetch-pack: expected shallow list");
 184                 }
 185         }
 186 }
 187
 188 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 189 {
 190         int len;
 191         char *line = packet_read_line(fd, &len);
 192         const char *arg;
 193
 194         if (!len)
 195                 die("git fetch-pack: expected ACK/NAK, got EOF");
 196         if (!strcmp(line, "NAK"))
 197                 return NAK;
 198         if (skip_prefix(line, "ACK ", &arg)) {
 199                 if (!get_sha1_hex(arg, result_sha1)) {
 200                         arg += 40;
 201                         len -= arg - line;
 202                         if (len < 1)
 203                                 return ACK;
 204                         if (strstr(arg, "continue"))
 205                                 return ACK_continue;
 206                         if (strstr(arg, "common"))
 207                                 return ACK_common;
 208                         if (strstr(arg, "ready"))
 209                                 return ACK_ready;
 210                         return ACK;
 211                 }
 212         }
 213         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 214 }
 215
 216 static void send_request(struct fetch_pack_args *args,
 217                          int fd, struct strbuf *buf)
 218 {
 219         if (args->stateless_rpc) {
 220                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 221                 packet_flush(fd);
 222         } else
 223                 write_or_die(fd, buf->buf, buf->len);
 224 }
 225
 226 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 227 {
 228         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 229 }
 230
 231 #define INITIAL_FLUSH 16
 232 #define PIPESAFE_FLUSH 32
 233 #define LARGE_FLUSH 1024
 234
 235 static int next_flush(struct fetch_pack_args *args, int count)
 236 {
 237         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 238
 239         if (count < flush_limit)
 240                 count <<= 1;
 241         else
 242                 count += flush_limit;
 243         return count;
 244 }
 245
 246 static int find_common(struct fetch_pack_args *args,
 247                        int fd[2], unsigned char *result_sha1,
 248                        struct ref *refs)
 249 {
 250         int fetching;
 251         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 252         const unsigned char *sha1;
 253         unsigned in_vain = 0;
 254         int got_continue = 0;
 255         int got_ready = 0;
 256         struct strbuf req_buf = STRBUF_INIT;
 257         size_t state_len = 0;
 258
 259         if (args->stateless_rpc && multi_ack == 1)
 260                 die("--stateless-rpc requires multi_ack_detailed");
 261         if (marked)
 262                 for_each_ref(clear_marks, NULL);
 263         marked = 1;
 264
 265         for_each_ref(rev_list_insert_ref, NULL);
 266         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 267
 268         fetching = 0;
 269         for ( ; refs ; refs = refs->next) {
 270                 unsigned char *remote = refs->old_sha1;
 271                 const char *remote_hex;
 272                 struct object *o;
 273
 274                 /*
 275                  * If that object is complete (i.e. it is an ancestor of a
 276                  * local ref), we tell them we have it but do not have to
 277                  * tell them about its ancestors, which they already know
 278                  * about.
 279                  *
 280                  * We use lookup_object here because we are only
 281                  * interested in the case we *know* the object is
 282                  * reachable and we have already scanned it.
 283                  */
 284                 if (((o = lookup_object(remote)) != NULL) &&
 285                                 (o->flags & COMPLETE)) {
 286                         continue;
 287                 }
 288
 289                 remote_hex = sha1_to_hex(remote);
 290                 if (!fetching) {
 291                         struct strbuf c = STRBUF_INIT;
 292                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 293                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 294                         if (no_done)            strbuf_addstr(&c, " no-done");
 295                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 296                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 297                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 298                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 299                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 300                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 301                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 302                                                             git_user_agent_sanitized());
 303                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 304                         strbuf_release(&c);
 305                 } else
 306                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 307                 fetching++;
 308         }
 309
 310         if (!fetching) {
 311                 strbuf_release(&req_buf);
 312                 packet_flush(fd[1]);
 313                 return 1;
 314         }
 315
 316         if (is_repository_shallow())
 317                 write_shallow_commits(&req_buf, 1, NULL);
 318         if (args->depth > 0)
 319                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 320         packet_buf_flush(&req_buf);
 321         state_len = req_buf.len;
 322
 323         if (args->depth > 0) {
 324                 char *line;
 325                 const char *arg;
 326                 unsigned char sha1[20];
 327
 328                 send_request(args, fd[1], &req_buf);
 329                 while ((line = packet_read_line(fd[0], NULL))) {
 330                         if (skip_prefix(line, "shallow ", &arg)) {
 331                                 if (get_sha1_hex(arg, sha1))
 332                                         die("invalid shallow line: %s", line);
 333                                 register_shallow(sha1);
 334                                 continue;
 335                         }
 336                         if (skip_prefix(line, "unshallow ", &arg)) {
 337                                 if (get_sha1_hex(arg, sha1))
 338                                         die("invalid unshallow line: %s", line);
 339                                 if (!lookup_object(sha1))
 340                                         die("object not found: %s", line);
 341                                 /* make sure that it is parsed as shallow */
 342                                 if (!parse_object(sha1))
 343                                         die("error in object: %s", line);
 344                                 if (unregister_shallow(sha1))
 345                                         die("no shallow found: %s", line);
 346                                 continue;
 347                         }
 348                         die("expected shallow/unshallow, got %s", line);
 349                 }
 350         } else if (!args->stateless_rpc)
 351                 send_request(args, fd[1], &req_buf);
 352
 353         if (!args->stateless_rpc) {
 354                 /* If we aren't using the stateless-rpc interface
 355                  * we don't need to retain the headers.
 356                  */
 357                 strbuf_setlen(&req_buf, 0);
 358                 state_len = 0;
 359         }
 360
 361         flushes = 0;
 362         retval = -1;
 363         while ((sha1 = get_rev())) {
 364                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 365                 if (args->verbose)
 366                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 367                 in_vain++;
 368                 if (flush_at <= ++count) {
 369                         int ack;
 370
 371                         packet_buf_flush(&req_buf);
 372                         send_request(args, fd[1], &req_buf);
 373                         strbuf_setlen(&req_buf, state_len);
 374                         flushes++;
 375                         flush_at = next_flush(args, count);
 376
 377                         /*
 378                          * We keep one window "ahead" of the other side, and
 379                          * will wait for an ACK only on the next one
 380                          */
 381                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 382                                 continue;
 383
 384                         consume_shallow_list(args, fd[0]);
 385                         do {
 386                                 ack = get_ack(fd[0], result_sha1);
 387                                 if (args->verbose && ack)
 388                                         fprintf(stderr, "got ack %d %s\n", ack,
 389                                                         sha1_to_hex(result_sha1));
 390                                 switch (ack) {
 391                                 case ACK:
 392                                         flushes = 0;
 393                                         multi_ack = 0;
 394                                         retval = 0;
 395                                         goto done;
 396                                 case ACK_common:
 397                                 case ACK_ready:
 398                                 case ACK_continue: {
 399                                         struct commit *commit =
 400                                                 lookup_commit(result_sha1);
 401                                         if (!commit)
 402                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 403                                         if (args->stateless_rpc
 404                                          && ack == ACK_common
 405                                          && !(commit->object.flags & COMMON)) {
 406                                                 /* We need to replay the have for this object
 407                                                  * on the next RPC request so the peer knows
 408                                                  * it is in common with us.
 409                                                  */
 410                                                 const char *hex = sha1_to_hex(result_sha1);
 411                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 412                                                 state_len = req_buf.len;
 413                                         }
 414                                         mark_common(commit, 0, 1);
 415                                         retval = 0;
 416                                         in_vain = 0;
 417                                         got_continue = 1;
 418                                         if (ack == ACK_ready) {
 419                                                 clear_prio_queue(&rev_list);
 420                                                 got_ready = 1;
 421                                         }
 422                                         break;
 423                                         }
 424                                 }
 425                         } while (ack);
 426                         flushes--;
 427                         if (got_continue && MAX_IN_VAIN < in_vain) {
 428                                 if (args->verbose)
 429                                         fprintf(stderr, "giving up\n");
 430                                 break; /* give up */
 431                         }
 432                 }
 433         }
 434 done:
 435         if (!got_ready || !no_done) {
 436                 packet_buf_write(&req_buf, "done\n");
 437                 send_request(args, fd[1], &req_buf);
 438         }
 439         if (args->verbose)
 440                 fprintf(stderr, "done\n");
 441         if (retval != 0) {
 442                 multi_ack = 0;
 443                 flushes++;
 444         }
 445         strbuf_release(&req_buf);
 446
 447         if (!got_ready || !no_done)
 448                 consume_shallow_list(args, fd[0]);
 449         while (flushes || multi_ack) {
 450                 int ack = get_ack(fd[0], result_sha1);
 451                 if (ack) {
 452                         if (args->verbose)
 453                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 454                                         sha1_to_hex(result_sha1));
 455                         if (ack == ACK)
 456                                 return 0;
 457                         multi_ack = 1;
 458                         continue;
 459                 }
 460                 flushes--;
 461         }
 462         /* it is no error to fetch into a completely empty repo */
 463         return count ? retval : 0;
 464 }
 465
 466 static struct commit_list *complete;
 467
 468 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 469 {
 470         struct object *o = parse_object(sha1);
 471
 472         while (o && o->type == OBJ_TAG) {
 473                 struct tag *t = (struct tag *) o;
 474                 if (!t->tagged)
 475                         break; /* broken repository */
 476                 o->flags |= COMPLETE;
 477                 o = parse_object(t->tagged->sha1);
 478         }
 479         if (o && o->type == OBJ_COMMIT) {
 480                 struct commit *commit = (struct commit *)o;
 481                 if (!(commit->object.flags & COMPLETE)) {
 482                         commit->object.flags |= COMPLETE;
 483                         commit_list_insert(commit, &complete);
 484                 }
 485         }
 486         return 0;
 487 }
 488
 489 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 490                                          unsigned long cutoff)
 491 {
 492         while (complete && cutoff <= complete->item->date) {
 493                 if (args->verbose)
 494                         fprintf(stderr, "Marking %s as complete\n",
 495                                 sha1_to_hex(complete->item->object.sha1));
 496                 pop_most_recent_commit(&complete, COMPLETE);
 497         }
 498 }
 499
 500 static void filter_refs(struct fetch_pack_args *args,
 501                         struct ref **refs,
 502                         struct ref **sought, int nr_sought)
 503 {
 504         struct ref *newlist = NULL;
 505         struct ref **newtail = &newlist;
 506         struct ref *ref, *next;
 507         int i;
 508
 509         i = 0;
 510         for (ref = *refs; ref; ref = next) {
 511                 int keep = 0;
 512                 next = ref->next;
 513
 514                 if (starts_with(ref->name, "refs/") &&
 515                     check_refname_format(ref->name, 0))
 516                         ; /* trash */
 517                 else {
 518                         while (i < nr_sought) {
 519                                 int cmp = strcmp(ref->name, sought[i]->name);
 520                                 if (cmp < 0)
 521                                         break; /* definitely do not have it */
 522                                 else if (cmp == 0) {
 523                                         keep = 1; /* definitely have it */
 524                                         sought[i]->matched = 1;
 525                                 }
 526                                 i++;
 527                         }
 528                 }
 529
 530                 if (!keep && args->fetch_all &&
 531                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 532                         keep = 1;
 533
 534                 if (keep) {
 535                         *newtail = ref;
 536                         ref->next = NULL;
 537                         newtail = &ref->next;
 538                 } else {
 539                         free(ref);
 540                 }
 541         }
 542
 543         /* Append unmatched requests to the list */
 544         if (allow_tip_sha1_in_want) {
 545                 for (i = 0; i < nr_sought; i++) {
 546                         ref = sought[i];
 547                         if (ref->matched)
 548                                 continue;
 549                         if (get_sha1_hex(ref->name, ref->old_sha1))
 550                                 continue;
 551
 552                         ref->matched = 1;
 553                         *newtail = ref;
 554                         ref->next = NULL;
 555                         newtail = &ref->next;
 556                 }
 557         }
 558         *refs = newlist;
 559 }
 560
 561 static void mark_alternate_complete(const struct ref *ref, void *unused)
 562 {
 563         mark_complete(NULL, ref->old_sha1, 0, NULL);
 564 }
 565
 566 static int everything_local(struct fetch_pack_args *args,
 567                             struct ref **refs,
 568                             struct ref **sought, int nr_sought)
 569 {
 570         struct ref *ref;
 571         int retval;
 572         unsigned long cutoff = 0;
 573
 574         save_commit_buffer = 0;
 575
 576         for (ref = *refs; ref; ref = ref->next) {
 577                 struct object *o;
 578
 579                 if (!has_sha1_file(ref->old_sha1))
 580                         continue;
 581
 582                 o = parse_object(ref->old_sha1);
 583                 if (!o)
 584                         continue;
 585
 586                 /* We already have it -- which may mean that we were
 587                  * in sync with the other side at some time after
 588                  * that (it is OK if we guess wrong here).
 589                  */
 590                 if (o->type == OBJ_COMMIT) {
 591                         struct commit *commit = (struct commit *)o;
 592                         if (!cutoff || cutoff < commit->date)
 593                                 cutoff = commit->date;
 594                 }
 595         }
 596
 597         if (!args->depth) {
 598                 for_each_ref(mark_complete, NULL);
 599                 for_each_alternate_ref(mark_alternate_complete, NULL);
 600                 commit_list_sort_by_date(&complete);
 601                 if (cutoff)
 602                         mark_recent_complete_commits(args, cutoff);
 603         }
 604
 605         /*
 606          * Mark all complete remote refs as common refs.
 607          * Don't mark them common yet; the server has to be told so first.
 608          */
 609         for (ref = *refs; ref; ref = ref->next) {
 610                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 611                                              NULL, 0);
 612
 613                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 614                         continue;
 615
 616                 if (!(o->flags & SEEN)) {
 617                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 618
 619                         mark_common((struct commit *)o, 1, 1);
 620                 }
 621         }
 622
 623         filter_refs(args, refs, sought, nr_sought);
 624
 625         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 626                 const unsigned char *remote = ref->old_sha1;
 627                 unsigned char local[20];
 628                 struct object *o;
 629
 630                 o = lookup_object(remote);
 631                 if (!o || !(o->flags & COMPLETE)) {
 632                         retval = 0;
 633                         if (!args->verbose)
 634                                 continue;
 635                         fprintf(stderr,
 636                                 "want %s (%s)\n", sha1_to_hex(remote),
 637                                 ref->name);
 638                         continue;
 639                 }
 640
 641                 hashcpy(ref->new_sha1, local);
 642                 if (!args->verbose)
 643                         continue;
 644                 fprintf(stderr,
 645                         "already have %s (%s)\n", sha1_to_hex(remote),
 646                         ref->name);
 647         }
 648         return retval;
 649 }
 650
 651 static int sideband_demux(int in, int out, void *data)
 652 {
 653         int *xd = data;
 654
 655         int ret = recv_sideband("fetch-pack", xd[0], out);
 656         close(out);
 657         return ret;
 658 }
 659
 660 static int get_pack(struct fetch_pack_args *args,
 661                     int xd[2], char **pack_lockfile)
 662 {
 663         struct async demux;
 664         const char *argv[22];
 665         char keep_arg[256];
 666         char hdr_arg[256];
 667         const char **av, *cmd_name;
 668         int do_keep = args->keep_pack;
 669         struct child_process cmd;
 670         int ret;
 671
 672         memset(&demux, 0, sizeof(demux));
 673         if (use_sideband) {
 674                 /* xd[] is talking with upload-pack; subprocess reads from
 675                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 676                  * through demux->out.
 677                  */
 678                 demux.proc = sideband_demux;
 679                 demux.data = xd;
 680                 demux.out = -1;
 681                 if (start_async(&demux))
 682                         die("fetch-pack: unable to fork off sideband"
 683                             " demultiplexer");
 684         }
 685         else
 686                 demux.out = xd[0];
 687
 688         memset(&cmd, 0, sizeof(cmd));
 689         cmd.argv = argv;
 690         av = argv;
 691         *hdr_arg = 0;
 692         if (!args->keep_pack && unpack_limit) {
 693                 struct pack_header header;
 694
 695                 if (read_pack_header(demux.out, &header))
 696                         die("protocol error: bad pack header");
 697                 snprintf(hdr_arg, sizeof(hdr_arg),
 698                          "--pack_header=%"PRIu32",%"PRIu32,
 699                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 700                 if (ntohl(header.hdr_entries) < unpack_limit)
 701                         do_keep = 0;
 702                 else
 703                         do_keep = 1;
 704         }
 705
 706         if (alternate_shallow_file) {
 707                 *av++ = "--shallow-file";
 708                 *av++ = alternate_shallow_file;
 709         }
 710
 711         if (do_keep) {
 712                 if (pack_lockfile)
 713                         cmd.out = -1;
 714                 *av++ = cmd_name = "index-pack";
 715                 *av++ = "--stdin";
 716                 if (!args->quiet && !args->no_progress)
 717                         *av++ = "-v";
 718                 if (args->use_thin_pack)
 719                         *av++ = "--fix-thin";
 720                 if (args->lock_pack || unpack_limit) {
 721                         int s = sprintf(keep_arg,
 722                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 723                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 724                                 strcpy(keep_arg + s, "localhost");
 725                         *av++ = keep_arg;
 726                 }
 727                 if (args->check_self_contained_and_connected)
 728                         *av++ = "--check-self-contained-and-connected";
 729         }
 730         else {
 731                 *av++ = cmd_name = "unpack-objects";
 732                 if (args->quiet || args->no_progress)
 733                         *av++ = "-q";
 734                 args->check_self_contained_and_connected = 0;
 735         }
 736         if (*hdr_arg)
 737                 *av++ = hdr_arg;
 738         if (fetch_fsck_objects >= 0
 739             ? fetch_fsck_objects
 740             : transfer_fsck_objects >= 0
 741             ? transfer_fsck_objects
 742             : 0)
 743                 *av++ = "--strict";
 744         *av++ = NULL;
 745
 746         cmd.in = demux.out;
 747         cmd.git_cmd = 1;
 748         if (start_command(&cmd))
 749                 die("fetch-pack: unable to fork off %s", cmd_name);
 750         if (do_keep && pack_lockfile) {
 751                 *pack_lockfile = index_pack_lockfile(cmd.out);
 752                 close(cmd.out);
 753         }
 754
 755         if (!use_sideband)
 756                 /* Closed by start_command() */
 757                 xd[0] = -1;
 758
 759         ret = finish_command(&cmd);
 760         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 761                 args->self_contained_and_connected =
 762                         args->check_self_contained_and_connected &&
 763                         ret == 0;
 764         else
 765                 die("%s failed", cmd_name);
 766         if (use_sideband && finish_async(&demux))
 767                 die("error in sideband demultiplexer");
 768         return 0;
 769 }
 770
 771 static int cmp_ref_by_name(const void *a_, const void *b_)
 772 {
 773         const struct ref *a = *((const struct ref **)a_);
 774         const struct ref *b = *((const struct ref **)b_);
 775         return strcmp(a->name, b->name);
 776 }
 777
 778 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 779                                  int fd[2],
 780                                  const struct ref *orig_ref,
 781                                  struct ref **sought, int nr_sought,
 782                                  struct shallow_info *si,
 783                                  char **pack_lockfile)
 784 {
 785         struct ref *ref = copy_ref_list(orig_ref);
 786         unsigned char sha1[20];
 787         const char *agent_feature;
 788         int agent_len;
 789
 790         sort_ref_list(&ref, ref_compare_name);
 791         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 792
 793         if (is_repository_shallow() && !server_supports("shallow"))
 794                 die("Server does not support shallow clients");
 795         if (server_supports("multi_ack_detailed")) {
 796                 if (args->verbose)
 797                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 798                 multi_ack = 2;
 799                 if (server_supports("no-done")) {
 800                         if (args->verbose)
 801                                 fprintf(stderr, "Server supports no-done\n");
 802                         if (args->stateless_rpc)
 803                                 no_done = 1;
 804                 }
 805         }
 806         else if (server_supports("multi_ack")) {
 807                 if (args->verbose)
 808                         fprintf(stderr, "Server supports multi_ack\n");
 809                 multi_ack = 1;
 810         }
 811         if (server_supports("side-band-64k")) {
 812                 if (args->verbose)
 813                         fprintf(stderr, "Server supports side-band-64k\n");
 814                 use_sideband = 2;
 815         }
 816         else if (server_supports("side-band")) {
 817                 if (args->verbose)
 818                         fprintf(stderr, "Server supports side-band\n");
 819                 use_sideband = 1;
 820         }
 821         if (server_supports("allow-tip-sha1-in-want")) {
 822                 if (args->verbose)
 823                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 824                 allow_tip_sha1_in_want = 1;
 825         }
 826         if (!server_supports("thin-pack"))
 827                 args->use_thin_pack = 0;
 828         if (!server_supports("no-progress"))
 829                 args->no_progress = 0;
 830         if (!server_supports("include-tag"))
 831                 args->include_tag = 0;
 832         if (server_supports("ofs-delta")) {
 833                 if (args->verbose)
 834                         fprintf(stderr, "Server supports ofs-delta\n");
 835         } else
 836                 prefer_ofs_delta = 0;
 837
 838         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 839                 agent_supported = 1;
 840                 if (args->verbose && agent_len)
 841                         fprintf(stderr, "Server version is %.*s\n",
 842                                 agent_len, agent_feature);
 843         }
 844
 845         if (everything_local(args, &ref, sought, nr_sought)) {
 846                 packet_flush(fd[1]);
 847                 goto all_done;
 848         }
 849         if (find_common(args, fd, sha1, ref) < 0)
 850                 if (!args->keep_pack)
 851                         /* When cloning, it is not unusual to have
 852                          * no common commit.
 853                          */
 854                         warning("no common commits");
 855
 856         if (args->stateless_rpc)
 857                 packet_flush(fd[1]);
 858         if (args->depth > 0)
 859                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 860                                         NULL);
 861         else if (si->nr_ours || si->nr_theirs)
 862                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 863         else
 864                 alternate_shallow_file = NULL;
 865         if (get_pack(args, fd, pack_lockfile))
 866                 die("git fetch-pack: fetch failed.");
 867
 868  all_done:
 869         return ref;
 870 }
 871
 872 static int fetch_pack_config(const char *var, const char *value, void *cb)
 873 {
 874         if (strcmp(var, "fetch.unpacklimit") == 0) {
 875                 fetch_unpack_limit = git_config_int(var, value);
 876                 return 0;
 877         }
 878
 879         if (strcmp(var, "transfer.unpacklimit") == 0) {
 880                 transfer_unpack_limit = git_config_int(var, value);
 881                 return 0;
 882         }
 883
 884         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 885                 prefer_ofs_delta = git_config_bool(var, value);
 886                 return 0;
 887         }
 888
 889         if (!strcmp(var, "fetch.fsckobjects")) {
 890                 fetch_fsck_objects = git_config_bool(var, value);
 891                 return 0;
 892         }
 893
 894         if (!strcmp(var, "transfer.fsckobjects")) {
 895                 transfer_fsck_objects = git_config_bool(var, value);
 896                 return 0;
 897         }
 898
 899         return git_default_config(var, value, cb);
 900 }
 901
 902 static void fetch_pack_setup(void)
 903 {
 904         static int did_setup;
 905         if (did_setup)
 906                 return;
 907         git_config(fetch_pack_config, NULL);
 908         if (0 <= transfer_unpack_limit)
 909                 unpack_limit = transfer_unpack_limit;
 910         else if (0 <= fetch_unpack_limit)
 911                 unpack_limit = fetch_unpack_limit;
 912         did_setup = 1;
 913 }
 914
 915 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 916 {
 917         struct string_list names = STRING_LIST_INIT_NODUP;
 918         int src, dst;
 919
 920         for (src = dst = 0; src < nr; src++) {
 921                 struct string_list_item *item;
 922                 item = string_list_insert(&names, ref[src]->name);
 923                 if (item->util)
 924                         continue; /* already have it */
 925                 item->util = ref[src];
 926                 if (src != dst)
 927                         ref[dst] = ref[src];
 928                 dst++;
 929         }
 930         for (src = dst; src < nr; src++)
 931                 ref[src] = NULL;
 932         string_list_clear(&names, 0);
 933         return dst;
 934 }
 935
 936 static void update_shallow(struct fetch_pack_args *args,
 937                            struct ref **sought, int nr_sought,
 938                            struct shallow_info *si)
 939 {
 940         struct sha1_array ref = SHA1_ARRAY_INIT;
 941         int *status;
 942         int i;
 943
 944         if (args->depth > 0 && alternate_shallow_file) {
 945                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 946                         unlink_or_warn(git_path("shallow"));
 947                         rollback_lock_file(&shallow_lock);
 948                 } else
 949                         commit_lock_file(&shallow_lock);
 950                 return;
 951         }
 952
 953         if (!si->shallow || !si->shallow->nr)
 954                 return;
 955
 956         if (args->cloning) {
 957                 /*
 958                  * remote is shallow, but this is a clone, there are
 959                  * no objects in repo to worry about. Accept any
 960                  * shallow points that exist in the pack (iow in repo
 961                  * after get_pack() and reprepare_packed_git())
 962                  */
 963                 struct sha1_array extra = SHA1_ARRAY_INIT;
 964                 unsigned char (*sha1)[20] = si->shallow->sha1;
 965                 for (i = 0; i < si->shallow->nr; i++)
 966                         if (has_sha1_file(sha1[i]))
 967                                 sha1_array_append(&extra, sha1[i]);
 968                 if (extra.nr) {
 969                         setup_alternate_shallow(&shallow_lock,
 970                                                 &alternate_shallow_file,
 971                                                 &extra);
 972                         commit_lock_file(&shallow_lock);
 973                 }
 974                 sha1_array_clear(&extra);
 975                 return;
 976         }
 977
 978         if (!si->nr_ours && !si->nr_theirs)
 979                 return;
 980
 981         remove_nonexistent_theirs_shallow(si);
 982         if (!si->nr_ours && !si->nr_theirs)
 983                 return;
 984         for (i = 0; i < nr_sought; i++)
 985                 sha1_array_append(&ref, sought[i]->old_sha1);
 986         si->ref = &ref;
 987
 988         if (args->update_shallow) {
 989                 /*
 990                  * remote is also shallow, .git/shallow may be updated
 991                  * so all refs can be accepted. Make sure we only add
 992                  * shallow roots that are actually reachable from new
 993                  * refs.
 994                  */
 995                 struct sha1_array extra = SHA1_ARRAY_INIT;
 996                 unsigned char (*sha1)[20] = si->shallow->sha1;
 997                 assign_shallow_commits_to_refs(si, NULL, NULL);
 998                 if (!si->nr_ours && !si->nr_theirs) {
 999                         sha1_array_clear(&ref);
1000                         return;
1001                 }
1002                 for (i = 0; i < si->nr_ours; i++)
1003                         sha1_array_append(&extra, sha1[si->ours[i]]);
1004                 for (i = 0; i < si->nr_theirs; i++)
1005                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1006                 setup_alternate_shallow(&shallow_lock,
1007                                         &alternate_shallow_file,
1008                                         &extra);
1009                 commit_lock_file(&shallow_lock);
1010                 sha1_array_clear(&extra);
1011                 sha1_array_clear(&ref);
1012                 return;
1013         }
1014
1015         /*
1016          * remote is also shallow, check what ref is safe to update
1017          * without updating .git/shallow
1018          */
1019         status = xcalloc(nr_sought, sizeof(*status));
1020         assign_shallow_commits_to_refs(si, NULL, status);
1021         if (si->nr_ours || si->nr_theirs) {
1022                 for (i = 0; i < nr_sought; i++)
1023                         if (status[i])
1024                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1025         }
1026         free(status);
1027         sha1_array_clear(&ref);
1028 }
1029
1030 struct ref *fetch_pack(struct fetch_pack_args *args,
1031                        int fd[], struct child_process *conn,
1032                        const struct ref *ref,
1033                        const char *dest,
1034                        struct ref **sought, int nr_sought,
1035                        struct sha1_array *shallow,
1036                        char **pack_lockfile)
1037 {
1038         struct ref *ref_cpy;
1039         struct shallow_info si;
1040
1041         fetch_pack_setup();
1042         if (nr_sought)
1043                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1044
1045         if (!ref) {
1046                 packet_flush(fd[1]);
1047                 die("no matching remote head");
1048         }
1049         prepare_shallow_info(&si, shallow);
1050         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1051                                 &si, pack_lockfile);
1052         reprepare_packed_git();
1053         update_shallow(args, sought, nr_sought, &si);
1054         clear_shallow_info(&si);
1055         return ref_cpy;
1056 }