fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "connect.h"
  13 #include "transport.h"
  14 #include "version.h"
  15 #include "prio-queue.h"
  16 #include "sha1-array.h"
  17
  18 static int transfer_unpack_limit = -1;
  19 static int fetch_unpack_limit = -1;
  20 static int unpack_limit = 100;
  21 static int prefer_ofs_delta = 1;
  22 static int no_done;
  23 static int fetch_fsck_objects = -1;
  24 static int transfer_fsck_objects = -1;
  25 static int agent_supported;
  26 static struct lock_file shallow_lock;
  27 static const char *alternate_shallow_file;
  28
  29 /* Remember to update object flag allocation in object.h */
  30 #define COMPLETE        (1U << 0)
  31 #define COMMON          (1U << 1)
  32 #define COMMON_REF      (1U << 2)
  33 #define SEEN            (1U << 3)
  34 #define POPPED          (1U << 4)
  35
  36 static int marked;
  37
  38 /*
  39  * After sending this many "have"s if we do not get any new ACK , we
  40  * give up traversing our history.
  41  */
  42 #define MAX_IN_VAIN 256
  43
  44 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  45 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  46
  47 static void rev_list_push(struct commit *commit, int mark)
  48 {
  49         if (!(commit->object.flags & mark)) {
  50                 commit->object.flags |= mark;
  51
  52                 if (parse_commit(commit))
  53                         return;
  54
  55                 prio_queue_put(&rev_list, commit);
  56
  57                 if (!(commit->object.flags & COMMON))
  58                         non_common_revs++;
  59         }
  60 }
  61
  62 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  63 {
  64         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  65
  66         if (o && o->type == OBJ_COMMIT)
  67                 rev_list_push((struct commit *)o, SEEN);
  68
  69         return 0;
  70 }
  71
  72 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  73 {
  74         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  75
  76         if (o && o->type == OBJ_COMMIT)
  77                 clear_commit_marks((struct commit *)o,
  78                                    COMMON | COMMON_REF | SEEN | POPPED);
  79         return 0;
  80 }
  81
  82 /*
  83    This function marks a rev and its ancestors as common.
  84    In some cases, it is desirable to mark only the ancestors (for example
  85    when only the server does not yet know that they are common).
  86 */
  87
  88 static void mark_common(struct commit *commit,
  89                 int ancestors_only, int dont_parse)
  90 {
  91         if (commit != NULL && !(commit->object.flags & COMMON)) {
  92                 struct object *o = (struct object *)commit;
  93
  94                 if (!ancestors_only)
  95                         o->flags |= COMMON;
  96
  97                 if (!(o->flags & SEEN))
  98                         rev_list_push(commit, SEEN);
  99                 else {
 100                         struct commit_list *parents;
 101
 102                         if (!ancestors_only && !(o->flags & POPPED))
 103                                 non_common_revs--;
 104                         if (!o->parsed && !dont_parse)
 105                                 if (parse_commit(commit))
 106                                         return;
 107
 108                         for (parents = commit->parents;
 109                                         parents;
 110                                         parents = parents->next)
 111                                 mark_common(parents->item, 0, dont_parse);
 112                 }
 113         }
 114 }
 115
 116 /*
 117   Get the next rev to send, ignoring the common.
 118 */
 119
 120 static const unsigned char *get_rev(void)
 121 {
 122         struct commit *commit = NULL;
 123
 124         while (commit == NULL) {
 125                 unsigned int mark;
 126                 struct commit_list *parents;
 127
 128                 if (rev_list.nr == 0 || non_common_revs == 0)
 129                         return NULL;
 130
 131                 commit = prio_queue_get(&rev_list);
 132                 parse_commit(commit);
 133                 parents = commit->parents;
 134
 135                 commit->object.flags |= POPPED;
 136                 if (!(commit->object.flags & COMMON))
 137                         non_common_revs--;
 138
 139                 if (commit->object.flags & COMMON) {
 140                         /* do not send "have", and ignore ancestors */
 141                         commit = NULL;
 142                         mark = COMMON | SEEN;
 143                 } else if (commit->object.flags & COMMON_REF)
 144                         /* send "have", and ignore ancestors */
 145                         mark = COMMON | SEEN;
 146                 else
 147                         /* send "have", also for its ancestors */
 148                         mark = SEEN;
 149
 150                 while (parents) {
 151                         if (!(parents->item->object.flags & SEEN))
 152                                 rev_list_push(parents->item, mark);
 153                         if (mark & COMMON)
 154                                 mark_common(parents->item, 1, 0);
 155                         parents = parents->next;
 156                 }
 157         }
 158
 159         return commit->object.sha1;
 160 }
 161
 162 enum ack_type {
 163         NAK = 0,
 164         ACK,
 165         ACK_continue,
 166         ACK_common,
 167         ACK_ready
 168 };
 169
 170 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 171 {
 172         if (args->stateless_rpc && args->depth > 0) {
 173                 /* If we sent a depth we will get back "duplicate"
 174                  * shallow and unshallow commands every time there
 175                  * is a block of have lines exchanged.
 176                  */
 177                 char *line;
 178                 while ((line = packet_read_line(fd, NULL))) {
 179                         if (starts_with(line, "shallow "))
 180                                 continue;
 181                         if (starts_with(line, "unshallow "))
 182                                 continue;
 183                         die("git fetch-pack: expected shallow list");
 184                 }
 185         }
 186 }
 187
 188 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 189 {
 190         int len;
 191         char *line = packet_read_line(fd, &len);
 192         const char *arg;
 193
 194         if (!len)
 195                 die("git fetch-pack: expected ACK/NAK, got EOF");
 196         if (!strcmp(line, "NAK"))
 197                 return NAK;
 198         if (skip_prefix(line, "ACK ", &arg)) {
 199                 if (!get_sha1_hex(arg, result_sha1)) {
 200                         arg += 40;
 201                         len -= arg - line;
 202                         if (len < 1)
 203                                 return ACK;
 204                         if (strstr(arg, "continue"))
 205                                 return ACK_continue;
 206                         if (strstr(arg, "common"))
 207                                 return ACK_common;
 208                         if (strstr(arg, "ready"))
 209                                 return ACK_ready;
 210                         return ACK;
 211                 }
 212         }
 213         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 214 }
 215
 216 static void send_request(struct fetch_pack_args *args,
 217                          int fd, struct strbuf *buf)
 218 {
 219         if (args->stateless_rpc) {
 220                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 221                 packet_flush(fd);
 222         } else
 223                 write_or_die(fd, buf->buf, buf->len);
 224 }
 225
 226 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 227 {
 228         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 229 }
 230
 231 #define INITIAL_FLUSH 16
 232 #define PIPESAFE_FLUSH 32
 233 #define LARGE_FLUSH 1024
 234
 235 static int next_flush(struct fetch_pack_args *args, int count)
 236 {
 237         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 238
 239         if (count < flush_limit)
 240                 count <<= 1;
 241         else
 242                 count += flush_limit;
 243         return count;
 244 }
 245
 246 static int find_common(struct fetch_pack_args *args,
 247                        int fd[2], unsigned char *result_sha1,
 248                        struct ref *refs)
 249 {
 250         int fetching;
 251         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 252         const unsigned char *sha1;
 253         unsigned in_vain = 0;
 254         int got_continue = 0;
 255         int got_ready = 0;
 256         struct strbuf req_buf = STRBUF_INIT;
 257         size_t state_len = 0;
 258
 259         if (args->stateless_rpc && multi_ack == 1)
 260                 die("--stateless-rpc requires multi_ack_detailed");
 261         if (marked)
 262                 for_each_ref(clear_marks, NULL);
 263         marked = 1;
 264
 265         for_each_ref(rev_list_insert_ref, NULL);
 266         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 267
 268         fetching = 0;
 269         for ( ; refs ; refs = refs->next) {
 270                 unsigned char *remote = refs->old_sha1;
 271                 const char *remote_hex;
 272                 struct object *o;
 273
 274                 /*
 275                  * If that object is complete (i.e. it is an ancestor of a
 276                  * local ref), we tell them we have it but do not have to
 277                  * tell them about its ancestors, which they already know
 278                  * about.
 279                  *
 280                  * We use lookup_object here because we are only
 281                  * interested in the case we *know* the object is
 282                  * reachable and we have already scanned it.
 283                  */
 284                 if (((o = lookup_object(remote)) != NULL) &&
 285                                 (o->flags & COMPLETE)) {
 286                         continue;
 287                 }
 288
 289                 remote_hex = sha1_to_hex(remote);
 290                 if (!fetching) {
 291                         struct strbuf c = STRBUF_INIT;
 292                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 293                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 294                         if (no_done)            strbuf_addstr(&c, " no-done");
 295                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 296                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 297                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 298                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 299                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 300                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 301                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 302                                                             git_user_agent_sanitized());
 303                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 304                         strbuf_release(&c);
 305                 } else
 306                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 307                 fetching++;
 308         }
 309
 310         if (!fetching) {
 311                 strbuf_release(&req_buf);
 312                 packet_flush(fd[1]);
 313                 return 1;
 314         }
 315
 316         if (is_repository_shallow())
 317                 write_shallow_commits(&req_buf, 1, NULL);
 318         if (args->depth > 0)
 319                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 320         packet_buf_flush(&req_buf);
 321         state_len = req_buf.len;
 322
 323         if (args->depth > 0) {
 324                 char *line;
 325                 const char *arg;
 326                 unsigned char sha1[20];
 327
 328                 send_request(args, fd[1], &req_buf);
 329                 while ((line = packet_read_line(fd[0], NULL))) {
 330                         if (skip_prefix(line, "shallow ", &arg)) {
 331                                 if (get_sha1_hex(arg, sha1))
 332                                         die("invalid shallow line: %s", line);
 333                                 register_shallow(sha1);
 334                                 continue;
 335                         }
 336                         if (skip_prefix(line, "unshallow ", &arg)) {
 337                                 if (get_sha1_hex(arg, sha1))
 338                                         die("invalid unshallow line: %s", line);
 339                                 if (!lookup_object(sha1))
 340                                         die("object not found: %s", line);
 341                                 /* make sure that it is parsed as shallow */
 342                                 if (!parse_object(sha1))
 343                                         die("error in object: %s", line);
 344                                 if (unregister_shallow(sha1))
 345                                         die("no shallow found: %s", line);
 346                                 continue;
 347                         }
 348                         die("expected shallow/unshallow, got %s", line);
 349                 }
 350         } else if (!args->stateless_rpc)
 351                 send_request(args, fd[1], &req_buf);
 352
 353         if (!args->stateless_rpc) {
 354                 /* If we aren't using the stateless-rpc interface
 355                  * we don't need to retain the headers.
 356                  */
 357                 strbuf_setlen(&req_buf, 0);
 358                 state_len = 0;
 359         }
 360
 361         flushes = 0;
 362         retval = -1;
 363         while ((sha1 = get_rev())) {
 364                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 365                 if (args->verbose)
 366                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 367                 in_vain++;
 368                 if (flush_at <= ++count) {
 369                         int ack;
 370
 371                         packet_buf_flush(&req_buf);
 372                         send_request(args, fd[1], &req_buf);
 373                         strbuf_setlen(&req_buf, state_len);
 374                         flushes++;
 375                         flush_at = next_flush(args, count);
 376
 377                         /*
 378                          * We keep one window "ahead" of the other side, and
 379                          * will wait for an ACK only on the next one
 380                          */
 381                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 382                                 continue;
 383
 384                         consume_shallow_list(args, fd[0]);
 385                         do {
 386                                 ack = get_ack(fd[0], result_sha1);
 387                                 if (args->verbose && ack)
 388                                         fprintf(stderr, "got ack %d %s\n", ack,
 389                                                         sha1_to_hex(result_sha1));
 390                                 switch (ack) {
 391                                 case ACK:
 392                                         flushes = 0;
 393                                         multi_ack = 0;
 394                                         retval = 0;
 395                                         goto done;
 396                                 case ACK_common:
 397                                 case ACK_ready:
 398                                 case ACK_continue: {
 399                                         struct commit *commit =
 400                                                 lookup_commit(result_sha1);
 401                                         if (!commit)
 402                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 403                                         if (args->stateless_rpc
 404                                          && ack == ACK_common
 405                                          && !(commit->object.flags & COMMON)) {
 406                                                 /* We need to replay the have for this object
 407                                                  * on the next RPC request so the peer knows
 408                                                  * it is in common with us.
 409                                                  */
 410                                                 const char *hex = sha1_to_hex(result_sha1);
 411                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 412                                                 state_len = req_buf.len;
 413                                         }
 414                                         mark_common(commit, 0, 1);
 415                                         retval = 0;
 416                                         in_vain = 0;
 417                                         got_continue = 1;
 418                                         if (ack == ACK_ready) {
 419                                                 clear_prio_queue(&rev_list);
 420                                                 got_ready = 1;
 421                                         }
 422                                         break;
 423                                         }
 424                                 }
 425                         } while (ack);
 426                         flushes--;
 427                         if (got_continue && MAX_IN_VAIN < in_vain) {
 428                                 if (args->verbose)
 429                                         fprintf(stderr, "giving up\n");
 430                                 break; /* give up */
 431                         }
 432                 }
 433         }
 434 done:
 435         if (!got_ready || !no_done) {
 436                 packet_buf_write(&req_buf, "done\n");
 437                 send_request(args, fd[1], &req_buf);
 438         }
 439         if (args->verbose)
 440                 fprintf(stderr, "done\n");
 441         if (retval != 0) {
 442                 multi_ack = 0;
 443                 flushes++;
 444         }
 445         strbuf_release(&req_buf);
 446
 447         if (!got_ready || !no_done)
 448                 consume_shallow_list(args, fd[0]);
 449         while (flushes || multi_ack) {
 450                 int ack = get_ack(fd[0], result_sha1);
 451                 if (ack) {
 452                         if (args->verbose)
 453                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 454                                         sha1_to_hex(result_sha1));
 455                         if (ack == ACK)
 456                                 return 0;
 457                         multi_ack = 1;
 458                         continue;
 459                 }
 460                 flushes--;
 461         }
 462         /* it is no error to fetch into a completely empty repo */
 463         return count ? retval : 0;
 464 }
 465
 466 static struct commit_list *complete;
 467
 468 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 469 {
 470         struct object *o = parse_object(sha1);
 471
 472         while (o && o->type == OBJ_TAG) {
 473                 struct tag *t = (struct tag *) o;
 474                 if (!t->tagged)
 475                         break; /* broken repository */
 476                 o->flags |= COMPLETE;
 477                 o = parse_object(t->tagged->sha1);
 478         }
 479         if (o && o->type == OBJ_COMMIT) {
 480                 struct commit *commit = (struct commit *)o;
 481                 if (!(commit->object.flags & COMPLETE)) {
 482                         commit->object.flags |= COMPLETE;
 483                         commit_list_insert(commit, &complete);
 484                 }
 485         }
 486         return 0;
 487 }
 488
 489 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 490                                          unsigned long cutoff)
 491 {
 492         while (complete && cutoff <= complete->item->date) {
 493                 if (args->verbose)
 494                         fprintf(stderr, "Marking %s as complete\n",
 495                                 sha1_to_hex(complete->item->object.sha1));
 496                 pop_most_recent_commit(&complete, COMPLETE);
 497         }
 498 }
 499
 500 static void filter_refs(struct fetch_pack_args *args,
 501                         struct ref **refs,
 502                         struct ref **sought, int nr_sought)
 503 {
 504         struct ref *newlist = NULL;
 505         struct ref **newtail = &newlist;
 506         struct ref *ref, *next;
 507         int i;
 508
 509         i = 0;
 510         for (ref = *refs; ref; ref = next) {
 511                 int keep = 0;
 512                 next = ref->next;
 513
 514                 if (starts_with(ref->name, "refs/") &&
 515                     check_refname_format(ref->name, 0))
 516                         ; /* trash */
 517                 else {
 518                         while (i < nr_sought) {
 519                                 int cmp = strcmp(ref->name, sought[i]->name);
 520                                 if (cmp < 0)
 521                                         break; /* definitely do not have it */
 522                                 else if (cmp == 0) {
 523                                         keep = 1; /* definitely have it */
 524                                         sought[i]->matched = 1;
 525                                 }
 526                                 i++;
 527                         }
 528                 }
 529
 530                 if (!keep && args->fetch_all &&
 531                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 532                         keep = 1;
 533
 534                 if (keep) {
 535                         *newtail = ref;
 536                         ref->next = NULL;
 537                         newtail = &ref->next;
 538                 } else {
 539                         free(ref);
 540                 }
 541         }
 542
 543         /* Append unmatched requests to the list */
 544         if (allow_tip_sha1_in_want) {
 545                 for (i = 0; i < nr_sought; i++) {
 546                         ref = sought[i];
 547                         if (ref->matched)
 548                                 continue;
 549                         if (get_sha1_hex(ref->name, ref->old_sha1))
 550                                 continue;
 551
 552                         ref->matched = 1;
 553                         *newtail = ref;
 554                         ref->next = NULL;
 555                         newtail = &ref->next;
 556                 }
 557         }
 558         *refs = newlist;
 559 }
 560
 561 static void mark_alternate_complete(const struct ref *ref, void *unused)
 562 {
 563         mark_complete(NULL, ref->old_sha1, 0, NULL);
 564 }
 565
 566 static int everything_local(struct fetch_pack_args *args,
 567                             struct ref **refs,
 568                             struct ref **sought, int nr_sought)
 569 {
 570         struct ref *ref;
 571         int retval;
 572         unsigned long cutoff = 0;
 573
 574         save_commit_buffer = 0;
 575
 576         for (ref = *refs; ref; ref = ref->next) {
 577                 struct object *o;
 578
 579                 if (!has_sha1_file(ref->old_sha1))
 580                         continue;
 581
 582                 o = parse_object(ref->old_sha1);
 583                 if (!o)
 584                         continue;
 585
 586                 /* We already have it -- which may mean that we were
 587                  * in sync with the other side at some time after
 588                  * that (it is OK if we guess wrong here).
 589                  */
 590                 if (o->type == OBJ_COMMIT) {
 591                         struct commit *commit = (struct commit *)o;
 592                         if (!cutoff || cutoff < commit->date)
 593                                 cutoff = commit->date;
 594                 }
 595         }
 596
 597         if (!args->depth) {
 598                 for_each_ref(mark_complete, NULL);
 599                 for_each_alternate_ref(mark_alternate_complete, NULL);
 600                 commit_list_sort_by_date(&complete);
 601                 if (cutoff)
 602                         mark_recent_complete_commits(args, cutoff);
 603         }
 604
 605         /*
 606          * Mark all complete remote refs as common refs.
 607          * Don't mark them common yet; the server has to be told so first.
 608          */
 609         for (ref = *refs; ref; ref = ref->next) {
 610                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 611                                              NULL, 0);
 612
 613                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 614                         continue;
 615
 616                 if (!(o->flags & SEEN)) {
 617                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 618
 619                         mark_common((struct commit *)o, 1, 1);
 620                 }
 621         }
 622
 623         filter_refs(args, refs, sought, nr_sought);
 624
 625         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 626                 const unsigned char *remote = ref->old_sha1;
 627                 unsigned char local[20];
 628                 struct object *o;
 629
 630                 o = lookup_object(remote);
 631                 if (!o || !(o->flags & COMPLETE)) {
 632                         retval = 0;
 633                         if (!args->verbose)
 634                                 continue;
 635                         fprintf(stderr,
 636                                 "want %s (%s)\n", sha1_to_hex(remote),
 637                                 ref->name);
 638                         continue;
 639                 }
 640
 641                 hashcpy(ref->new_sha1, local);
 642                 if (!args->verbose)
 643                         continue;
 644                 fprintf(stderr,
 645                         "already have %s (%s)\n", sha1_to_hex(remote),
 646                         ref->name);
 647         }
 648         return retval;
 649 }
 650
 651 static int sideband_demux(int in, int out, void *data)
 652 {
 653         int *xd = data;
 654
 655         int ret = recv_sideband("fetch-pack", xd[0], out);
 656         close(out);
 657         return ret;
 658 }
 659
 660 static int get_pack(struct fetch_pack_args *args,
 661                     int xd[2], char **pack_lockfile)
 662 {
 663         struct async demux;
 664         const char *argv[22];
 665         char keep_arg[256];
 666         char hdr_arg[256];
 667         const char **av, *cmd_name;
 668         int do_keep = args->keep_pack;
 669         struct child_process cmd = CHILD_PROCESS_INIT;
 670         int ret;
 671
 672         memset(&demux, 0, sizeof(demux));
 673         if (use_sideband) {
 674                 /* xd[] is talking with upload-pack; subprocess reads from
 675                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 676                  * through demux->out.
 677                  */
 678                 demux.proc = sideband_demux;
 679                 demux.data = xd;
 680                 demux.out = -1;
 681                 if (start_async(&demux))
 682                         die("fetch-pack: unable to fork off sideband"
 683                             " demultiplexer");
 684         }
 685         else
 686                 demux.out = xd[0];
 687
 688         cmd.argv = argv;
 689         av = argv;
 690         *hdr_arg = 0;
 691         if (!args->keep_pack && unpack_limit) {
 692                 struct pack_header header;
 693
 694                 if (read_pack_header(demux.out, &header))
 695                         die("protocol error: bad pack header");
 696                 snprintf(hdr_arg, sizeof(hdr_arg),
 697                          "--pack_header=%"PRIu32",%"PRIu32,
 698                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 699                 if (ntohl(header.hdr_entries) < unpack_limit)
 700                         do_keep = 0;
 701                 else
 702                         do_keep = 1;
 703         }
 704
 705         if (alternate_shallow_file) {
 706                 *av++ = "--shallow-file";
 707                 *av++ = alternate_shallow_file;
 708         }
 709
 710         if (do_keep) {
 711                 if (pack_lockfile)
 712                         cmd.out = -1;
 713                 *av++ = cmd_name = "index-pack";
 714                 *av++ = "--stdin";
 715                 if (!args->quiet && !args->no_progress)
 716                         *av++ = "-v";
 717                 if (args->use_thin_pack)
 718                         *av++ = "--fix-thin";
 719                 if (args->lock_pack || unpack_limit) {
 720                         int s = sprintf(keep_arg,
 721                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 722                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 723                                 strcpy(keep_arg + s, "localhost");
 724                         *av++ = keep_arg;
 725                 }
 726                 if (args->check_self_contained_and_connected)
 727                         *av++ = "--check-self-contained-and-connected";
 728         }
 729         else {
 730                 *av++ = cmd_name = "unpack-objects";
 731                 if (args->quiet || args->no_progress)
 732                         *av++ = "-q";
 733                 args->check_self_contained_and_connected = 0;
 734         }
 735         if (*hdr_arg)
 736                 *av++ = hdr_arg;
 737         if (fetch_fsck_objects >= 0
 738             ? fetch_fsck_objects
 739             : transfer_fsck_objects >= 0
 740             ? transfer_fsck_objects
 741             : 0)
 742                 *av++ = "--strict";
 743         *av++ = NULL;
 744
 745         cmd.in = demux.out;
 746         cmd.git_cmd = 1;
 747         if (start_command(&cmd))
 748                 die("fetch-pack: unable to fork off %s", cmd_name);
 749         if (do_keep && pack_lockfile) {
 750                 *pack_lockfile = index_pack_lockfile(cmd.out);
 751                 close(cmd.out);
 752         }
 753
 754         if (!use_sideband)
 755                 /* Closed by start_command() */
 756                 xd[0] = -1;
 757
 758         ret = finish_command(&cmd);
 759         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 760                 args->self_contained_and_connected =
 761                         args->check_self_contained_and_connected &&
 762                         ret == 0;
 763         else
 764                 die("%s failed", cmd_name);
 765         if (use_sideband && finish_async(&demux))
 766                 die("error in sideband demultiplexer");
 767         return 0;
 768 }
 769
 770 static int cmp_ref_by_name(const void *a_, const void *b_)
 771 {
 772         const struct ref *a = *((const struct ref **)a_);
 773         const struct ref *b = *((const struct ref **)b_);
 774         return strcmp(a->name, b->name);
 775 }
 776
 777 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 778                                  int fd[2],
 779                                  const struct ref *orig_ref,
 780                                  struct ref **sought, int nr_sought,
 781                                  struct shallow_info *si,
 782                                  char **pack_lockfile)
 783 {
 784         struct ref *ref = copy_ref_list(orig_ref);
 785         unsigned char sha1[20];
 786         const char *agent_feature;
 787         int agent_len;
 788
 789         sort_ref_list(&ref, ref_compare_name);
 790         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 791
 792         if (is_repository_shallow() && !server_supports("shallow"))
 793                 die("Server does not support shallow clients");
 794         if (server_supports("multi_ack_detailed")) {
 795                 if (args->verbose)
 796                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 797                 multi_ack = 2;
 798                 if (server_supports("no-done")) {
 799                         if (args->verbose)
 800                                 fprintf(stderr, "Server supports no-done\n");
 801                         if (args->stateless_rpc)
 802                                 no_done = 1;
 803                 }
 804         }
 805         else if (server_supports("multi_ack")) {
 806                 if (args->verbose)
 807                         fprintf(stderr, "Server supports multi_ack\n");
 808                 multi_ack = 1;
 809         }
 810         if (server_supports("side-band-64k")) {
 811                 if (args->verbose)
 812                         fprintf(stderr, "Server supports side-band-64k\n");
 813                 use_sideband = 2;
 814         }
 815         else if (server_supports("side-band")) {
 816                 if (args->verbose)
 817                         fprintf(stderr, "Server supports side-band\n");
 818                 use_sideband = 1;
 819         }
 820         if (server_supports("allow-tip-sha1-in-want")) {
 821                 if (args->verbose)
 822                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 823                 allow_tip_sha1_in_want = 1;
 824         }
 825         if (!server_supports("thin-pack"))
 826                 args->use_thin_pack = 0;
 827         if (!server_supports("no-progress"))
 828                 args->no_progress = 0;
 829         if (!server_supports("include-tag"))
 830                 args->include_tag = 0;
 831         if (server_supports("ofs-delta")) {
 832                 if (args->verbose)
 833                         fprintf(stderr, "Server supports ofs-delta\n");
 834         } else
 835                 prefer_ofs_delta = 0;
 836
 837         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 838                 agent_supported = 1;
 839                 if (args->verbose && agent_len)
 840                         fprintf(stderr, "Server version is %.*s\n",
 841                                 agent_len, agent_feature);
 842         }
 843
 844         if (everything_local(args, &ref, sought, nr_sought)) {
 845                 packet_flush(fd[1]);
 846                 goto all_done;
 847         }
 848         if (find_common(args, fd, sha1, ref) < 0)
 849                 if (!args->keep_pack)
 850                         /* When cloning, it is not unusual to have
 851                          * no common commit.
 852                          */
 853                         warning("no common commits");
 854
 855         if (args->stateless_rpc)
 856                 packet_flush(fd[1]);
 857         if (args->depth > 0)
 858                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 859                                         NULL);
 860         else if (si->nr_ours || si->nr_theirs)
 861                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 862         else
 863                 alternate_shallow_file = NULL;
 864         if (get_pack(args, fd, pack_lockfile))
 865                 die("git fetch-pack: fetch failed.");
 866
 867  all_done:
 868         return ref;
 869 }
 870
 871 static void fetch_pack_config(void)
 872 {
 873         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 874         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 875         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 876         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 877         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 878
 879         git_config(git_default_config, NULL);
 880 }
 881
 882 static void fetch_pack_setup(void)
 883 {
 884         static int did_setup;
 885         if (did_setup)
 886                 return;
 887         fetch_pack_config();
 888         if (0 <= transfer_unpack_limit)
 889                 unpack_limit = transfer_unpack_limit;
 890         else if (0 <= fetch_unpack_limit)
 891                 unpack_limit = fetch_unpack_limit;
 892         did_setup = 1;
 893 }
 894
 895 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 896 {
 897         struct string_list names = STRING_LIST_INIT_NODUP;
 898         int src, dst;
 899
 900         for (src = dst = 0; src < nr; src++) {
 901                 struct string_list_item *item;
 902                 item = string_list_insert(&names, ref[src]->name);
 903                 if (item->util)
 904                         continue; /* already have it */
 905                 item->util = ref[src];
 906                 if (src != dst)
 907                         ref[dst] = ref[src];
 908                 dst++;
 909         }
 910         for (src = dst; src < nr; src++)
 911                 ref[src] = NULL;
 912         string_list_clear(&names, 0);
 913         return dst;
 914 }
 915
 916 static void update_shallow(struct fetch_pack_args *args,
 917                            struct ref **sought, int nr_sought,
 918                            struct shallow_info *si)
 919 {
 920         struct sha1_array ref = SHA1_ARRAY_INIT;
 921         int *status;
 922         int i;
 923
 924         if (args->depth > 0 && alternate_shallow_file) {
 925                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 926                         unlink_or_warn(git_path("shallow"));
 927                         rollback_lock_file(&shallow_lock);
 928                 } else
 929                         commit_lock_file(&shallow_lock);
 930                 return;
 931         }
 932
 933         if (!si->shallow || !si->shallow->nr)
 934                 return;
 935
 936         if (args->cloning) {
 937                 /*
 938                  * remote is shallow, but this is a clone, there are
 939                  * no objects in repo to worry about. Accept any
 940                  * shallow points that exist in the pack (iow in repo
 941                  * after get_pack() and reprepare_packed_git())
 942                  */
 943                 struct sha1_array extra = SHA1_ARRAY_INIT;
 944                 unsigned char (*sha1)[20] = si->shallow->sha1;
 945                 for (i = 0; i < si->shallow->nr; i++)
 946                         if (has_sha1_file(sha1[i]))
 947                                 sha1_array_append(&extra, sha1[i]);
 948                 if (extra.nr) {
 949                         setup_alternate_shallow(&shallow_lock,
 950                                                 &alternate_shallow_file,
 951                                                 &extra);
 952                         commit_lock_file(&shallow_lock);
 953                 }
 954                 sha1_array_clear(&extra);
 955                 return;
 956         }
 957
 958         if (!si->nr_ours && !si->nr_theirs)
 959                 return;
 960
 961         remove_nonexistent_theirs_shallow(si);
 962         if (!si->nr_ours && !si->nr_theirs)
 963                 return;
 964         for (i = 0; i < nr_sought; i++)
 965                 sha1_array_append(&ref, sought[i]->old_sha1);
 966         si->ref = &ref;
 967
 968         if (args->update_shallow) {
 969                 /*
 970                  * remote is also shallow, .git/shallow may be updated
 971                  * so all refs can be accepted. Make sure we only add
 972                  * shallow roots that are actually reachable from new
 973                  * refs.
 974                  */
 975                 struct sha1_array extra = SHA1_ARRAY_INIT;
 976                 unsigned char (*sha1)[20] = si->shallow->sha1;
 977                 assign_shallow_commits_to_refs(si, NULL, NULL);
 978                 if (!si->nr_ours && !si->nr_theirs) {
 979                         sha1_array_clear(&ref);
 980                         return;
 981                 }
 982                 for (i = 0; i < si->nr_ours; i++)
 983                         sha1_array_append(&extra, sha1[si->ours[i]]);
 984                 for (i = 0; i < si->nr_theirs; i++)
 985                         sha1_array_append(&extra, sha1[si->theirs[i]]);
 986                 setup_alternate_shallow(&shallow_lock,
 987                                         &alternate_shallow_file,
 988                                         &extra);
 989                 commit_lock_file(&shallow_lock);
 990                 sha1_array_clear(&extra);
 991                 sha1_array_clear(&ref);
 992                 return;
 993         }
 994
 995         /*
 996          * remote is also shallow, check what ref is safe to update
 997          * without updating .git/shallow
 998          */
 999         status = xcalloc(nr_sought, sizeof(*status));
1000         assign_shallow_commits_to_refs(si, NULL, status);
1001         if (si->nr_ours || si->nr_theirs) {
1002                 for (i = 0; i < nr_sought; i++)
1003                         if (status[i])
1004                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1005         }
1006         free(status);
1007         sha1_array_clear(&ref);
1008 }
1009
1010 struct ref *fetch_pack(struct fetch_pack_args *args,
1011                        int fd[], struct child_process *conn,
1012                        const struct ref *ref,
1013                        const char *dest,
1014                        struct ref **sought, int nr_sought,
1015                        struct sha1_array *shallow,
1016                        char **pack_lockfile)
1017 {
1018         struct ref *ref_cpy;
1019         struct shallow_info si;
1020
1021         fetch_pack_setup();
1022         if (nr_sought)
1023                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1024
1025         if (!ref) {
1026                 packet_flush(fd[1]);
1027                 die("no matching remote head");
1028         }
1029         prepare_shallow_info(&si, shallow);
1030         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1031                                 &si, pack_lockfile);
1032         reprepare_packed_git();
1033         update_shallow(args, sought, nr_sought, &si);
1034         clear_shallow_info(&si);
1035         return ref_cpy;
1036 }