fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "connect.h"
  13 #include "transport.h"
  14 #include "version.h"
  15 #include "prio-queue.h"
  16 #include "sha1-array.h"
  17
  18 static int transfer_unpack_limit = -1;
  19 static int fetch_unpack_limit = -1;
  20 static int unpack_limit = 100;
  21 static int prefer_ofs_delta = 1;
  22 static int no_done;
  23 static int fetch_fsck_objects = -1;
  24 static int transfer_fsck_objects = -1;
  25 static int agent_supported;
  26 static struct lock_file shallow_lock;
  27 static const char *alternate_shallow_file;
  28
  29 #define COMPLETE        (1U << 0)
  30 #define COMMON          (1U << 1)
  31 #define COMMON_REF      (1U << 2)
  32 #define SEEN            (1U << 3)
  33 #define POPPED          (1U << 4)
  34
  35 static int marked;
  36
  37 /*
  38  * After sending this many "have"s if we do not get any new ACK , we
  39  * give up traversing our history.
  40  */
  41 #define MAX_IN_VAIN 256
  42
  43 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  44 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  45
  46 static void rev_list_push(struct commit *commit, int mark)
  47 {
  48         if (!(commit->object.flags & mark)) {
  49                 commit->object.flags |= mark;
  50
  51                 if (parse_commit(commit))
  52                         return;
  53
  54                 prio_queue_put(&rev_list, commit);
  55
  56                 if (!(commit->object.flags & COMMON))
  57                         non_common_revs++;
  58         }
  59 }
  60
  61 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  62 {
  63         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  64
  65         if (o && o->type == OBJ_COMMIT)
  66                 rev_list_push((struct commit *)o, SEEN);
  67
  68         return 0;
  69 }
  70
  71 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  72 {
  73         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  74
  75         if (o && o->type == OBJ_COMMIT)
  76                 clear_commit_marks((struct commit *)o,
  77                                    COMMON | COMMON_REF | SEEN | POPPED);
  78         return 0;
  79 }
  80
  81 /*
  82    This function marks a rev and its ancestors as common.
  83    In some cases, it is desirable to mark only the ancestors (for example
  84    when only the server does not yet know that they are common).
  85 */
  86
  87 static void mark_common(struct commit *commit,
  88                 int ancestors_only, int dont_parse)
  89 {
  90         if (commit != NULL && !(commit->object.flags & COMMON)) {
  91                 struct object *o = (struct object *)commit;
  92
  93                 if (!ancestors_only)
  94                         o->flags |= COMMON;
  95
  96                 if (!(o->flags & SEEN))
  97                         rev_list_push(commit, SEEN);
  98                 else {
  99                         struct commit_list *parents;
 100
 101                         if (!ancestors_only && !(o->flags & POPPED))
 102                                 non_common_revs--;
 103                         if (!o->parsed && !dont_parse)
 104                                 if (parse_commit(commit))
 105                                         return;
 106
 107                         for (parents = commit->parents;
 108                                         parents;
 109                                         parents = parents->next)
 110                                 mark_common(parents->item, 0, dont_parse);
 111                 }
 112         }
 113 }
 114
 115 /*
 116   Get the next rev to send, ignoring the common.
 117 */
 118
 119 static const unsigned char *get_rev(void)
 120 {
 121         struct commit *commit = NULL;
 122
 123         while (commit == NULL) {
 124                 unsigned int mark;
 125                 struct commit_list *parents;
 126
 127                 if (rev_list.nr == 0 || non_common_revs == 0)
 128                         return NULL;
 129
 130                 commit = prio_queue_get(&rev_list);
 131                 parse_commit(commit);
 132                 parents = commit->parents;
 133
 134                 commit->object.flags |= POPPED;
 135                 if (!(commit->object.flags & COMMON))
 136                         non_common_revs--;
 137
 138                 if (commit->object.flags & COMMON) {
 139                         /* do not send "have", and ignore ancestors */
 140                         commit = NULL;
 141                         mark = COMMON | SEEN;
 142                 } else if (commit->object.flags & COMMON_REF)
 143                         /* send "have", and ignore ancestors */
 144                         mark = COMMON | SEEN;
 145                 else
 146                         /* send "have", also for its ancestors */
 147                         mark = SEEN;
 148
 149                 while (parents) {
 150                         if (!(parents->item->object.flags & SEEN))
 151                                 rev_list_push(parents->item, mark);
 152                         if (mark & COMMON)
 153                                 mark_common(parents->item, 1, 0);
 154                         parents = parents->next;
 155                 }
 156         }
 157
 158         return commit->object.sha1;
 159 }
 160
 161 enum ack_type {
 162         NAK = 0,
 163         ACK,
 164         ACK_continue,
 165         ACK_common,
 166         ACK_ready
 167 };
 168
 169 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 170 {
 171         if (args->stateless_rpc && args->depth > 0) {
 172                 /* If we sent a depth we will get back "duplicate"
 173                  * shallow and unshallow commands every time there
 174                  * is a block of have lines exchanged.
 175                  */
 176                 char *line;
 177                 while ((line = packet_read_line(fd, NULL))) {
 178                         if (starts_with(line, "shallow "))
 179                                 continue;
 180                         if (starts_with(line, "unshallow "))
 181                                 continue;
 182                         die("git fetch-pack: expected shallow list");
 183                 }
 184         }
 185 }
 186
 187 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 188 {
 189         int len;
 190         char *line = packet_read_line(fd, &len);
 191
 192         if (!len)
 193                 die("git fetch-pack: expected ACK/NAK, got EOF");
 194         if (!strcmp(line, "NAK"))
 195                 return NAK;
 196         if (starts_with(line, "ACK ")) {
 197                 if (!get_sha1_hex(line+4, result_sha1)) {
 198                         if (len < 45)
 199                                 return ACK;
 200                         if (strstr(line+45, "continue"))
 201                                 return ACK_continue;
 202                         if (strstr(line+45, "common"))
 203                                 return ACK_common;
 204                         if (strstr(line+45, "ready"))
 205                                 return ACK_ready;
 206                         return ACK;
 207                 }
 208         }
 209         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 210 }
 211
 212 static void send_request(struct fetch_pack_args *args,
 213                          int fd, struct strbuf *buf)
 214 {
 215         if (args->stateless_rpc) {
 216                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 217                 packet_flush(fd);
 218         } else
 219                 write_or_die(fd, buf->buf, buf->len);
 220 }
 221
 222 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 223 {
 224         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 225 }
 226
 227 #define INITIAL_FLUSH 16
 228 #define PIPESAFE_FLUSH 32
 229 #define LARGE_FLUSH 1024
 230
 231 static int next_flush(struct fetch_pack_args *args, int count)
 232 {
 233         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 234
 235         if (count < flush_limit)
 236                 count <<= 1;
 237         else
 238                 count += flush_limit;
 239         return count;
 240 }
 241
 242 static int find_common(struct fetch_pack_args *args,
 243                        int fd[2], unsigned char *result_sha1,
 244                        struct ref *refs)
 245 {
 246         int fetching;
 247         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 248         const unsigned char *sha1;
 249         unsigned in_vain = 0;
 250         int got_continue = 0;
 251         int got_ready = 0;
 252         struct strbuf req_buf = STRBUF_INIT;
 253         size_t state_len = 0;
 254
 255         if (args->stateless_rpc && multi_ack == 1)
 256                 die("--stateless-rpc requires multi_ack_detailed");
 257         if (marked)
 258                 for_each_ref(clear_marks, NULL);
 259         marked = 1;
 260
 261         for_each_ref(rev_list_insert_ref, NULL);
 262         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 263
 264         fetching = 0;
 265         for ( ; refs ; refs = refs->next) {
 266                 unsigned char *remote = refs->old_sha1;
 267                 const char *remote_hex;
 268                 struct object *o;
 269
 270                 /*
 271                  * If that object is complete (i.e. it is an ancestor of a
 272                  * local ref), we tell them we have it but do not have to
 273                  * tell them about its ancestors, which they already know
 274                  * about.
 275                  *
 276                  * We use lookup_object here because we are only
 277                  * interested in the case we *know* the object is
 278                  * reachable and we have already scanned it.
 279                  */
 280                 if (((o = lookup_object(remote)) != NULL) &&
 281                                 (o->flags & COMPLETE)) {
 282                         continue;
 283                 }
 284
 285                 remote_hex = sha1_to_hex(remote);
 286                 if (!fetching) {
 287                         struct strbuf c = STRBUF_INIT;
 288                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 289                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 290                         if (no_done)            strbuf_addstr(&c, " no-done");
 291                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 292                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 293                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 294                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 295                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 296                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 297                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 298                                                             git_user_agent_sanitized());
 299                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 300                         strbuf_release(&c);
 301                 } else
 302                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 303                 fetching++;
 304         }
 305
 306         if (!fetching) {
 307                 strbuf_release(&req_buf);
 308                 packet_flush(fd[1]);
 309                 return 1;
 310         }
 311
 312         if (is_repository_shallow())
 313                 write_shallow_commits(&req_buf, 1, NULL);
 314         if (args->depth > 0)
 315                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 316         packet_buf_flush(&req_buf);
 317         state_len = req_buf.len;
 318
 319         if (args->depth > 0) {
 320                 char *line;
 321                 unsigned char sha1[20];
 322
 323                 send_request(args, fd[1], &req_buf);
 324                 while ((line = packet_read_line(fd[0], NULL))) {
 325                         if (starts_with(line, "shallow ")) {
 326                                 if (get_sha1_hex(line + 8, sha1))
 327                                         die("invalid shallow line: %s", line);
 328                                 register_shallow(sha1);
 329                                 continue;
 330                         }
 331                         if (starts_with(line, "unshallow ")) {
 332                                 if (get_sha1_hex(line + 10, sha1))
 333                                         die("invalid unshallow line: %s", line);
 334                                 if (!lookup_object(sha1))
 335                                         die("object not found: %s", line);
 336                                 /* make sure that it is parsed as shallow */
 337                                 if (!parse_object(sha1))
 338                                         die("error in object: %s", line);
 339                                 if (unregister_shallow(sha1))
 340                                         die("no shallow found: %s", line);
 341                                 continue;
 342                         }
 343                         die("expected shallow/unshallow, got %s", line);
 344                 }
 345         } else if (!args->stateless_rpc)
 346                 send_request(args, fd[1], &req_buf);
 347
 348         if (!args->stateless_rpc) {
 349                 /* If we aren't using the stateless-rpc interface
 350                  * we don't need to retain the headers.
 351                  */
 352                 strbuf_setlen(&req_buf, 0);
 353                 state_len = 0;
 354         }
 355
 356         flushes = 0;
 357         retval = -1;
 358         while ((sha1 = get_rev())) {
 359                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 360                 if (args->verbose)
 361                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 362                 in_vain++;
 363                 if (flush_at <= ++count) {
 364                         int ack;
 365
 366                         packet_buf_flush(&req_buf);
 367                         send_request(args, fd[1], &req_buf);
 368                         strbuf_setlen(&req_buf, state_len);
 369                         flushes++;
 370                         flush_at = next_flush(args, count);
 371
 372                         /*
 373                          * We keep one window "ahead" of the other side, and
 374                          * will wait for an ACK only on the next one
 375                          */
 376                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 377                                 continue;
 378
 379                         consume_shallow_list(args, fd[0]);
 380                         do {
 381                                 ack = get_ack(fd[0], result_sha1);
 382                                 if (args->verbose && ack)
 383                                         fprintf(stderr, "got ack %d %s\n", ack,
 384                                                         sha1_to_hex(result_sha1));
 385                                 switch (ack) {
 386                                 case ACK:
 387                                         flushes = 0;
 388                                         multi_ack = 0;
 389                                         retval = 0;
 390                                         goto done;
 391                                 case ACK_common:
 392                                 case ACK_ready:
 393                                 case ACK_continue: {
 394                                         struct commit *commit =
 395                                                 lookup_commit(result_sha1);
 396                                         if (!commit)
 397                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 398                                         if (args->stateless_rpc
 399                                          && ack == ACK_common
 400                                          && !(commit->object.flags & COMMON)) {
 401                                                 /* We need to replay the have for this object
 402                                                  * on the next RPC request so the peer knows
 403                                                  * it is in common with us.
 404                                                  */
 405                                                 const char *hex = sha1_to_hex(result_sha1);
 406                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 407                                                 state_len = req_buf.len;
 408                                         }
 409                                         mark_common(commit, 0, 1);
 410                                         retval = 0;
 411                                         in_vain = 0;
 412                                         got_continue = 1;
 413                                         if (ack == ACK_ready) {
 414                                                 clear_prio_queue(&rev_list);
 415                                                 got_ready = 1;
 416                                         }
 417                                         break;
 418                                         }
 419                                 }
 420                         } while (ack);
 421                         flushes--;
 422                         if (got_continue && MAX_IN_VAIN < in_vain) {
 423                                 if (args->verbose)
 424                                         fprintf(stderr, "giving up\n");
 425                                 break; /* give up */
 426                         }
 427                 }
 428         }
 429 done:
 430         if (!got_ready || !no_done) {
 431                 packet_buf_write(&req_buf, "done\n");
 432                 send_request(args, fd[1], &req_buf);
 433         }
 434         if (args->verbose)
 435                 fprintf(stderr, "done\n");
 436         if (retval != 0) {
 437                 multi_ack = 0;
 438                 flushes++;
 439         }
 440         strbuf_release(&req_buf);
 441
 442         consume_shallow_list(args, fd[0]);
 443         while (flushes || multi_ack) {
 444                 int ack = get_ack(fd[0], result_sha1);
 445                 if (ack) {
 446                         if (args->verbose)
 447                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 448                                         sha1_to_hex(result_sha1));
 449                         if (ack == ACK)
 450                                 return 0;
 451                         multi_ack = 1;
 452                         continue;
 453                 }
 454                 flushes--;
 455         }
 456         /* it is no error to fetch into a completely empty repo */
 457         return count ? retval : 0;
 458 }
 459
 460 static struct commit_list *complete;
 461
 462 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 463 {
 464         struct object *o = parse_object(sha1);
 465
 466         while (o && o->type == OBJ_TAG) {
 467                 struct tag *t = (struct tag *) o;
 468                 if (!t->tagged)
 469                         break; /* broken repository */
 470                 o->flags |= COMPLETE;
 471                 o = parse_object(t->tagged->sha1);
 472         }
 473         if (o && o->type == OBJ_COMMIT) {
 474                 struct commit *commit = (struct commit *)o;
 475                 if (!(commit->object.flags & COMPLETE)) {
 476                         commit->object.flags |= COMPLETE;
 477                         commit_list_insert(commit, &complete);
 478                 }
 479         }
 480         return 0;
 481 }
 482
 483 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 484                                          unsigned long cutoff)
 485 {
 486         while (complete && cutoff <= complete->item->date) {
 487                 if (args->verbose)
 488                         fprintf(stderr, "Marking %s as complete\n",
 489                                 sha1_to_hex(complete->item->object.sha1));
 490                 pop_most_recent_commit(&complete, COMPLETE);
 491         }
 492 }
 493
 494 static void filter_refs(struct fetch_pack_args *args,
 495                         struct ref **refs,
 496                         struct ref **sought, int nr_sought)
 497 {
 498         struct ref *newlist = NULL;
 499         struct ref **newtail = &newlist;
 500         struct ref *ref, *next;
 501         int i;
 502
 503         i = 0;
 504         for (ref = *refs; ref; ref = next) {
 505                 int keep = 0;
 506                 next = ref->next;
 507
 508                 if (!memcmp(ref->name, "refs/", 5) &&
 509                     check_refname_format(ref->name + 5, 0))
 510                         ; /* trash */
 511                 else {
 512                         while (i < nr_sought) {
 513                                 int cmp = strcmp(ref->name, sought[i]->name);
 514                                 if (cmp < 0)
 515                                         break; /* definitely do not have it */
 516                                 else if (cmp == 0) {
 517                                         keep = 1; /* definitely have it */
 518                                         sought[i]->matched = 1;
 519                                 }
 520                                 i++;
 521                         }
 522                 }
 523
 524                 if (!keep && args->fetch_all &&
 525                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 526                         keep = 1;
 527
 528                 if (keep) {
 529                         *newtail = ref;
 530                         ref->next = NULL;
 531                         newtail = &ref->next;
 532                 } else {
 533                         free(ref);
 534                 }
 535         }
 536
 537         /* Append unmatched requests to the list */
 538         if (allow_tip_sha1_in_want) {
 539                 for (i = 0; i < nr_sought; i++) {
 540                         ref = sought[i];
 541                         if (ref->matched)
 542                                 continue;
 543                         if (get_sha1_hex(ref->name, ref->old_sha1))
 544                                 continue;
 545
 546                         ref->matched = 1;
 547                         *newtail = ref;
 548                         ref->next = NULL;
 549                         newtail = &ref->next;
 550                 }
 551         }
 552         *refs = newlist;
 553 }
 554
 555 static void mark_alternate_complete(const struct ref *ref, void *unused)
 556 {
 557         mark_complete(NULL, ref->old_sha1, 0, NULL);
 558 }
 559
 560 static int everything_local(struct fetch_pack_args *args,
 561                             struct ref **refs,
 562                             struct ref **sought, int nr_sought)
 563 {
 564         struct ref *ref;
 565         int retval;
 566         unsigned long cutoff = 0;
 567
 568         save_commit_buffer = 0;
 569
 570         for (ref = *refs; ref; ref = ref->next) {
 571                 struct object *o;
 572
 573                 if (!has_sha1_file(ref->old_sha1))
 574                         continue;
 575
 576                 o = parse_object(ref->old_sha1);
 577                 if (!o)
 578                         continue;
 579
 580                 /* We already have it -- which may mean that we were
 581                  * in sync with the other side at some time after
 582                  * that (it is OK if we guess wrong here).
 583                  */
 584                 if (o->type == OBJ_COMMIT) {
 585                         struct commit *commit = (struct commit *)o;
 586                         if (!cutoff || cutoff < commit->date)
 587                                 cutoff = commit->date;
 588                 }
 589         }
 590
 591         if (!args->depth) {
 592                 for_each_ref(mark_complete, NULL);
 593                 for_each_alternate_ref(mark_alternate_complete, NULL);
 594                 commit_list_sort_by_date(&complete);
 595                 if (cutoff)
 596                         mark_recent_complete_commits(args, cutoff);
 597         }
 598
 599         /*
 600          * Mark all complete remote refs as common refs.
 601          * Don't mark them common yet; the server has to be told so first.
 602          */
 603         for (ref = *refs; ref; ref = ref->next) {
 604                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 605                                              NULL, 0);
 606
 607                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 608                         continue;
 609
 610                 if (!(o->flags & SEEN)) {
 611                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 612
 613                         mark_common((struct commit *)o, 1, 1);
 614                 }
 615         }
 616
 617         filter_refs(args, refs, sought, nr_sought);
 618
 619         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 620                 const unsigned char *remote = ref->old_sha1;
 621                 unsigned char local[20];
 622                 struct object *o;
 623
 624                 o = lookup_object(remote);
 625                 if (!o || !(o->flags & COMPLETE)) {
 626                         retval = 0;
 627                         if (!args->verbose)
 628                                 continue;
 629                         fprintf(stderr,
 630                                 "want %s (%s)\n", sha1_to_hex(remote),
 631                                 ref->name);
 632                         continue;
 633                 }
 634
 635                 hashcpy(ref->new_sha1, local);
 636                 if (!args->verbose)
 637                         continue;
 638                 fprintf(stderr,
 639                         "already have %s (%s)\n", sha1_to_hex(remote),
 640                         ref->name);
 641         }
 642         return retval;
 643 }
 644
 645 static int sideband_demux(int in, int out, void *data)
 646 {
 647         int *xd = data;
 648
 649         int ret = recv_sideband("fetch-pack", xd[0], out);
 650         close(out);
 651         return ret;
 652 }
 653
 654 static int get_pack(struct fetch_pack_args *args,
 655                     int xd[2], char **pack_lockfile)
 656 {
 657         struct async demux;
 658         const char *argv[22];
 659         char keep_arg[256];
 660         char hdr_arg[256];
 661         const char **av, *cmd_name;
 662         int do_keep = args->keep_pack;
 663         struct child_process cmd;
 664         int ret;
 665
 666         memset(&demux, 0, sizeof(demux));
 667         if (use_sideband) {
 668                 /* xd[] is talking with upload-pack; subprocess reads from
 669                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 670                  * through demux->out.
 671                  */
 672                 demux.proc = sideband_demux;
 673                 demux.data = xd;
 674                 demux.out = -1;
 675                 if (start_async(&demux))
 676                         die("fetch-pack: unable to fork off sideband"
 677                             " demultiplexer");
 678         }
 679         else
 680                 demux.out = xd[0];
 681
 682         memset(&cmd, 0, sizeof(cmd));
 683         cmd.argv = argv;
 684         av = argv;
 685         *hdr_arg = 0;
 686         if (!args->keep_pack && unpack_limit) {
 687                 struct pack_header header;
 688
 689                 if (read_pack_header(demux.out, &header))
 690                         die("protocol error: bad pack header");
 691                 snprintf(hdr_arg, sizeof(hdr_arg),
 692                          "--pack_header=%"PRIu32",%"PRIu32,
 693                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 694                 if (ntohl(header.hdr_entries) < unpack_limit)
 695                         do_keep = 0;
 696                 else
 697                         do_keep = 1;
 698         }
 699
 700         if (alternate_shallow_file) {
 701                 *av++ = "--shallow-file";
 702                 *av++ = alternate_shallow_file;
 703         }
 704
 705         if (do_keep) {
 706                 if (pack_lockfile)
 707                         cmd.out = -1;
 708                 *av++ = cmd_name = "index-pack";
 709                 *av++ = "--stdin";
 710                 if (!args->quiet && !args->no_progress)
 711                         *av++ = "-v";
 712                 if (args->use_thin_pack)
 713                         *av++ = "--fix-thin";
 714                 if (args->lock_pack || unpack_limit) {
 715                         int s = sprintf(keep_arg,
 716                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 717                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 718                                 strcpy(keep_arg + s, "localhost");
 719                         *av++ = keep_arg;
 720                 }
 721                 if (args->check_self_contained_and_connected)
 722                         *av++ = "--check-self-contained-and-connected";
 723         }
 724         else {
 725                 *av++ = cmd_name = "unpack-objects";
 726                 if (args->quiet || args->no_progress)
 727                         *av++ = "-q";
 728                 args->check_self_contained_and_connected = 0;
 729         }
 730         if (*hdr_arg)
 731                 *av++ = hdr_arg;
 732         if (fetch_fsck_objects >= 0
 733             ? fetch_fsck_objects
 734             : transfer_fsck_objects >= 0
 735             ? transfer_fsck_objects
 736             : 0)
 737                 *av++ = "--strict";
 738         *av++ = NULL;
 739
 740         cmd.in = demux.out;
 741         cmd.git_cmd = 1;
 742         if (start_command(&cmd))
 743                 die("fetch-pack: unable to fork off %s", cmd_name);
 744         if (do_keep && pack_lockfile) {
 745                 *pack_lockfile = index_pack_lockfile(cmd.out);
 746                 close(cmd.out);
 747         }
 748
 749         if (!use_sideband)
 750                 /* Closed by start_command() */
 751                 xd[0] = -1;
 752
 753         ret = finish_command(&cmd);
 754         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 755                 args->self_contained_and_connected =
 756                         args->check_self_contained_and_connected &&
 757                         ret == 0;
 758         else
 759                 die("%s failed", cmd_name);
 760         if (use_sideband && finish_async(&demux))
 761                 die("error in sideband demultiplexer");
 762         return 0;
 763 }
 764
 765 static int cmp_ref_by_name(const void *a_, const void *b_)
 766 {
 767         const struct ref *a = *((const struct ref **)a_);
 768         const struct ref *b = *((const struct ref **)b_);
 769         return strcmp(a->name, b->name);
 770 }
 771
 772 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 773                                  int fd[2],
 774                                  const struct ref *orig_ref,
 775                                  struct ref **sought, int nr_sought,
 776                                  struct shallow_info *si,
 777                                  char **pack_lockfile)
 778 {
 779         struct ref *ref = copy_ref_list(orig_ref);
 780         unsigned char sha1[20];
 781         const char *agent_feature;
 782         int agent_len;
 783
 784         sort_ref_list(&ref, ref_compare_name);
 785         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 786
 787         if (is_repository_shallow() && !server_supports("shallow"))
 788                 die("Server does not support shallow clients");
 789         if (server_supports("multi_ack_detailed")) {
 790                 if (args->verbose)
 791                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 792                 multi_ack = 2;
 793                 if (server_supports("no-done")) {
 794                         if (args->verbose)
 795                                 fprintf(stderr, "Server supports no-done\n");
 796                         if (args->stateless_rpc)
 797                                 no_done = 1;
 798                 }
 799         }
 800         else if (server_supports("multi_ack")) {
 801                 if (args->verbose)
 802                         fprintf(stderr, "Server supports multi_ack\n");
 803                 multi_ack = 1;
 804         }
 805         if (server_supports("side-band-64k")) {
 806                 if (args->verbose)
 807                         fprintf(stderr, "Server supports side-band-64k\n");
 808                 use_sideband = 2;
 809         }
 810         else if (server_supports("side-band")) {
 811                 if (args->verbose)
 812                         fprintf(stderr, "Server supports side-band\n");
 813                 use_sideband = 1;
 814         }
 815         if (server_supports("allow-tip-sha1-in-want")) {
 816                 if (args->verbose)
 817                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 818                 allow_tip_sha1_in_want = 1;
 819         }
 820         if (!server_supports("thin-pack"))
 821                 args->use_thin_pack = 0;
 822         if (!server_supports("no-progress"))
 823                 args->no_progress = 0;
 824         if (!server_supports("include-tag"))
 825                 args->include_tag = 0;
 826         if (server_supports("ofs-delta")) {
 827                 if (args->verbose)
 828                         fprintf(stderr, "Server supports ofs-delta\n");
 829         } else
 830                 prefer_ofs_delta = 0;
 831
 832         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 833                 agent_supported = 1;
 834                 if (args->verbose && agent_len)
 835                         fprintf(stderr, "Server version is %.*s\n",
 836                                 agent_len, agent_feature);
 837         }
 838
 839         if (everything_local(args, &ref, sought, nr_sought)) {
 840                 packet_flush(fd[1]);
 841                 goto all_done;
 842         }
 843         if (find_common(args, fd, sha1, ref) < 0)
 844                 if (!args->keep_pack)
 845                         /* When cloning, it is not unusual to have
 846                          * no common commit.
 847                          */
 848                         warning("no common commits");
 849
 850         if (args->stateless_rpc)
 851                 packet_flush(fd[1]);
 852         if (args->depth > 0)
 853                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 854                                         NULL);
 855         else if (si->nr_ours || si->nr_theirs)
 856                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 857         else
 858                 alternate_shallow_file = NULL;
 859         if (get_pack(args, fd, pack_lockfile))
 860                 die("git fetch-pack: fetch failed.");
 861
 862  all_done:
 863         return ref;
 864 }
 865
 866 static int fetch_pack_config(const char *var, const char *value, void *cb)
 867 {
 868         if (strcmp(var, "fetch.unpacklimit") == 0) {
 869                 fetch_unpack_limit = git_config_int(var, value);
 870                 return 0;
 871         }
 872
 873         if (strcmp(var, "transfer.unpacklimit") == 0) {
 874                 transfer_unpack_limit = git_config_int(var, value);
 875                 return 0;
 876         }
 877
 878         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 879                 prefer_ofs_delta = git_config_bool(var, value);
 880                 return 0;
 881         }
 882
 883         if (!strcmp(var, "fetch.fsckobjects")) {
 884                 fetch_fsck_objects = git_config_bool(var, value);
 885                 return 0;
 886         }
 887
 888         if (!strcmp(var, "transfer.fsckobjects")) {
 889                 transfer_fsck_objects = git_config_bool(var, value);
 890                 return 0;
 891         }
 892
 893         return git_default_config(var, value, cb);
 894 }
 895
 896 static void fetch_pack_setup(void)
 897 {
 898         static int did_setup;
 899         if (did_setup)
 900                 return;
 901         git_config(fetch_pack_config, NULL);
 902         if (0 <= transfer_unpack_limit)
 903                 unpack_limit = transfer_unpack_limit;
 904         else if (0 <= fetch_unpack_limit)
 905                 unpack_limit = fetch_unpack_limit;
 906         did_setup = 1;
 907 }
 908
 909 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 910 {
 911         struct string_list names = STRING_LIST_INIT_NODUP;
 912         int src, dst;
 913
 914         for (src = dst = 0; src < nr; src++) {
 915                 struct string_list_item *item;
 916                 item = string_list_insert(&names, ref[src]->name);
 917                 if (item->util)
 918                         continue; /* already have it */
 919                 item->util = ref[src];
 920                 if (src != dst)
 921                         ref[dst] = ref[src];
 922                 dst++;
 923         }
 924         for (src = dst; src < nr; src++)
 925                 ref[src] = NULL;
 926         string_list_clear(&names, 0);
 927         return dst;
 928 }
 929
 930 static void update_shallow(struct fetch_pack_args *args,
 931                            struct ref **sought, int nr_sought,
 932                            struct shallow_info *si)
 933 {
 934         struct sha1_array ref = SHA1_ARRAY_INIT;
 935         int *status;
 936         int i;
 937
 938         if (args->depth > 0 && alternate_shallow_file) {
 939                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 940                         unlink_or_warn(git_path("shallow"));
 941                         rollback_lock_file(&shallow_lock);
 942                 } else
 943                         commit_lock_file(&shallow_lock);
 944                 return;
 945         }
 946
 947         if (!si->shallow || !si->shallow->nr)
 948                 return;
 949
 950         if (alternate_shallow_file) {
 951                 /*
 952                  * The temporary shallow file is only useful for
 953                  * index-pack and unpack-objects because it may
 954                  * contain more roots than we want. Delete it.
 955                  */
 956                 if (*alternate_shallow_file)
 957                         unlink(alternate_shallow_file);
 958                 free((char *)alternate_shallow_file);
 959         }
 960
 961         if (args->cloning) {
 962                 /*
 963                  * remote is shallow, but this is a clone, there are
 964                  * no objects in repo to worry about. Accept any
 965                  * shallow points that exist in the pack (iow in repo
 966                  * after get_pack() and reprepare_packed_git())
 967                  */
 968                 struct sha1_array extra = SHA1_ARRAY_INIT;
 969                 unsigned char (*sha1)[20] = si->shallow->sha1;
 970                 for (i = 0; i < si->shallow->nr; i++)
 971                         if (has_sha1_file(sha1[i]))
 972                                 sha1_array_append(&extra, sha1[i]);
 973                 if (extra.nr) {
 974                         setup_alternate_shallow(&shallow_lock,
 975                                                 &alternate_shallow_file,
 976                                                 &extra);
 977                         commit_lock_file(&shallow_lock);
 978                 }
 979                 sha1_array_clear(&extra);
 980                 return;
 981         }
 982
 983         if (!si->nr_ours && !si->nr_theirs)
 984                 return;
 985
 986         remove_nonexistent_theirs_shallow(si);
 987         if (!si->nr_ours && !si->nr_theirs)
 988                 return;
 989         for (i = 0; i < nr_sought; i++)
 990                 sha1_array_append(&ref, sought[i]->old_sha1);
 991         si->ref = &ref;
 992
 993         if (args->update_shallow) {
 994                 /*
 995                  * remote is also shallow, .git/shallow may be updated
 996                  * so all refs can be accepted. Make sure we only add
 997                  * shallow roots that are actually reachable from new
 998                  * refs.
 999                  */
1000                 struct sha1_array extra = SHA1_ARRAY_INIT;
1001                 unsigned char (*sha1)[20] = si->shallow->sha1;
1002                 assign_shallow_commits_to_refs(si, NULL, NULL);
1003                 if (!si->nr_ours && !si->nr_theirs) {
1004                         sha1_array_clear(&ref);
1005                         return;
1006                 }
1007                 for (i = 0; i < si->nr_ours; i++)
1008                         sha1_array_append(&extra, sha1[si->ours[i]]);
1009                 for (i = 0; i < si->nr_theirs; i++)
1010                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1011                 setup_alternate_shallow(&shallow_lock,
1012                                         &alternate_shallow_file,
1013                                         &extra);
1014                 commit_lock_file(&shallow_lock);
1015                 sha1_array_clear(&extra);
1016                 sha1_array_clear(&ref);
1017                 return;
1018         }
1019
1020         /*
1021          * remote is also shallow, check what ref is safe to update
1022          * without updating .git/shallow
1023          */
1024         status = xcalloc(nr_sought, sizeof(*status));
1025         assign_shallow_commits_to_refs(si, NULL, status);
1026         if (si->nr_ours || si->nr_theirs) {
1027                 for (i = 0; i < nr_sought; i++)
1028                         if (status[i])
1029                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1030         }
1031         free(status);
1032         sha1_array_clear(&ref);
1033 }
1034
1035 struct ref *fetch_pack(struct fetch_pack_args *args,
1036                        int fd[], struct child_process *conn,
1037                        const struct ref *ref,
1038                        const char *dest,
1039                        struct ref **sought, int nr_sought,
1040                        struct sha1_array *shallow,
1041                        char **pack_lockfile)
1042 {
1043         struct ref *ref_cpy;
1044         struct shallow_info si;
1045
1046         fetch_pack_setup();
1047         if (nr_sought)
1048                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1049
1050         if (!ref) {
1051                 packet_flush(fd[1]);
1052                 die("no matching remote head");
1053         }
1054         prepare_shallow_info(&si, shallow);
1055         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1056                                 &si, pack_lockfile);
1057         reprepare_packed_git();
1058         update_shallow(args, sought, nr_sought, &si);
1059         clear_shallow_info(&si);
1060         return ref_cpy;
1061 }