fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "transport.h"
  13 #include "version.h"
  14 #include "prio-queue.h"
  15
  16 static int transfer_unpack_limit = -1;
  17 static int fetch_unpack_limit = -1;
  18 static int unpack_limit = 100;
  19 static int prefer_ofs_delta = 1;
  20 static int no_done;
  21 static int fetch_fsck_objects = -1;
  22 static int transfer_fsck_objects = -1;
  23 static int agent_supported;
  24 static struct lock_file shallow_lock;
  25 static const char *alternate_shallow_file;
  26
  27 #define COMPLETE        (1U << 0)
  28 #define COMMON          (1U << 1)
  29 #define COMMON_REF      (1U << 2)
  30 #define SEEN            (1U << 3)
  31 #define POPPED          (1U << 4)
  32
  33 static int marked;
  34
  35 /*
  36  * After sending this many "have"s if we do not get any new ACK , we
  37  * give up traversing our history.
  38  */
  39 #define MAX_IN_VAIN 256
  40
  41 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  42 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  43
  44 static void rev_list_push(struct commit *commit, int mark)
  45 {
  46         if (!(commit->object.flags & mark)) {
  47                 commit->object.flags |= mark;
  48
  49                 if (parse_commit(commit))
  50                         return;
  51
  52                 prio_queue_put(&rev_list, commit);
  53
  54                 if (!(commit->object.flags & COMMON))
  55                         non_common_revs++;
  56         }
  57 }
  58
  59 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  60 {
  61         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  62
  63         if (o && o->type == OBJ_COMMIT)
  64                 rev_list_push((struct commit *)o, SEEN);
  65
  66         return 0;
  67 }
  68
  69 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  70 {
  71         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  72
  73         if (o && o->type == OBJ_COMMIT)
  74                 clear_commit_marks((struct commit *)o,
  75                                    COMMON | COMMON_REF | SEEN | POPPED);
  76         return 0;
  77 }
  78
  79 /*
  80    This function marks a rev and its ancestors as common.
  81    In some cases, it is desirable to mark only the ancestors (for example
  82    when only the server does not yet know that they are common).
  83 */
  84
  85 static void mark_common(struct commit *commit,
  86                 int ancestors_only, int dont_parse)
  87 {
  88         if (commit != NULL && !(commit->object.flags & COMMON)) {
  89                 struct object *o = (struct object *)commit;
  90
  91                 if (!ancestors_only)
  92                         o->flags |= COMMON;
  93
  94                 if (!(o->flags & SEEN))
  95                         rev_list_push(commit, SEEN);
  96                 else {
  97                         struct commit_list *parents;
  98
  99                         if (!ancestors_only && !(o->flags & POPPED))
 100                                 non_common_revs--;
 101                         if (!o->parsed && !dont_parse)
 102                                 if (parse_commit(commit))
 103                                         return;
 104
 105                         for (parents = commit->parents;
 106                                         parents;
 107                                         parents = parents->next)
 108                                 mark_common(parents->item, 0, dont_parse);
 109                 }
 110         }
 111 }
 112
 113 /*
 114   Get the next rev to send, ignoring the common.
 115 */
 116
 117 static const unsigned char *get_rev(void)
 118 {
 119         struct commit *commit = NULL;
 120
 121         while (commit == NULL) {
 122                 unsigned int mark;
 123                 struct commit_list *parents;
 124
 125                 if (rev_list.nr == 0 || non_common_revs == 0)
 126                         return NULL;
 127
 128                 commit = prio_queue_get(&rev_list);
 129                 parse_commit(commit);
 130                 parents = commit->parents;
 131
 132                 commit->object.flags |= POPPED;
 133                 if (!(commit->object.flags & COMMON))
 134                         non_common_revs--;
 135
 136                 if (commit->object.flags & COMMON) {
 137                         /* do not send "have", and ignore ancestors */
 138                         commit = NULL;
 139                         mark = COMMON | SEEN;
 140                 } else if (commit->object.flags & COMMON_REF)
 141                         /* send "have", and ignore ancestors */
 142                         mark = COMMON | SEEN;
 143                 else
 144                         /* send "have", also for its ancestors */
 145                         mark = SEEN;
 146
 147                 while (parents) {
 148                         if (!(parents->item->object.flags & SEEN))
 149                                 rev_list_push(parents->item, mark);
 150                         if (mark & COMMON)
 151                                 mark_common(parents->item, 1, 0);
 152                         parents = parents->next;
 153                 }
 154         }
 155
 156         return commit->object.sha1;
 157 }
 158
 159 enum ack_type {
 160         NAK = 0,
 161         ACK,
 162         ACK_continue,
 163         ACK_common,
 164         ACK_ready
 165 };
 166
 167 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 168 {
 169         if (args->stateless_rpc && args->depth > 0) {
 170                 /* If we sent a depth we will get back "duplicate"
 171                  * shallow and unshallow commands every time there
 172                  * is a block of have lines exchanged.
 173                  */
 174                 char *line;
 175                 while ((line = packet_read_line(fd, NULL))) {
 176                         if (!prefixcmp(line, "shallow "))
 177                                 continue;
 178                         if (!prefixcmp(line, "unshallow "))
 179                                 continue;
 180                         die("git fetch-pack: expected shallow list");
 181                 }
 182         }
 183 }
 184
 185 struct write_shallow_data {
 186         struct strbuf *out;
 187         int use_pack_protocol;
 188         int count;
 189 };
 190
 191 static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
 192 {
 193         struct write_shallow_data *data = cb_data;
 194         const char *hex = sha1_to_hex(graft->sha1);
 195         data->count++;
 196         if (data->use_pack_protocol)
 197                 packet_buf_write(data->out, "shallow %s", hex);
 198         else {
 199                 strbuf_addstr(data->out, hex);
 200                 strbuf_addch(data->out, '\n');
 201         }
 202         return 0;
 203 }
 204
 205 static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
 206 {
 207         struct write_shallow_data data;
 208         data.out = out;
 209         data.use_pack_protocol = use_pack_protocol;
 210         data.count = 0;
 211         for_each_commit_graft(write_one_shallow, &data);
 212         return data.count;
 213 }
 214
 215 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 216 {
 217         int len;
 218         char *line = packet_read_line(fd, &len);
 219
 220         if (!len)
 221                 die("git fetch-pack: expected ACK/NAK, got EOF");
 222         if (!strcmp(line, "NAK"))
 223                 return NAK;
 224         if (!prefixcmp(line, "ACK ")) {
 225                 if (!get_sha1_hex(line+4, result_sha1)) {
 226                         if (len < 45)
 227                                 return ACK;
 228                         if (strstr(line+45, "continue"))
 229                                 return ACK_continue;
 230                         if (strstr(line+45, "common"))
 231                                 return ACK_common;
 232                         if (strstr(line+45, "ready"))
 233                                 return ACK_ready;
 234                         return ACK;
 235                 }
 236         }
 237         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 238 }
 239
 240 static void send_request(struct fetch_pack_args *args,
 241                          int fd, struct strbuf *buf)
 242 {
 243         if (args->stateless_rpc) {
 244                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 245                 packet_flush(fd);
 246         } else
 247                 write_or_die(fd, buf->buf, buf->len);
 248 }
 249
 250 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 251 {
 252         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 253 }
 254
 255 #define INITIAL_FLUSH 16
 256 #define PIPESAFE_FLUSH 32
 257 #define LARGE_FLUSH 1024
 258
 259 static int next_flush(struct fetch_pack_args *args, int count)
 260 {
 261         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 262
 263         if (count < flush_limit)
 264                 count <<= 1;
 265         else
 266                 count += flush_limit;
 267         return count;
 268 }
 269
 270 static int find_common(struct fetch_pack_args *args,
 271                        int fd[2], unsigned char *result_sha1,
 272                        struct ref *refs)
 273 {
 274         int fetching;
 275         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 276         const unsigned char *sha1;
 277         unsigned in_vain = 0;
 278         int got_continue = 0;
 279         int got_ready = 0;
 280         struct strbuf req_buf = STRBUF_INIT;
 281         size_t state_len = 0;
 282
 283         if (args->stateless_rpc && multi_ack == 1)
 284                 die("--stateless-rpc requires multi_ack_detailed");
 285         if (marked)
 286                 for_each_ref(clear_marks, NULL);
 287         marked = 1;
 288
 289         for_each_ref(rev_list_insert_ref, NULL);
 290         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 291
 292         fetching = 0;
 293         for ( ; refs ; refs = refs->next) {
 294                 unsigned char *remote = refs->old_sha1;
 295                 const char *remote_hex;
 296                 struct object *o;
 297
 298                 /*
 299                  * If that object is complete (i.e. it is an ancestor of a
 300                  * local ref), we tell them we have it but do not have to
 301                  * tell them about its ancestors, which they already know
 302                  * about.
 303                  *
 304                  * We use lookup_object here because we are only
 305                  * interested in the case we *know* the object is
 306                  * reachable and we have already scanned it.
 307                  */
 308                 if (((o = lookup_object(remote)) != NULL) &&
 309                                 (o->flags & COMPLETE)) {
 310                         continue;
 311                 }
 312
 313                 remote_hex = sha1_to_hex(remote);
 314                 if (!fetching) {
 315                         struct strbuf c = STRBUF_INIT;
 316                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 317                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 318                         if (no_done)            strbuf_addstr(&c, " no-done");
 319                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 320                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 321                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 322                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 323                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 324                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 325                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 326                                                             git_user_agent_sanitized());
 327                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 328                         strbuf_release(&c);
 329                 } else
 330                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 331                 fetching++;
 332         }
 333
 334         if (!fetching) {
 335                 strbuf_release(&req_buf);
 336                 packet_flush(fd[1]);
 337                 return 1;
 338         }
 339
 340         if (is_repository_shallow())
 341                 write_shallow_commits(&req_buf, 1);
 342         if (args->depth > 0)
 343                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 344         packet_buf_flush(&req_buf);
 345         state_len = req_buf.len;
 346
 347         if (args->depth > 0) {
 348                 char *line;
 349                 unsigned char sha1[20];
 350
 351                 send_request(args, fd[1], &req_buf);
 352                 while ((line = packet_read_line(fd[0], NULL))) {
 353                         if (!prefixcmp(line, "shallow ")) {
 354                                 if (get_sha1_hex(line + 8, sha1))
 355                                         die("invalid shallow line: %s", line);
 356                                 register_shallow(sha1);
 357                                 continue;
 358                         }
 359                         if (!prefixcmp(line, "unshallow ")) {
 360                                 if (get_sha1_hex(line + 10, sha1))
 361                                         die("invalid unshallow line: %s", line);
 362                                 if (!lookup_object(sha1))
 363                                         die("object not found: %s", line);
 364                                 /* make sure that it is parsed as shallow */
 365                                 if (!parse_object(sha1))
 366                                         die("error in object: %s", line);
 367                                 if (unregister_shallow(sha1))
 368                                         die("no shallow found: %s", line);
 369                                 continue;
 370                         }
 371                         die("expected shallow/unshallow, got %s", line);
 372                 }
 373         } else if (!args->stateless_rpc)
 374                 send_request(args, fd[1], &req_buf);
 375
 376         if (!args->stateless_rpc) {
 377                 /* If we aren't using the stateless-rpc interface
 378                  * we don't need to retain the headers.
 379                  */
 380                 strbuf_setlen(&req_buf, 0);
 381                 state_len = 0;
 382         }
 383
 384         flushes = 0;
 385         retval = -1;
 386         while ((sha1 = get_rev())) {
 387                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 388                 if (args->verbose)
 389                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 390                 in_vain++;
 391                 if (flush_at <= ++count) {
 392                         int ack;
 393
 394                         packet_buf_flush(&req_buf);
 395                         send_request(args, fd[1], &req_buf);
 396                         strbuf_setlen(&req_buf, state_len);
 397                         flushes++;
 398                         flush_at = next_flush(args, count);
 399
 400                         /*
 401                          * We keep one window "ahead" of the other side, and
 402                          * will wait for an ACK only on the next one
 403                          */
 404                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 405                                 continue;
 406
 407                         consume_shallow_list(args, fd[0]);
 408                         do {
 409                                 ack = get_ack(fd[0], result_sha1);
 410                                 if (args->verbose && ack)
 411                                         fprintf(stderr, "got ack %d %s\n", ack,
 412                                                         sha1_to_hex(result_sha1));
 413                                 switch (ack) {
 414                                 case ACK:
 415                                         flushes = 0;
 416                                         multi_ack = 0;
 417                                         retval = 0;
 418                                         goto done;
 419                                 case ACK_common:
 420                                 case ACK_ready:
 421                                 case ACK_continue: {
 422                                         struct commit *commit =
 423                                                 lookup_commit(result_sha1);
 424                                         if (!commit)
 425                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 426                                         if (args->stateless_rpc
 427                                          && ack == ACK_common
 428                                          && !(commit->object.flags & COMMON)) {
 429                                                 /* We need to replay the have for this object
 430                                                  * on the next RPC request so the peer knows
 431                                                  * it is in common with us.
 432                                                  */
 433                                                 const char *hex = sha1_to_hex(result_sha1);
 434                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 435                                                 state_len = req_buf.len;
 436                                         }
 437                                         mark_common(commit, 0, 1);
 438                                         retval = 0;
 439                                         in_vain = 0;
 440                                         got_continue = 1;
 441                                         if (ack == ACK_ready) {
 442                                                 clear_prio_queue(&rev_list);
 443                                                 got_ready = 1;
 444                                         }
 445                                         break;
 446                                         }
 447                                 }
 448                         } while (ack);
 449                         flushes--;
 450                         if (got_continue && MAX_IN_VAIN < in_vain) {
 451                                 if (args->verbose)
 452                                         fprintf(stderr, "giving up\n");
 453                                 break; /* give up */
 454                         }
 455                 }
 456         }
 457 done:
 458         if (!got_ready || !no_done) {
 459                 packet_buf_write(&req_buf, "done\n");
 460                 send_request(args, fd[1], &req_buf);
 461         }
 462         if (args->verbose)
 463                 fprintf(stderr, "done\n");
 464         if (retval != 0) {
 465                 multi_ack = 0;
 466                 flushes++;
 467         }
 468         strbuf_release(&req_buf);
 469
 470         consume_shallow_list(args, fd[0]);
 471         while (flushes || multi_ack) {
 472                 int ack = get_ack(fd[0], result_sha1);
 473                 if (ack) {
 474                         if (args->verbose)
 475                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 476                                         sha1_to_hex(result_sha1));
 477                         if (ack == ACK)
 478                                 return 0;
 479                         multi_ack = 1;
 480                         continue;
 481                 }
 482                 flushes--;
 483         }
 484         /* it is no error to fetch into a completely empty repo */
 485         return count ? retval : 0;
 486 }
 487
 488 static struct commit_list *complete;
 489
 490 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 491 {
 492         struct object *o = parse_object(sha1);
 493
 494         while (o && o->type == OBJ_TAG) {
 495                 struct tag *t = (struct tag *) o;
 496                 if (!t->tagged)
 497                         break; /* broken repository */
 498                 o->flags |= COMPLETE;
 499                 o = parse_object(t->tagged->sha1);
 500         }
 501         if (o && o->type == OBJ_COMMIT) {
 502                 struct commit *commit = (struct commit *)o;
 503                 if (!(commit->object.flags & COMPLETE)) {
 504                         commit->object.flags |= COMPLETE;
 505                         commit_list_insert(commit, &complete);
 506                 }
 507         }
 508         return 0;
 509 }
 510
 511 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 512                                          unsigned long cutoff)
 513 {
 514         while (complete && cutoff <= complete->item->date) {
 515                 if (args->verbose)
 516                         fprintf(stderr, "Marking %s as complete\n",
 517                                 sha1_to_hex(complete->item->object.sha1));
 518                 pop_most_recent_commit(&complete, COMPLETE);
 519         }
 520 }
 521
 522 static void filter_refs(struct fetch_pack_args *args,
 523                         struct ref **refs,
 524                         struct ref **sought, int nr_sought)
 525 {
 526         struct ref *newlist = NULL;
 527         struct ref **newtail = &newlist;
 528         struct ref *ref, *next;
 529         int i;
 530
 531         i = 0;
 532         for (ref = *refs; ref; ref = next) {
 533                 int keep = 0;
 534                 next = ref->next;
 535
 536                 if (!memcmp(ref->name, "refs/", 5) &&
 537                     check_refname_format(ref->name + 5, 0))
 538                         ; /* trash */
 539                 else {
 540                         while (i < nr_sought) {
 541                                 int cmp = strcmp(ref->name, sought[i]->name);
 542                                 if (cmp < 0)
 543                                         break; /* definitely do not have it */
 544                                 else if (cmp == 0) {
 545                                         keep = 1; /* definitely have it */
 546                                         sought[i]->matched = 1;
 547                                 }
 548                                 i++;
 549                         }
 550                 }
 551
 552                 if (!keep && args->fetch_all &&
 553                     (!args->depth || prefixcmp(ref->name, "refs/tags/")))
 554                         keep = 1;
 555
 556                 if (keep) {
 557                         *newtail = ref;
 558                         ref->next = NULL;
 559                         newtail = &ref->next;
 560                 } else {
 561                         free(ref);
 562                 }
 563         }
 564
 565         /* Append unmatched requests to the list */
 566         if (allow_tip_sha1_in_want) {
 567                 for (i = 0; i < nr_sought; i++) {
 568                         ref = sought[i];
 569                         if (ref->matched)
 570                                 continue;
 571                         if (get_sha1_hex(ref->name, ref->old_sha1))
 572                                 continue;
 573
 574                         ref->matched = 1;
 575                         *newtail = ref;
 576                         ref->next = NULL;
 577                         newtail = &ref->next;
 578                 }
 579         }
 580         *refs = newlist;
 581 }
 582
 583 static void mark_alternate_complete(const struct ref *ref, void *unused)
 584 {
 585         mark_complete(NULL, ref->old_sha1, 0, NULL);
 586 }
 587
 588 static int everything_local(struct fetch_pack_args *args,
 589                             struct ref **refs,
 590                             struct ref **sought, int nr_sought)
 591 {
 592         struct ref *ref;
 593         int retval;
 594         unsigned long cutoff = 0;
 595
 596         save_commit_buffer = 0;
 597
 598         for (ref = *refs; ref; ref = ref->next) {
 599                 struct object *o;
 600
 601                 if (!has_sha1_file(ref->old_sha1))
 602                         continue;
 603
 604                 o = parse_object(ref->old_sha1);
 605                 if (!o)
 606                         continue;
 607
 608                 /* We already have it -- which may mean that we were
 609                  * in sync with the other side at some time after
 610                  * that (it is OK if we guess wrong here).
 611                  */
 612                 if (o->type == OBJ_COMMIT) {
 613                         struct commit *commit = (struct commit *)o;
 614                         if (!cutoff || cutoff < commit->date)
 615                                 cutoff = commit->date;
 616                 }
 617         }
 618
 619         if (!args->depth) {
 620                 for_each_ref(mark_complete, NULL);
 621                 for_each_alternate_ref(mark_alternate_complete, NULL);
 622                 commit_list_sort_by_date(&complete);
 623                 if (cutoff)
 624                         mark_recent_complete_commits(args, cutoff);
 625         }
 626
 627         /*
 628          * Mark all complete remote refs as common refs.
 629          * Don't mark them common yet; the server has to be told so first.
 630          */
 631         for (ref = *refs; ref; ref = ref->next) {
 632                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 633                                              NULL, 0);
 634
 635                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 636                         continue;
 637
 638                 if (!(o->flags & SEEN)) {
 639                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 640
 641                         mark_common((struct commit *)o, 1, 1);
 642                 }
 643         }
 644
 645         filter_refs(args, refs, sought, nr_sought);
 646
 647         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 648                 const unsigned char *remote = ref->old_sha1;
 649                 unsigned char local[20];
 650                 struct object *o;
 651
 652                 o = lookup_object(remote);
 653                 if (!o || !(o->flags & COMPLETE)) {
 654                         retval = 0;
 655                         if (!args->verbose)
 656                                 continue;
 657                         fprintf(stderr,
 658                                 "want %s (%s)\n", sha1_to_hex(remote),
 659                                 ref->name);
 660                         continue;
 661                 }
 662
 663                 hashcpy(ref->new_sha1, local);
 664                 if (!args->verbose)
 665                         continue;
 666                 fprintf(stderr,
 667                         "already have %s (%s)\n", sha1_to_hex(remote),
 668                         ref->name);
 669         }
 670         return retval;
 671 }
 672
 673 static int sideband_demux(int in, int out, void *data)
 674 {
 675         int *xd = data;
 676
 677         int ret = recv_sideband("fetch-pack", xd[0], out);
 678         close(out);
 679         return ret;
 680 }
 681
 682 static int get_pack(struct fetch_pack_args *args,
 683                     int xd[2], char **pack_lockfile)
 684 {
 685         struct async demux;
 686         const char *argv[22];
 687         char keep_arg[256];
 688         char hdr_arg[256];
 689         const char **av;
 690         int do_keep = args->keep_pack;
 691         struct child_process cmd;
 692         int ret;
 693
 694         memset(&demux, 0, sizeof(demux));
 695         if (use_sideband) {
 696                 /* xd[] is talking with upload-pack; subprocess reads from
 697                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 698                  * through demux->out.
 699                  */
 700                 demux.proc = sideband_demux;
 701                 demux.data = xd;
 702                 demux.out = -1;
 703                 if (start_async(&demux))
 704                         die("fetch-pack: unable to fork off sideband"
 705                             " demultiplexer");
 706         }
 707         else
 708                 demux.out = xd[0];
 709
 710         memset(&cmd, 0, sizeof(cmd));
 711         cmd.argv = argv;
 712         av = argv;
 713         *hdr_arg = 0;
 714         if (!args->keep_pack && unpack_limit) {
 715                 struct pack_header header;
 716
 717                 if (read_pack_header(demux.out, &header))
 718                         die("protocol error: bad pack header");
 719                 snprintf(hdr_arg, sizeof(hdr_arg),
 720                          "--pack_header=%"PRIu32",%"PRIu32,
 721                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 722                 if (ntohl(header.hdr_entries) < unpack_limit)
 723                         do_keep = 0;
 724                 else
 725                         do_keep = 1;
 726         }
 727
 728         if (alternate_shallow_file) {
 729                 *av++ = "--shallow-file";
 730                 *av++ = alternate_shallow_file;
 731         }
 732
 733         if (do_keep) {
 734                 if (pack_lockfile)
 735                         cmd.out = -1;
 736                 *av++ = "index-pack";
 737                 *av++ = "--stdin";
 738                 if (!args->quiet && !args->no_progress)
 739                         *av++ = "-v";
 740                 if (args->use_thin_pack)
 741                         *av++ = "--fix-thin";
 742                 if (args->lock_pack || unpack_limit) {
 743                         int s = sprintf(keep_arg,
 744                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 745                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 746                                 strcpy(keep_arg + s, "localhost");
 747                         *av++ = keep_arg;
 748                 }
 749                 if (args->check_self_contained_and_connected)
 750                         *av++ = "--check-self-contained-and-connected";
 751         }
 752         else {
 753                 *av++ = "unpack-objects";
 754                 if (args->quiet || args->no_progress)
 755                         *av++ = "-q";
 756                 args->check_self_contained_and_connected = 0;
 757         }
 758         if (*hdr_arg)
 759                 *av++ = hdr_arg;
 760         if (fetch_fsck_objects >= 0
 761             ? fetch_fsck_objects
 762             : transfer_fsck_objects >= 0
 763             ? transfer_fsck_objects
 764             : 0)
 765                 *av++ = "--strict";
 766         *av++ = NULL;
 767
 768         cmd.in = demux.out;
 769         cmd.git_cmd = 1;
 770         if (start_command(&cmd))
 771                 die("fetch-pack: unable to fork off %s", argv[0]);
 772         if (do_keep && pack_lockfile) {
 773                 *pack_lockfile = index_pack_lockfile(cmd.out);
 774                 close(cmd.out);
 775         }
 776
 777         ret = finish_command(&cmd);
 778         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 779                 args->self_contained_and_connected =
 780                         args->check_self_contained_and_connected &&
 781                         ret == 0;
 782         else
 783                 die("%s failed", argv[0]);
 784         if (use_sideband && finish_async(&demux))
 785                 die("error in sideband demultiplexer");
 786         return 0;
 787 }
 788
 789 static int cmp_ref_by_name(const void *a_, const void *b_)
 790 {
 791         const struct ref *a = *((const struct ref **)a_);
 792         const struct ref *b = *((const struct ref **)b_);
 793         return strcmp(a->name, b->name);
 794 }
 795
 796 static void setup_alternate_shallow(void)
 797 {
 798         struct strbuf sb = STRBUF_INIT;
 799         int fd;
 800
 801         check_shallow_file_for_update();
 802         fd = hold_lock_file_for_update(&shallow_lock, git_path("shallow"),
 803                                        LOCK_DIE_ON_ERROR);
 804         if (write_shallow_commits(&sb, 0)) {
 805                 if (write_in_full(fd, sb.buf, sb.len) != sb.len)
 806                         die_errno("failed to write to %s", shallow_lock.filename);
 807                 alternate_shallow_file = shallow_lock.filename;
 808         } else
 809                 /*
 810                  * is_repository_shallow() sees empty string as "no
 811                  * shallow file".
 812                  */
 813                 alternate_shallow_file = "";
 814         strbuf_release(&sb);
 815 }
 816
 817 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 818                                  int fd[2],
 819                                  const struct ref *orig_ref,
 820                                  struct ref **sought, int nr_sought,
 821                                  char **pack_lockfile)
 822 {
 823         struct ref *ref = copy_ref_list(orig_ref);
 824         unsigned char sha1[20];
 825         const char *agent_feature;
 826         int agent_len;
 827
 828         sort_ref_list(&ref, ref_compare_name);
 829         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 830
 831         if (is_repository_shallow() && !server_supports("shallow"))
 832                 die("Server does not support shallow clients");
 833         if (server_supports("multi_ack_detailed")) {
 834                 if (args->verbose)
 835                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 836                 multi_ack = 2;
 837                 if (server_supports("no-done")) {
 838                         if (args->verbose)
 839                                 fprintf(stderr, "Server supports no-done\n");
 840                         if (args->stateless_rpc)
 841                                 no_done = 1;
 842                 }
 843         }
 844         else if (server_supports("multi_ack")) {
 845                 if (args->verbose)
 846                         fprintf(stderr, "Server supports multi_ack\n");
 847                 multi_ack = 1;
 848         }
 849         if (server_supports("side-band-64k")) {
 850                 if (args->verbose)
 851                         fprintf(stderr, "Server supports side-band-64k\n");
 852                 use_sideband = 2;
 853         }
 854         else if (server_supports("side-band")) {
 855                 if (args->verbose)
 856                         fprintf(stderr, "Server supports side-band\n");
 857                 use_sideband = 1;
 858         }
 859         if (server_supports("allow-tip-sha1-in-want")) {
 860                 if (args->verbose)
 861                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 862                 allow_tip_sha1_in_want = 1;
 863         }
 864         if (!server_supports("thin-pack"))
 865                 args->use_thin_pack = 0;
 866         if (!server_supports("no-progress"))
 867                 args->no_progress = 0;
 868         if (!server_supports("include-tag"))
 869                 args->include_tag = 0;
 870         if (server_supports("ofs-delta")) {
 871                 if (args->verbose)
 872                         fprintf(stderr, "Server supports ofs-delta\n");
 873         } else
 874                 prefer_ofs_delta = 0;
 875
 876         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 877                 agent_supported = 1;
 878                 if (args->verbose && agent_len)
 879                         fprintf(stderr, "Server version is %.*s\n",
 880                                 agent_len, agent_feature);
 881         }
 882
 883         if (everything_local(args, &ref, sought, nr_sought)) {
 884                 packet_flush(fd[1]);
 885                 goto all_done;
 886         }
 887         if (find_common(args, fd, sha1, ref) < 0)
 888                 if (!args->keep_pack)
 889                         /* When cloning, it is not unusual to have
 890                          * no common commit.
 891                          */
 892                         warning("no common commits");
 893
 894         if (args->stateless_rpc)
 895                 packet_flush(fd[1]);
 896         if (args->depth > 0)
 897                 setup_alternate_shallow();
 898         if (get_pack(args, fd, pack_lockfile))
 899                 die("git fetch-pack: fetch failed.");
 900
 901  all_done:
 902         return ref;
 903 }
 904
 905 static int fetch_pack_config(const char *var, const char *value, void *cb)
 906 {
 907         if (strcmp(var, "fetch.unpacklimit") == 0) {
 908                 fetch_unpack_limit = git_config_int(var, value);
 909                 return 0;
 910         }
 911
 912         if (strcmp(var, "transfer.unpacklimit") == 0) {
 913                 transfer_unpack_limit = git_config_int(var, value);
 914                 return 0;
 915         }
 916
 917         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 918                 prefer_ofs_delta = git_config_bool(var, value);
 919                 return 0;
 920         }
 921
 922         if (!strcmp(var, "fetch.fsckobjects")) {
 923                 fetch_fsck_objects = git_config_bool(var, value);
 924                 return 0;
 925         }
 926
 927         if (!strcmp(var, "transfer.fsckobjects")) {
 928                 transfer_fsck_objects = git_config_bool(var, value);
 929                 return 0;
 930         }
 931
 932         return git_default_config(var, value, cb);
 933 }
 934
 935 static void fetch_pack_setup(void)
 936 {
 937         static int did_setup;
 938         if (did_setup)
 939                 return;
 940         git_config(fetch_pack_config, NULL);
 941         if (0 <= transfer_unpack_limit)
 942                 unpack_limit = transfer_unpack_limit;
 943         else if (0 <= fetch_unpack_limit)
 944                 unpack_limit = fetch_unpack_limit;
 945         did_setup = 1;
 946 }
 947
 948 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 949 {
 950         struct string_list names = STRING_LIST_INIT_NODUP;
 951         int src, dst;
 952
 953         for (src = dst = 0; src < nr; src++) {
 954                 struct string_list_item *item;
 955                 item = string_list_insert(&names, ref[src]->name);
 956                 if (item->util)
 957                         continue; /* already have it */
 958                 item->util = ref[src];
 959                 if (src != dst)
 960                         ref[dst] = ref[src];
 961                 dst++;
 962         }
 963         for (src = dst; src < nr; src++)
 964                 ref[src] = NULL;
 965         string_list_clear(&names, 0);
 966         return dst;
 967 }
 968
 969 struct ref *fetch_pack(struct fetch_pack_args *args,
 970                        int fd[], struct child_process *conn,
 971                        const struct ref *ref,
 972                        const char *dest,
 973                        struct ref **sought, int nr_sought,
 974                        char **pack_lockfile)
 975 {
 976         struct ref *ref_cpy;
 977
 978         fetch_pack_setup();
 979         if (nr_sought)
 980                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
 981
 982         if (!ref) {
 983                 packet_flush(fd[1]);
 984                 die("no matching remote head");
 985         }
 986         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile);
 987
 988         if (alternate_shallow_file) {
 989                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 990                         unlink_or_warn(git_path("shallow"));
 991                         rollback_lock_file(&shallow_lock);
 992                 } else
 993                         commit_lock_file(&shallow_lock);
 994         }
 995
 996         reprepare_packed_git();
 997         return ref_cpy;
 998 }