fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "connect.h"
  13 #include "transport.h"
  14 #include "version.h"
  15 #include "prio-queue.h"
  16
  17 static int transfer_unpack_limit = -1;
  18 static int fetch_unpack_limit = -1;
  19 static int unpack_limit = 100;
  20 static int prefer_ofs_delta = 1;
  21 static int no_done;
  22 static int fetch_fsck_objects = -1;
  23 static int transfer_fsck_objects = -1;
  24 static int agent_supported;
  25 static struct lock_file shallow_lock;
  26 static const char *alternate_shallow_file;
  27
  28 #define COMPLETE        (1U << 0)
  29 #define COMMON          (1U << 1)
  30 #define COMMON_REF      (1U << 2)
  31 #define SEEN            (1U << 3)
  32 #define POPPED          (1U << 4)
  33
  34 static int marked;
  35
  36 /*
  37  * After sending this many "have"s if we do not get any new ACK , we
  38  * give up traversing our history.
  39  */
  40 #define MAX_IN_VAIN 256
  41
  42 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  43 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  44
  45 static void rev_list_push(struct commit *commit, int mark)
  46 {
  47         if (!(commit->object.flags & mark)) {
  48                 commit->object.flags |= mark;
  49
  50                 if (!(commit->object.parsed))
  51                         if (parse_commit(commit))
  52                                 return;
  53
  54                 prio_queue_put(&rev_list, commit);
  55
  56                 if (!(commit->object.flags & COMMON))
  57                         non_common_revs++;
  58         }
  59 }
  60
  61 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  62 {
  63         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  64
  65         if (o && o->type == OBJ_COMMIT)
  66                 rev_list_push((struct commit *)o, SEEN);
  67
  68         return 0;
  69 }
  70
  71 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  72 {
  73         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  74
  75         if (o && o->type == OBJ_COMMIT)
  76                 clear_commit_marks((struct commit *)o,
  77                                    COMMON | COMMON_REF | SEEN | POPPED);
  78         return 0;
  79 }
  80
  81 /*
  82    This function marks a rev and its ancestors as common.
  83    In some cases, it is desirable to mark only the ancestors (for example
  84    when only the server does not yet know that they are common).
  85 */
  86
  87 static void mark_common(struct commit *commit,
  88                 int ancestors_only, int dont_parse)
  89 {
  90         if (commit != NULL && !(commit->object.flags & COMMON)) {
  91                 struct object *o = (struct object *)commit;
  92
  93                 if (!ancestors_only)
  94                         o->flags |= COMMON;
  95
  96                 if (!(o->flags & SEEN))
  97                         rev_list_push(commit, SEEN);
  98                 else {
  99                         struct commit_list *parents;
 100
 101                         if (!ancestors_only && !(o->flags & POPPED))
 102                                 non_common_revs--;
 103                         if (!o->parsed && !dont_parse)
 104                                 if (parse_commit(commit))
 105                                         return;
 106
 107                         for (parents = commit->parents;
 108                                         parents;
 109                                         parents = parents->next)
 110                                 mark_common(parents->item, 0, dont_parse);
 111                 }
 112         }
 113 }
 114
 115 /*
 116   Get the next rev to send, ignoring the common.
 117 */
 118
 119 static const unsigned char *get_rev(void)
 120 {
 121         struct commit *commit = NULL;
 122
 123         while (commit == NULL) {
 124                 unsigned int mark;
 125                 struct commit_list *parents;
 126
 127                 if (rev_list.nr == 0 || non_common_revs == 0)
 128                         return NULL;
 129
 130                 commit = prio_queue_get(&rev_list);
 131                 if (!commit->object.parsed)
 132                         parse_commit(commit);
 133                 parents = commit->parents;
 134
 135                 commit->object.flags |= POPPED;
 136                 if (!(commit->object.flags & COMMON))
 137                         non_common_revs--;
 138
 139                 if (commit->object.flags & COMMON) {
 140                         /* do not send "have", and ignore ancestors */
 141                         commit = NULL;
 142                         mark = COMMON | SEEN;
 143                 } else if (commit->object.flags & COMMON_REF)
 144                         /* send "have", and ignore ancestors */
 145                         mark = COMMON | SEEN;
 146                 else
 147                         /* send "have", also for its ancestors */
 148                         mark = SEEN;
 149
 150                 while (parents) {
 151                         if (!(parents->item->object.flags & SEEN))
 152                                 rev_list_push(parents->item, mark);
 153                         if (mark & COMMON)
 154                                 mark_common(parents->item, 1, 0);
 155                         parents = parents->next;
 156                 }
 157         }
 158
 159         return commit->object.sha1;
 160 }
 161
 162 enum ack_type {
 163         NAK = 0,
 164         ACK,
 165         ACK_continue,
 166         ACK_common,
 167         ACK_ready
 168 };
 169
 170 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 171 {
 172         if (args->stateless_rpc && args->depth > 0) {
 173                 /* If we sent a depth we will get back "duplicate"
 174                  * shallow and unshallow commands every time there
 175                  * is a block of have lines exchanged.
 176                  */
 177                 char *line;
 178                 while ((line = packet_read_line(fd, NULL))) {
 179                         if (!prefixcmp(line, "shallow "))
 180                                 continue;
 181                         if (!prefixcmp(line, "unshallow "))
 182                                 continue;
 183                         die("git fetch-pack: expected shallow list");
 184                 }
 185         }
 186 }
 187
 188 struct write_shallow_data {
 189         struct strbuf *out;
 190         int use_pack_protocol;
 191         int count;
 192 };
 193
 194 static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
 195 {
 196         struct write_shallow_data *data = cb_data;
 197         const char *hex = sha1_to_hex(graft->sha1);
 198         data->count++;
 199         if (data->use_pack_protocol)
 200                 packet_buf_write(data->out, "shallow %s", hex);
 201         else {
 202                 strbuf_addstr(data->out, hex);
 203                 strbuf_addch(data->out, '\n');
 204         }
 205         return 0;
 206 }
 207
 208 static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
 209 {
 210         struct write_shallow_data data;
 211         data.out = out;
 212         data.use_pack_protocol = use_pack_protocol;
 213         data.count = 0;
 214         for_each_commit_graft(write_one_shallow, &data);
 215         return data.count;
 216 }
 217
 218 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 219 {
 220         int len;
 221         char *line = packet_read_line(fd, &len);
 222
 223         if (!len)
 224                 die("git fetch-pack: expected ACK/NAK, got EOF");
 225         if (!strcmp(line, "NAK"))
 226                 return NAK;
 227         if (!prefixcmp(line, "ACK ")) {
 228                 if (!get_sha1_hex(line+4, result_sha1)) {
 229                         if (len < 45)
 230                                 return ACK;
 231                         if (strstr(line+45, "continue"))
 232                                 return ACK_continue;
 233                         if (strstr(line+45, "common"))
 234                                 return ACK_common;
 235                         if (strstr(line+45, "ready"))
 236                                 return ACK_ready;
 237                         return ACK;
 238                 }
 239         }
 240         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 241 }
 242
 243 static void send_request(struct fetch_pack_args *args,
 244                          int fd, struct strbuf *buf)
 245 {
 246         if (args->stateless_rpc) {
 247                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 248                 packet_flush(fd);
 249         } else
 250                 write_or_die(fd, buf->buf, buf->len);
 251 }
 252
 253 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 254 {
 255         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 256 }
 257
 258 #define INITIAL_FLUSH 16
 259 #define PIPESAFE_FLUSH 32
 260 #define LARGE_FLUSH 1024
 261
 262 static int next_flush(struct fetch_pack_args *args, int count)
 263 {
 264         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 265
 266         if (count < flush_limit)
 267                 count <<= 1;
 268         else
 269                 count += flush_limit;
 270         return count;
 271 }
 272
 273 static int find_common(struct fetch_pack_args *args,
 274                        int fd[2], unsigned char *result_sha1,
 275                        struct ref *refs)
 276 {
 277         int fetching;
 278         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 279         const unsigned char *sha1;
 280         unsigned in_vain = 0;
 281         int got_continue = 0;
 282         int got_ready = 0;
 283         struct strbuf req_buf = STRBUF_INIT;
 284         size_t state_len = 0;
 285
 286         if (args->stateless_rpc && multi_ack == 1)
 287                 die("--stateless-rpc requires multi_ack_detailed");
 288         if (marked)
 289                 for_each_ref(clear_marks, NULL);
 290         marked = 1;
 291
 292         for_each_ref(rev_list_insert_ref, NULL);
 293         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 294
 295         fetching = 0;
 296         for ( ; refs ; refs = refs->next) {
 297                 unsigned char *remote = refs->old_sha1;
 298                 const char *remote_hex;
 299                 struct object *o;
 300
 301                 /*
 302                  * If that object is complete (i.e. it is an ancestor of a
 303                  * local ref), we tell them we have it but do not have to
 304                  * tell them about its ancestors, which they already know
 305                  * about.
 306                  *
 307                  * We use lookup_object here because we are only
 308                  * interested in the case we *know* the object is
 309                  * reachable and we have already scanned it.
 310                  */
 311                 if (((o = lookup_object(remote)) != NULL) &&
 312                                 (o->flags & COMPLETE)) {
 313                         continue;
 314                 }
 315
 316                 remote_hex = sha1_to_hex(remote);
 317                 if (!fetching) {
 318                         struct strbuf c = STRBUF_INIT;
 319                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 320                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 321                         if (no_done)            strbuf_addstr(&c, " no-done");
 322                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 323                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 324                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 325                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 326                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 327                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 328                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 329                                                             git_user_agent_sanitized());
 330                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 331                         strbuf_release(&c);
 332                 } else
 333                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 334                 fetching++;
 335         }
 336
 337         if (!fetching) {
 338                 strbuf_release(&req_buf);
 339                 packet_flush(fd[1]);
 340                 return 1;
 341         }
 342
 343         if (is_repository_shallow())
 344                 write_shallow_commits(&req_buf, 1);
 345         if (args->depth > 0)
 346                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 347         packet_buf_flush(&req_buf);
 348         state_len = req_buf.len;
 349
 350         if (args->depth > 0) {
 351                 char *line;
 352                 unsigned char sha1[20];
 353
 354                 send_request(args, fd[1], &req_buf);
 355                 while ((line = packet_read_line(fd[0], NULL))) {
 356                         if (!prefixcmp(line, "shallow ")) {
 357                                 if (get_sha1_hex(line + 8, sha1))
 358                                         die("invalid shallow line: %s", line);
 359                                 register_shallow(sha1);
 360                                 continue;
 361                         }
 362                         if (!prefixcmp(line, "unshallow ")) {
 363                                 if (get_sha1_hex(line + 10, sha1))
 364                                         die("invalid unshallow line: %s", line);
 365                                 if (!lookup_object(sha1))
 366                                         die("object not found: %s", line);
 367                                 /* make sure that it is parsed as shallow */
 368                                 if (!parse_object(sha1))
 369                                         die("error in object: %s", line);
 370                                 if (unregister_shallow(sha1))
 371                                         die("no shallow found: %s", line);
 372                                 continue;
 373                         }
 374                         die("expected shallow/unshallow, got %s", line);
 375                 }
 376         } else if (!args->stateless_rpc)
 377                 send_request(args, fd[1], &req_buf);
 378
 379         if (!args->stateless_rpc) {
 380                 /* If we aren't using the stateless-rpc interface
 381                  * we don't need to retain the headers.
 382                  */
 383                 strbuf_setlen(&req_buf, 0);
 384                 state_len = 0;
 385         }
 386
 387         flushes = 0;
 388         retval = -1;
 389         while ((sha1 = get_rev())) {
 390                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 391                 if (args->verbose)
 392                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 393                 in_vain++;
 394                 if (flush_at <= ++count) {
 395                         int ack;
 396
 397                         packet_buf_flush(&req_buf);
 398                         send_request(args, fd[1], &req_buf);
 399                         strbuf_setlen(&req_buf, state_len);
 400                         flushes++;
 401                         flush_at = next_flush(args, count);
 402
 403                         /*
 404                          * We keep one window "ahead" of the other side, and
 405                          * will wait for an ACK only on the next one
 406                          */
 407                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 408                                 continue;
 409
 410                         consume_shallow_list(args, fd[0]);
 411                         do {
 412                                 ack = get_ack(fd[0], result_sha1);
 413                                 if (args->verbose && ack)
 414                                         fprintf(stderr, "got ack %d %s\n", ack,
 415                                                         sha1_to_hex(result_sha1));
 416                                 switch (ack) {
 417                                 case ACK:
 418                                         flushes = 0;
 419                                         multi_ack = 0;
 420                                         retval = 0;
 421                                         goto done;
 422                                 case ACK_common:
 423                                 case ACK_ready:
 424                                 case ACK_continue: {
 425                                         struct commit *commit =
 426                                                 lookup_commit(result_sha1);
 427                                         if (!commit)
 428                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 429                                         if (args->stateless_rpc
 430                                          && ack == ACK_common
 431                                          && !(commit->object.flags & COMMON)) {
 432                                                 /* We need to replay the have for this object
 433                                                  * on the next RPC request so the peer knows
 434                                                  * it is in common with us.
 435                                                  */
 436                                                 const char *hex = sha1_to_hex(result_sha1);
 437                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 438                                                 state_len = req_buf.len;
 439                                         }
 440                                         mark_common(commit, 0, 1);
 441                                         retval = 0;
 442                                         in_vain = 0;
 443                                         got_continue = 1;
 444                                         if (ack == ACK_ready) {
 445                                                 clear_prio_queue(&rev_list);
 446                                                 got_ready = 1;
 447                                         }
 448                                         break;
 449                                         }
 450                                 }
 451                         } while (ack);
 452                         flushes--;
 453                         if (got_continue && MAX_IN_VAIN < in_vain) {
 454                                 if (args->verbose)
 455                                         fprintf(stderr, "giving up\n");
 456                                 break; /* give up */
 457                         }
 458                 }
 459         }
 460 done:
 461         if (!got_ready || !no_done) {
 462                 packet_buf_write(&req_buf, "done\n");
 463                 send_request(args, fd[1], &req_buf);
 464         }
 465         if (args->verbose)
 466                 fprintf(stderr, "done\n");
 467         if (retval != 0) {
 468                 multi_ack = 0;
 469                 flushes++;
 470         }
 471         strbuf_release(&req_buf);
 472
 473         consume_shallow_list(args, fd[0]);
 474         while (flushes || multi_ack) {
 475                 int ack = get_ack(fd[0], result_sha1);
 476                 if (ack) {
 477                         if (args->verbose)
 478                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 479                                         sha1_to_hex(result_sha1));
 480                         if (ack == ACK)
 481                                 return 0;
 482                         multi_ack = 1;
 483                         continue;
 484                 }
 485                 flushes--;
 486         }
 487         /* it is no error to fetch into a completely empty repo */
 488         return count ? retval : 0;
 489 }
 490
 491 static struct commit_list *complete;
 492
 493 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 494 {
 495         struct object *o = parse_object(sha1);
 496
 497         while (o && o->type == OBJ_TAG) {
 498                 struct tag *t = (struct tag *) o;
 499                 if (!t->tagged)
 500                         break; /* broken repository */
 501                 o->flags |= COMPLETE;
 502                 o = parse_object(t->tagged->sha1);
 503         }
 504         if (o && o->type == OBJ_COMMIT) {
 505                 struct commit *commit = (struct commit *)o;
 506                 if (!(commit->object.flags & COMPLETE)) {
 507                         commit->object.flags |= COMPLETE;
 508                         commit_list_insert(commit, &complete);
 509                 }
 510         }
 511         return 0;
 512 }
 513
 514 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 515                                          unsigned long cutoff)
 516 {
 517         while (complete && cutoff <= complete->item->date) {
 518                 if (args->verbose)
 519                         fprintf(stderr, "Marking %s as complete\n",
 520                                 sha1_to_hex(complete->item->object.sha1));
 521                 pop_most_recent_commit(&complete, COMPLETE);
 522         }
 523 }
 524
 525 static void filter_refs(struct fetch_pack_args *args,
 526                         struct ref **refs,
 527                         struct ref **sought, int nr_sought)
 528 {
 529         struct ref *newlist = NULL;
 530         struct ref **newtail = &newlist;
 531         struct ref *ref, *next;
 532         int i;
 533
 534         i = 0;
 535         for (ref = *refs; ref; ref = next) {
 536                 int keep = 0;
 537                 next = ref->next;
 538
 539                 if (!memcmp(ref->name, "refs/", 5) &&
 540                     check_refname_format(ref->name + 5, 0))
 541                         ; /* trash */
 542                 else {
 543                         while (i < nr_sought) {
 544                                 int cmp = strcmp(ref->name, sought[i]->name);
 545                                 if (cmp < 0)
 546                                         break; /* definitely do not have it */
 547                                 else if (cmp == 0) {
 548                                         keep = 1; /* definitely have it */
 549                                         sought[i]->matched = 1;
 550                                 }
 551                                 i++;
 552                         }
 553                 }
 554
 555                 if (!keep && args->fetch_all &&
 556                     (!args->depth || prefixcmp(ref->name, "refs/tags/")))
 557                         keep = 1;
 558
 559                 if (keep) {
 560                         *newtail = ref;
 561                         ref->next = NULL;
 562                         newtail = &ref->next;
 563                 } else {
 564                         free(ref);
 565                 }
 566         }
 567
 568         /* Append unmatched requests to the list */
 569         if (allow_tip_sha1_in_want) {
 570                 for (i = 0; i < nr_sought; i++) {
 571                         ref = sought[i];
 572                         if (ref->matched)
 573                                 continue;
 574                         if (get_sha1_hex(ref->name, ref->old_sha1))
 575                                 continue;
 576
 577                         ref->matched = 1;
 578                         *newtail = ref;
 579                         ref->next = NULL;
 580                         newtail = &ref->next;
 581                 }
 582         }
 583         *refs = newlist;
 584 }
 585
 586 static void mark_alternate_complete(const struct ref *ref, void *unused)
 587 {
 588         mark_complete(NULL, ref->old_sha1, 0, NULL);
 589 }
 590
 591 static int everything_local(struct fetch_pack_args *args,
 592                             struct ref **refs,
 593                             struct ref **sought, int nr_sought)
 594 {
 595         struct ref *ref;
 596         int retval;
 597         unsigned long cutoff = 0;
 598
 599         save_commit_buffer = 0;
 600
 601         for (ref = *refs; ref; ref = ref->next) {
 602                 struct object *o;
 603
 604                 if (!has_sha1_file(ref->old_sha1))
 605                         continue;
 606
 607                 o = parse_object(ref->old_sha1);
 608                 if (!o)
 609                         continue;
 610
 611                 /* We already have it -- which may mean that we were
 612                  * in sync with the other side at some time after
 613                  * that (it is OK if we guess wrong here).
 614                  */
 615                 if (o->type == OBJ_COMMIT) {
 616                         struct commit *commit = (struct commit *)o;
 617                         if (!cutoff || cutoff < commit->date)
 618                                 cutoff = commit->date;
 619                 }
 620         }
 621
 622         if (!args->depth) {
 623                 for_each_ref(mark_complete, NULL);
 624                 for_each_alternate_ref(mark_alternate_complete, NULL);
 625                 commit_list_sort_by_date(&complete);
 626                 if (cutoff)
 627                         mark_recent_complete_commits(args, cutoff);
 628         }
 629
 630         /*
 631          * Mark all complete remote refs as common refs.
 632          * Don't mark them common yet; the server has to be told so first.
 633          */
 634         for (ref = *refs; ref; ref = ref->next) {
 635                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 636                                              NULL, 0);
 637
 638                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 639                         continue;
 640
 641                 if (!(o->flags & SEEN)) {
 642                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 643
 644                         mark_common((struct commit *)o, 1, 1);
 645                 }
 646         }
 647
 648         filter_refs(args, refs, sought, nr_sought);
 649
 650         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 651                 const unsigned char *remote = ref->old_sha1;
 652                 unsigned char local[20];
 653                 struct object *o;
 654
 655                 o = lookup_object(remote);
 656                 if (!o || !(o->flags & COMPLETE)) {
 657                         retval = 0;
 658                         if (!args->verbose)
 659                                 continue;
 660                         fprintf(stderr,
 661                                 "want %s (%s)\n", sha1_to_hex(remote),
 662                                 ref->name);
 663                         continue;
 664                 }
 665
 666                 hashcpy(ref->new_sha1, local);
 667                 if (!args->verbose)
 668                         continue;
 669                 fprintf(stderr,
 670                         "already have %s (%s)\n", sha1_to_hex(remote),
 671                         ref->name);
 672         }
 673         return retval;
 674 }
 675
 676 static int sideband_demux(int in, int out, void *data)
 677 {
 678         int *xd = data;
 679
 680         int ret = recv_sideband("fetch-pack", xd[0], out);
 681         close(out);
 682         return ret;
 683 }
 684
 685 static int get_pack(struct fetch_pack_args *args,
 686                     int xd[2], char **pack_lockfile)
 687 {
 688         struct async demux;
 689         const char *argv[22];
 690         char keep_arg[256];
 691         char hdr_arg[256];
 692         const char **av;
 693         int do_keep = args->keep_pack;
 694         struct child_process cmd;
 695         int ret;
 696
 697         memset(&demux, 0, sizeof(demux));
 698         if (use_sideband) {
 699                 /* xd[] is talking with upload-pack; subprocess reads from
 700                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 701                  * through demux->out.
 702                  */
 703                 demux.proc = sideband_demux;
 704                 demux.data = xd;
 705                 demux.out = -1;
 706                 if (start_async(&demux))
 707                         die("fetch-pack: unable to fork off sideband"
 708                             " demultiplexer");
 709         }
 710         else
 711                 demux.out = xd[0];
 712
 713         memset(&cmd, 0, sizeof(cmd));
 714         cmd.argv = argv;
 715         av = argv;
 716         *hdr_arg = 0;
 717         if (!args->keep_pack && unpack_limit) {
 718                 struct pack_header header;
 719
 720                 if (read_pack_header(demux.out, &header))
 721                         die("protocol error: bad pack header");
 722                 snprintf(hdr_arg, sizeof(hdr_arg),
 723                          "--pack_header=%"PRIu32",%"PRIu32,
 724                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 725                 if (ntohl(header.hdr_entries) < unpack_limit)
 726                         do_keep = 0;
 727                 else
 728                         do_keep = 1;
 729         }
 730
 731         if (alternate_shallow_file) {
 732                 *av++ = "--shallow-file";
 733                 *av++ = alternate_shallow_file;
 734         }
 735
 736         if (do_keep) {
 737                 if (pack_lockfile)
 738                         cmd.out = -1;
 739                 *av++ = "index-pack";
 740                 *av++ = "--stdin";
 741                 if (!args->quiet && !args->no_progress)
 742                         *av++ = "-v";
 743                 if (args->use_thin_pack)
 744                         *av++ = "--fix-thin";
 745                 if (args->lock_pack || unpack_limit) {
 746                         int s = sprintf(keep_arg,
 747                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 748                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 749                                 strcpy(keep_arg + s, "localhost");
 750                         *av++ = keep_arg;
 751                 }
 752                 if (args->check_self_contained_and_connected)
 753                         *av++ = "--check-self-contained-and-connected";
 754         }
 755         else {
 756                 *av++ = "unpack-objects";
 757                 if (args->quiet || args->no_progress)
 758                         *av++ = "-q";
 759                 args->check_self_contained_and_connected = 0;
 760         }
 761         if (*hdr_arg)
 762                 *av++ = hdr_arg;
 763         if (fetch_fsck_objects >= 0
 764             ? fetch_fsck_objects
 765             : transfer_fsck_objects >= 0
 766             ? transfer_fsck_objects
 767             : 0)
 768                 *av++ = "--strict";
 769         *av++ = NULL;
 770
 771         cmd.in = demux.out;
 772         cmd.git_cmd = 1;
 773         if (start_command(&cmd))
 774                 die("fetch-pack: unable to fork off %s", argv[0]);
 775         if (do_keep && pack_lockfile) {
 776                 *pack_lockfile = index_pack_lockfile(cmd.out);
 777                 close(cmd.out);
 778         }
 779
 780         ret = finish_command(&cmd);
 781         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 782                 args->self_contained_and_connected =
 783                         args->check_self_contained_and_connected &&
 784                         ret == 0;
 785         else
 786                 die("%s failed", argv[0]);
 787         if (use_sideband && finish_async(&demux))
 788                 die("error in sideband demultiplexer");
 789         return 0;
 790 }
 791
 792 static int cmp_ref_by_name(const void *a_, const void *b_)
 793 {
 794         const struct ref *a = *((const struct ref **)a_);
 795         const struct ref *b = *((const struct ref **)b_);
 796         return strcmp(a->name, b->name);
 797 }
 798
 799 static void setup_alternate_shallow(void)
 800 {
 801         struct strbuf sb = STRBUF_INIT;
 802         int fd;
 803
 804         check_shallow_file_for_update();
 805         fd = hold_lock_file_for_update(&shallow_lock, git_path("shallow"),
 806                                        LOCK_DIE_ON_ERROR);
 807         if (write_shallow_commits(&sb, 0)) {
 808                 if (write_in_full(fd, sb.buf, sb.len) != sb.len)
 809                         die_errno("failed to write to %s", shallow_lock.filename);
 810                 alternate_shallow_file = shallow_lock.filename;
 811         } else
 812                 /*
 813                  * is_repository_shallow() sees empty string as "no
 814                  * shallow file".
 815                  */
 816                 alternate_shallow_file = "";
 817         strbuf_release(&sb);
 818 }
 819
 820 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 821                                  int fd[2],
 822                                  const struct ref *orig_ref,
 823                                  struct ref **sought, int nr_sought,
 824                                  char **pack_lockfile)
 825 {
 826         struct ref *ref = copy_ref_list(orig_ref);
 827         unsigned char sha1[20];
 828         const char *agent_feature;
 829         int agent_len;
 830
 831         sort_ref_list(&ref, ref_compare_name);
 832         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 833
 834         if (is_repository_shallow() && !server_supports("shallow"))
 835                 die("Server does not support shallow clients");
 836         if (server_supports("multi_ack_detailed")) {
 837                 if (args->verbose)
 838                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 839                 multi_ack = 2;
 840                 if (server_supports("no-done")) {
 841                         if (args->verbose)
 842                                 fprintf(stderr, "Server supports no-done\n");
 843                         if (args->stateless_rpc)
 844                                 no_done = 1;
 845                 }
 846         }
 847         else if (server_supports("multi_ack")) {
 848                 if (args->verbose)
 849                         fprintf(stderr, "Server supports multi_ack\n");
 850                 multi_ack = 1;
 851         }
 852         if (server_supports("side-band-64k")) {
 853                 if (args->verbose)
 854                         fprintf(stderr, "Server supports side-band-64k\n");
 855                 use_sideband = 2;
 856         }
 857         else if (server_supports("side-band")) {
 858                 if (args->verbose)
 859                         fprintf(stderr, "Server supports side-band\n");
 860                 use_sideband = 1;
 861         }
 862         if (server_supports("allow-tip-sha1-in-want")) {
 863                 if (args->verbose)
 864                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 865                 allow_tip_sha1_in_want = 1;
 866         }
 867         if (!server_supports("thin-pack"))
 868                 args->use_thin_pack = 0;
 869         if (!server_supports("no-progress"))
 870                 args->no_progress = 0;
 871         if (!server_supports("include-tag"))
 872                 args->include_tag = 0;
 873         if (server_supports("ofs-delta")) {
 874                 if (args->verbose)
 875                         fprintf(stderr, "Server supports ofs-delta\n");
 876         } else
 877                 prefer_ofs_delta = 0;
 878
 879         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 880                 agent_supported = 1;
 881                 if (args->verbose && agent_len)
 882                         fprintf(stderr, "Server version is %.*s\n",
 883                                 agent_len, agent_feature);
 884         }
 885
 886         if (everything_local(args, &ref, sought, nr_sought)) {
 887                 packet_flush(fd[1]);
 888                 goto all_done;
 889         }
 890         if (find_common(args, fd, sha1, ref) < 0)
 891                 if (!args->keep_pack)
 892                         /* When cloning, it is not unusual to have
 893                          * no common commit.
 894                          */
 895                         warning("no common commits");
 896
 897         if (args->stateless_rpc)
 898                 packet_flush(fd[1]);
 899         if (args->depth > 0)
 900                 setup_alternate_shallow();
 901         else
 902                 alternate_shallow_file = NULL;
 903         if (get_pack(args, fd, pack_lockfile))
 904                 die("git fetch-pack: fetch failed.");
 905
 906  all_done:
 907         return ref;
 908 }
 909
 910 static int fetch_pack_config(const char *var, const char *value, void *cb)
 911 {
 912         if (strcmp(var, "fetch.unpacklimit") == 0) {
 913                 fetch_unpack_limit = git_config_int(var, value);
 914                 return 0;
 915         }
 916
 917         if (strcmp(var, "transfer.unpacklimit") == 0) {
 918                 transfer_unpack_limit = git_config_int(var, value);
 919                 return 0;
 920         }
 921
 922         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 923                 prefer_ofs_delta = git_config_bool(var, value);
 924                 return 0;
 925         }
 926
 927         if (!strcmp(var, "fetch.fsckobjects")) {
 928                 fetch_fsck_objects = git_config_bool(var, value);
 929                 return 0;
 930         }
 931
 932         if (!strcmp(var, "transfer.fsckobjects")) {
 933                 transfer_fsck_objects = git_config_bool(var, value);
 934                 return 0;
 935         }
 936
 937         return git_default_config(var, value, cb);
 938 }
 939
 940 static void fetch_pack_setup(void)
 941 {
 942         static int did_setup;
 943         if (did_setup)
 944                 return;
 945         git_config(fetch_pack_config, NULL);
 946         if (0 <= transfer_unpack_limit)
 947                 unpack_limit = transfer_unpack_limit;
 948         else if (0 <= fetch_unpack_limit)
 949                 unpack_limit = fetch_unpack_limit;
 950         did_setup = 1;
 951 }
 952
 953 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 954 {
 955         struct string_list names = STRING_LIST_INIT_NODUP;
 956         int src, dst;
 957
 958         for (src = dst = 0; src < nr; src++) {
 959                 struct string_list_item *item;
 960                 item = string_list_insert(&names, ref[src]->name);
 961                 if (item->util)
 962                         continue; /* already have it */
 963                 item->util = ref[src];
 964                 if (src != dst)
 965                         ref[dst] = ref[src];
 966                 dst++;
 967         }
 968         for (src = dst; src < nr; src++)
 969                 ref[src] = NULL;
 970         string_list_clear(&names, 0);
 971         return dst;
 972 }
 973
 974 struct ref *fetch_pack(struct fetch_pack_args *args,
 975                        int fd[], struct child_process *conn,
 976                        const struct ref *ref,
 977                        const char *dest,
 978                        struct ref **sought, int nr_sought,
 979                        char **pack_lockfile)
 980 {
 981         struct ref *ref_cpy;
 982
 983         fetch_pack_setup();
 984         if (nr_sought)
 985                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
 986
 987         if (!ref) {
 988                 packet_flush(fd[1]);
 989                 die("no matching remote head");
 990         }
 991         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile);
 992
 993         if (args->depth > 0 && alternate_shallow_file) {
 994                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 995                         unlink_or_warn(git_path("shallow"));
 996                         rollback_lock_file(&shallow_lock);
 997                 } else
 998                         commit_lock_file(&shallow_lock);
 999         }
1000
1001         reprepare_packed_git();
1002         return ref_cpy;
1003 }