fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "transport.h"
  13 #include "version.h"
  14
  15 static int transfer_unpack_limit = -1;
  16 static int fetch_unpack_limit = -1;
  17 static int unpack_limit = 100;
  18 static int prefer_ofs_delta = 1;
  19 static int no_done;
  20 static int fetch_fsck_objects = -1;
  21 static int transfer_fsck_objects = -1;
  22 static int agent_supported;
  23
  24 #define COMPLETE        (1U << 0)
  25 #define COMMON          (1U << 1)
  26 #define COMMON_REF      (1U << 2)
  27 #define SEEN            (1U << 3)
  28 #define POPPED          (1U << 4)
  29
  30 static int marked;
  31
  32 /*
  33  * After sending this many "have"s if we do not get any new ACK , we
  34  * give up traversing our history.
  35  */
  36 #define MAX_IN_VAIN 256
  37
  38 static struct commit_list *rev_list;
  39 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  40
  41 static void rev_list_push(struct commit *commit, int mark)
  42 {
  43         if (!(commit->object.flags & mark)) {
  44                 commit->object.flags |= mark;
  45
  46                 if (!(commit->object.parsed))
  47                         if (parse_commit(commit))
  48                                 return;
  49
  50                 commit_list_insert_by_date(commit, &rev_list);
  51
  52                 if (!(commit->object.flags & COMMON))
  53                         non_common_revs++;
  54         }
  55 }
  56
  57 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  58 {
  59         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  60
  61         if (o && o->type == OBJ_COMMIT)
  62                 rev_list_push((struct commit *)o, SEEN);
  63
  64         return 0;
  65 }
  66
  67 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  68 {
  69         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  70
  71         if (o && o->type == OBJ_COMMIT)
  72                 clear_commit_marks((struct commit *)o,
  73                                    COMMON | COMMON_REF | SEEN | POPPED);
  74         return 0;
  75 }
  76
  77 /*
  78    This function marks a rev and its ancestors as common.
  79    In some cases, it is desirable to mark only the ancestors (for example
  80    when only the server does not yet know that they are common).
  81 */
  82
  83 static void mark_common(struct commit *commit,
  84                 int ancestors_only, int dont_parse)
  85 {
  86         if (commit != NULL && !(commit->object.flags & COMMON)) {
  87                 struct object *o = (struct object *)commit;
  88
  89                 if (!ancestors_only)
  90                         o->flags |= COMMON;
  91
  92                 if (!(o->flags & SEEN))
  93                         rev_list_push(commit, SEEN);
  94                 else {
  95                         struct commit_list *parents;
  96
  97                         if (!ancestors_only && !(o->flags & POPPED))
  98                                 non_common_revs--;
  99                         if (!o->parsed && !dont_parse)
 100                                 if (parse_commit(commit))
 101                                         return;
 102
 103                         for (parents = commit->parents;
 104                                         parents;
 105                                         parents = parents->next)
 106                                 mark_common(parents->item, 0, dont_parse);
 107                 }
 108         }
 109 }
 110
 111 /*
 112   Get the next rev to send, ignoring the common.
 113 */
 114
 115 static const unsigned char *get_rev(void)
 116 {
 117         struct commit *commit = NULL;
 118
 119         while (commit == NULL) {
 120                 unsigned int mark;
 121                 struct commit_list *parents;
 122
 123                 if (rev_list == NULL || non_common_revs == 0)
 124                         return NULL;
 125
 126                 commit = rev_list->item;
 127                 if (!commit->object.parsed)
 128                         parse_commit(commit);
 129                 parents = commit->parents;
 130
 131                 commit->object.flags |= POPPED;
 132                 if (!(commit->object.flags & COMMON))
 133                         non_common_revs--;
 134
 135                 if (commit->object.flags & COMMON) {
 136                         /* do not send "have", and ignore ancestors */
 137                         commit = NULL;
 138                         mark = COMMON | SEEN;
 139                 } else if (commit->object.flags & COMMON_REF)
 140                         /* send "have", and ignore ancestors */
 141                         mark = COMMON | SEEN;
 142                 else
 143                         /* send "have", also for its ancestors */
 144                         mark = SEEN;
 145
 146                 while (parents) {
 147                         if (!(parents->item->object.flags & SEEN))
 148                                 rev_list_push(parents->item, mark);
 149                         if (mark & COMMON)
 150                                 mark_common(parents->item, 1, 0);
 151                         parents = parents->next;
 152                 }
 153
 154                 rev_list = rev_list->next;
 155         }
 156
 157         return commit->object.sha1;
 158 }
 159
 160 enum ack_type {
 161         NAK = 0,
 162         ACK,
 163         ACK_continue,
 164         ACK_common,
 165         ACK_ready
 166 };
 167
 168 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 169 {
 170         if (args->stateless_rpc && args->depth > 0) {
 171                 /* If we sent a depth we will get back "duplicate"
 172                  * shallow and unshallow commands every time there
 173                  * is a block of have lines exchanged.
 174                  */
 175                 char line[1000];
 176                 while (packet_read_line(fd, line, sizeof(line))) {
 177                         if (!prefixcmp(line, "shallow "))
 178                                 continue;
 179                         if (!prefixcmp(line, "unshallow "))
 180                                 continue;
 181                         die("git fetch-pack: expected shallow list");
 182                 }
 183         }
 184 }
 185
 186 struct write_shallow_data {
 187         struct strbuf *out;
 188         int use_pack_protocol;
 189         int count;
 190 };
 191
 192 static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
 193 {
 194         struct write_shallow_data *data = cb_data;
 195         const char *hex = sha1_to_hex(graft->sha1);
 196         data->count++;
 197         if (data->use_pack_protocol)
 198                 packet_buf_write(data->out, "shallow %s", hex);
 199         else {
 200                 strbuf_addstr(data->out, hex);
 201                 strbuf_addch(data->out, '\n');
 202         }
 203         return 0;
 204 }
 205
 206 static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
 207 {
 208         struct write_shallow_data data;
 209         data.out = out;
 210         data.use_pack_protocol = use_pack_protocol;
 211         data.count = 0;
 212         for_each_commit_graft(write_one_shallow, &data);
 213         return data.count;
 214 }
 215
 216 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 217 {
 218         static char line[1000];
 219         int len = packet_read_line(fd, line, sizeof(line));
 220
 221         if (!len)
 222                 die("git fetch-pack: expected ACK/NAK, got EOF");
 223         if (line[len-1] == '\n')
 224                 line[--len] = 0;
 225         if (!strcmp(line, "NAK"))
 226                 return NAK;
 227         if (!prefixcmp(line, "ACK ")) {
 228                 if (!get_sha1_hex(line+4, result_sha1)) {
 229                         if (strstr(line+45, "continue"))
 230                                 return ACK_continue;
 231                         if (strstr(line+45, "common"))
 232                                 return ACK_common;
 233                         if (strstr(line+45, "ready"))
 234                                 return ACK_ready;
 235                         return ACK;
 236                 }
 237         }
 238         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 239 }
 240
 241 static void send_request(struct fetch_pack_args *args,
 242                          int fd, struct strbuf *buf)
 243 {
 244         if (args->stateless_rpc) {
 245                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 246                 packet_flush(fd);
 247         } else
 248                 safe_write(fd, buf->buf, buf->len);
 249 }
 250
 251 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 252 {
 253         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 254 }
 255
 256 #define INITIAL_FLUSH 16
 257 #define PIPESAFE_FLUSH 32
 258 #define LARGE_FLUSH 1024
 259
 260 static int next_flush(struct fetch_pack_args *args, int count)
 261 {
 262         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 263
 264         if (count < flush_limit)
 265                 count <<= 1;
 266         else
 267                 count += flush_limit;
 268         return count;
 269 }
 270
 271 static int find_common(struct fetch_pack_args *args,
 272                        int fd[2], unsigned char *result_sha1,
 273                        struct ref *refs)
 274 {
 275         int fetching;
 276         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 277         const unsigned char *sha1;
 278         unsigned in_vain = 0;
 279         int got_continue = 0;
 280         int got_ready = 0;
 281         struct strbuf req_buf = STRBUF_INIT;
 282         size_t state_len = 0;
 283
 284         if (args->stateless_rpc && multi_ack == 1)
 285                 die("--stateless-rpc requires multi_ack_detailed");
 286         if (marked)
 287                 for_each_ref(clear_marks, NULL);
 288         marked = 1;
 289
 290         for_each_ref(rev_list_insert_ref, NULL);
 291         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 292
 293         fetching = 0;
 294         for ( ; refs ; refs = refs->next) {
 295                 unsigned char *remote = refs->old_sha1;
 296                 const char *remote_hex;
 297                 struct object *o;
 298
 299                 /*
 300                  * If that object is complete (i.e. it is an ancestor of a
 301                  * local ref), we tell them we have it but do not have to
 302                  * tell them about its ancestors, which they already know
 303                  * about.
 304                  *
 305                  * We use lookup_object here because we are only
 306                  * interested in the case we *know* the object is
 307                  * reachable and we have already scanned it.
 308                  */
 309                 if (((o = lookup_object(remote)) != NULL) &&
 310                                 (o->flags & COMPLETE)) {
 311                         continue;
 312                 }
 313
 314                 remote_hex = sha1_to_hex(remote);
 315                 if (!fetching) {
 316                         struct strbuf c = STRBUF_INIT;
 317                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 318                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 319                         if (no_done)            strbuf_addstr(&c, " no-done");
 320                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 321                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 322                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 323                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 324                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 325                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 326                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 327                                                             git_user_agent_sanitized());
 328                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 329                         strbuf_release(&c);
 330                 } else
 331                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 332                 fetching++;
 333         }
 334
 335         if (!fetching) {
 336                 strbuf_release(&req_buf);
 337                 packet_flush(fd[1]);
 338                 return 1;
 339         }
 340
 341         if (is_repository_shallow())
 342                 write_shallow_commits(&req_buf, 1);
 343         if (args->depth > 0)
 344                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 345         packet_buf_flush(&req_buf);
 346         state_len = req_buf.len;
 347
 348         if (args->depth > 0) {
 349                 char line[1024];
 350                 unsigned char sha1[20];
 351
 352                 send_request(args, fd[1], &req_buf);
 353                 while (packet_read_line(fd[0], line, sizeof(line))) {
 354                         if (!prefixcmp(line, "shallow ")) {
 355                                 if (get_sha1_hex(line + 8, sha1))
 356                                         die("invalid shallow line: %s", line);
 357                                 register_shallow(sha1);
 358                                 continue;
 359                         }
 360                         if (!prefixcmp(line, "unshallow ")) {
 361                                 if (get_sha1_hex(line + 10, sha1))
 362                                         die("invalid unshallow line: %s", line);
 363                                 if (!lookup_object(sha1))
 364                                         die("object not found: %s", line);
 365                                 /* make sure that it is parsed as shallow */
 366                                 if (!parse_object(sha1))
 367                                         die("error in object: %s", line);
 368                                 if (unregister_shallow(sha1))
 369                                         die("no shallow found: %s", line);
 370                                 continue;
 371                         }
 372                         die("expected shallow/unshallow, got %s", line);
 373                 }
 374         } else if (!args->stateless_rpc)
 375                 send_request(args, fd[1], &req_buf);
 376
 377         if (!args->stateless_rpc) {
 378                 /* If we aren't using the stateless-rpc interface
 379                  * we don't need to retain the headers.
 380                  */
 381                 strbuf_setlen(&req_buf, 0);
 382                 state_len = 0;
 383         }
 384
 385         flushes = 0;
 386         retval = -1;
 387         while ((sha1 = get_rev())) {
 388                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 389                 if (args->verbose)
 390                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 391                 in_vain++;
 392                 if (flush_at <= ++count) {
 393                         int ack;
 394
 395                         packet_buf_flush(&req_buf);
 396                         send_request(args, fd[1], &req_buf);
 397                         strbuf_setlen(&req_buf, state_len);
 398                         flushes++;
 399                         flush_at = next_flush(args, count);
 400
 401                         /*
 402                          * We keep one window "ahead" of the other side, and
 403                          * will wait for an ACK only on the next one
 404                          */
 405                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 406                                 continue;
 407
 408                         consume_shallow_list(args, fd[0]);
 409                         do {
 410                                 ack = get_ack(fd[0], result_sha1);
 411                                 if (args->verbose && ack)
 412                                         fprintf(stderr, "got ack %d %s\n", ack,
 413                                                         sha1_to_hex(result_sha1));
 414                                 switch (ack) {
 415                                 case ACK:
 416                                         flushes = 0;
 417                                         multi_ack = 0;
 418                                         retval = 0;
 419                                         goto done;
 420                                 case ACK_common:
 421                                 case ACK_ready:
 422                                 case ACK_continue: {
 423                                         struct commit *commit =
 424                                                 lookup_commit(result_sha1);
 425                                         if (!commit)
 426                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 427                                         if (args->stateless_rpc
 428                                          && ack == ACK_common
 429                                          && !(commit->object.flags & COMMON)) {
 430                                                 /* We need to replay the have for this object
 431                                                  * on the next RPC request so the peer knows
 432                                                  * it is in common with us.
 433                                                  */
 434                                                 const char *hex = sha1_to_hex(result_sha1);
 435                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 436                                                 state_len = req_buf.len;
 437                                         }
 438                                         mark_common(commit, 0, 1);
 439                                         retval = 0;
 440                                         in_vain = 0;
 441                                         got_continue = 1;
 442                                         if (ack == ACK_ready) {
 443                                                 rev_list = NULL;
 444                                                 got_ready = 1;
 445                                         }
 446                                         break;
 447                                         }
 448                                 }
 449                         } while (ack);
 450                         flushes--;
 451                         if (got_continue && MAX_IN_VAIN < in_vain) {
 452                                 if (args->verbose)
 453                                         fprintf(stderr, "giving up\n");
 454                                 break; /* give up */
 455                         }
 456                 }
 457         }
 458 done:
 459         if (!got_ready || !no_done) {
 460                 packet_buf_write(&req_buf, "done\n");
 461                 send_request(args, fd[1], &req_buf);
 462         }
 463         if (args->verbose)
 464                 fprintf(stderr, "done\n");
 465         if (retval != 0) {
 466                 multi_ack = 0;
 467                 flushes++;
 468         }
 469         strbuf_release(&req_buf);
 470
 471         consume_shallow_list(args, fd[0]);
 472         while (flushes || multi_ack) {
 473                 int ack = get_ack(fd[0], result_sha1);
 474                 if (ack) {
 475                         if (args->verbose)
 476                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 477                                         sha1_to_hex(result_sha1));
 478                         if (ack == ACK)
 479                                 return 0;
 480                         multi_ack = 1;
 481                         continue;
 482                 }
 483                 flushes--;
 484         }
 485         /* it is no error to fetch into a completely empty repo */
 486         return count ? retval : 0;
 487 }
 488
 489 static struct commit_list *complete;
 490
 491 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 492 {
 493         struct object *o = parse_object(sha1);
 494
 495         while (o && o->type == OBJ_TAG) {
 496                 struct tag *t = (struct tag *) o;
 497                 if (!t->tagged)
 498                         break; /* broken repository */
 499                 o->flags |= COMPLETE;
 500                 o = parse_object(t->tagged->sha1);
 501         }
 502         if (o && o->type == OBJ_COMMIT) {
 503                 struct commit *commit = (struct commit *)o;
 504                 if (!(commit->object.flags & COMPLETE)) {
 505                         commit->object.flags |= COMPLETE;
 506                         commit_list_insert_by_date(commit, &complete);
 507                 }
 508         }
 509         return 0;
 510 }
 511
 512 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 513                                          unsigned long cutoff)
 514 {
 515         while (complete && cutoff <= complete->item->date) {
 516                 if (args->verbose)
 517                         fprintf(stderr, "Marking %s as complete\n",
 518                                 sha1_to_hex(complete->item->object.sha1));
 519                 pop_most_recent_commit(&complete, COMPLETE);
 520         }
 521 }
 522
 523 static void filter_refs(struct fetch_pack_args *args,
 524                         struct ref **refs,
 525                         struct ref **sought, int nr_sought)
 526 {
 527         struct ref *newlist = NULL;
 528         struct ref **newtail = &newlist;
 529         struct ref *ref, *next;
 530         int i;
 531
 532         i = 0;
 533         for (ref = *refs; ref; ref = next) {
 534                 int keep = 0;
 535                 next = ref->next;
 536
 537                 if (!memcmp(ref->name, "refs/", 5) &&
 538                     check_refname_format(ref->name + 5, 0))
 539                         ; /* trash */
 540                 else {
 541                         while (i < nr_sought) {
 542                                 int cmp = strcmp(ref->name, sought[i]->name);
 543                                 if (cmp < 0)
 544                                         break; /* definitely do not have it */
 545                                 else if (cmp == 0) {
 546                                         keep = 1; /* definitely have it */
 547                                         sought[i]->matched = 1;
 548                                 }
 549                                 i++;
 550                         }
 551                 }
 552
 553                 if (!keep && args->fetch_all &&
 554                     (!args->depth || prefixcmp(ref->name, "refs/tags/")))
 555                         keep = 1;
 556
 557                 if (keep) {
 558                         *newtail = ref;
 559                         ref->next = NULL;
 560                         newtail = &ref->next;
 561                 } else {
 562                         free(ref);
 563                 }
 564         }
 565
 566         /* Append unmatched requests to the list */
 567         if (allow_tip_sha1_in_want) {
 568                 for (i = 0; i < nr_sought; i++) {
 569                         ref = sought[i];
 570                         if (ref->matched)
 571                                 continue;
 572                         if (get_sha1_hex(ref->name, ref->old_sha1))
 573                                 continue;
 574
 575                         ref->matched = 1;
 576                         *newtail = ref;
 577                         ref->next = NULL;
 578                         newtail = &ref->next;
 579                 }
 580         }
 581         *refs = newlist;
 582 }
 583
 584 static void mark_alternate_complete(const struct ref *ref, void *unused)
 585 {
 586         mark_complete(NULL, ref->old_sha1, 0, NULL);
 587 }
 588
 589 static int everything_local(struct fetch_pack_args *args,
 590                             struct ref **refs,
 591                             struct ref **sought, int nr_sought)
 592 {
 593         struct ref *ref;
 594         int retval;
 595         unsigned long cutoff = 0;
 596
 597         save_commit_buffer = 0;
 598
 599         for (ref = *refs; ref; ref = ref->next) {
 600                 struct object *o;
 601
 602                 if (!has_sha1_file(ref->old_sha1))
 603                         continue;
 604
 605                 o = parse_object(ref->old_sha1);
 606                 if (!o)
 607                         continue;
 608
 609                 /* We already have it -- which may mean that we were
 610                  * in sync with the other side at some time after
 611                  * that (it is OK if we guess wrong here).
 612                  */
 613                 if (o->type == OBJ_COMMIT) {
 614                         struct commit *commit = (struct commit *)o;
 615                         if (!cutoff || cutoff < commit->date)
 616                                 cutoff = commit->date;
 617                 }
 618         }
 619
 620         if (!args->depth) {
 621                 for_each_ref(mark_complete, NULL);
 622                 for_each_alternate_ref(mark_alternate_complete, NULL);
 623                 if (cutoff)
 624                         mark_recent_complete_commits(args, cutoff);
 625         }
 626
 627         /*
 628          * Mark all complete remote refs as common refs.
 629          * Don't mark them common yet; the server has to be told so first.
 630          */
 631         for (ref = *refs; ref; ref = ref->next) {
 632                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 633                                              NULL, 0);
 634
 635                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 636                         continue;
 637
 638                 if (!(o->flags & SEEN)) {
 639                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 640
 641                         mark_common((struct commit *)o, 1, 1);
 642                 }
 643         }
 644
 645         filter_refs(args, refs, sought, nr_sought);
 646
 647         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 648                 const unsigned char *remote = ref->old_sha1;
 649                 unsigned char local[20];
 650                 struct object *o;
 651
 652                 o = lookup_object(remote);
 653                 if (!o || !(o->flags & COMPLETE)) {
 654                         retval = 0;
 655                         if (!args->verbose)
 656                                 continue;
 657                         fprintf(stderr,
 658                                 "want %s (%s)\n", sha1_to_hex(remote),
 659                                 ref->name);
 660                         continue;
 661                 }
 662
 663                 hashcpy(ref->new_sha1, local);
 664                 if (!args->verbose)
 665                         continue;
 666                 fprintf(stderr,
 667                         "already have %s (%s)\n", sha1_to_hex(remote),
 668                         ref->name);
 669         }
 670         return retval;
 671 }
 672
 673 static int sideband_demux(int in, int out, void *data)
 674 {
 675         int *xd = data;
 676
 677         int ret = recv_sideband("fetch-pack", xd[0], out);
 678         close(out);
 679         return ret;
 680 }
 681
 682 static int get_pack(struct fetch_pack_args *args,
 683                     int xd[2], char **pack_lockfile)
 684 {
 685         struct async demux;
 686         const char *argv[20];
 687         char keep_arg[256];
 688         char hdr_arg[256];
 689         const char **av;
 690         int do_keep = args->keep_pack;
 691         struct child_process cmd;
 692
 693         memset(&demux, 0, sizeof(demux));
 694         if (use_sideband) {
 695                 /* xd[] is talking with upload-pack; subprocess reads from
 696                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 697                  * through demux->out.
 698                  */
 699                 demux.proc = sideband_demux;
 700                 demux.data = xd;
 701                 demux.out = -1;
 702                 if (start_async(&demux))
 703                         die("fetch-pack: unable to fork off sideband"
 704                             " demultiplexer");
 705         }
 706         else
 707                 demux.out = xd[0];
 708
 709         memset(&cmd, 0, sizeof(cmd));
 710         cmd.argv = argv;
 711         av = argv;
 712         *hdr_arg = 0;
 713         if (!args->keep_pack && unpack_limit) {
 714                 struct pack_header header;
 715
 716                 if (read_pack_header(demux.out, &header))
 717                         die("protocol error: bad pack header");
 718                 snprintf(hdr_arg, sizeof(hdr_arg),
 719                          "--pack_header=%"PRIu32",%"PRIu32,
 720                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 721                 if (ntohl(header.hdr_entries) < unpack_limit)
 722                         do_keep = 0;
 723                 else
 724                         do_keep = 1;
 725         }
 726
 727         if (do_keep) {
 728                 if (pack_lockfile)
 729                         cmd.out = -1;
 730                 *av++ = "index-pack";
 731                 *av++ = "--stdin";
 732                 if (!args->quiet && !args->no_progress)
 733                         *av++ = "-v";
 734                 if (args->use_thin_pack)
 735                         *av++ = "--fix-thin";
 736                 if (args->lock_pack || unpack_limit) {
 737                         int s = sprintf(keep_arg,
 738                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 739                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 740                                 strcpy(keep_arg + s, "localhost");
 741                         *av++ = keep_arg;
 742                 }
 743         }
 744         else {
 745                 *av++ = "unpack-objects";
 746                 if (args->quiet || args->no_progress)
 747                         *av++ = "-q";
 748         }
 749         if (*hdr_arg)
 750                 *av++ = hdr_arg;
 751         if (fetch_fsck_objects >= 0
 752             ? fetch_fsck_objects
 753             : transfer_fsck_objects >= 0
 754             ? transfer_fsck_objects
 755             : 0)
 756                 *av++ = "--strict";
 757         *av++ = NULL;
 758
 759         cmd.in = demux.out;
 760         cmd.git_cmd = 1;
 761         if (start_command(&cmd))
 762                 die("fetch-pack: unable to fork off %s", argv[0]);
 763         if (do_keep && pack_lockfile) {
 764                 *pack_lockfile = index_pack_lockfile(cmd.out);
 765                 close(cmd.out);
 766         }
 767
 768         if (finish_command(&cmd))
 769                 die("%s failed", argv[0]);
 770         if (use_sideband && finish_async(&demux))
 771                 die("error in sideband demultiplexer");
 772         return 0;
 773 }
 774
 775 static int cmp_ref_by_name(const void *a_, const void *b_)
 776 {
 777         const struct ref *a = *((const struct ref **)a_);
 778         const struct ref *b = *((const struct ref **)b_);
 779         return strcmp(a->name, b->name);
 780 }
 781
 782 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 783                                  int fd[2],
 784                                  const struct ref *orig_ref,
 785                                  struct ref **sought, int nr_sought,
 786                                  char **pack_lockfile)
 787 {
 788         struct ref *ref = copy_ref_list(orig_ref);
 789         unsigned char sha1[20];
 790         const char *agent_feature;
 791         int agent_len;
 792
 793         sort_ref_list(&ref, ref_compare_name);
 794         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 795
 796         if (is_repository_shallow() && !server_supports("shallow"))
 797                 die("Server does not support shallow clients");
 798         if (server_supports("multi_ack_detailed")) {
 799                 if (args->verbose)
 800                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 801                 multi_ack = 2;
 802                 if (server_supports("no-done")) {
 803                         if (args->verbose)
 804                                 fprintf(stderr, "Server supports no-done\n");
 805                         if (args->stateless_rpc)
 806                                 no_done = 1;
 807                 }
 808         }
 809         else if (server_supports("multi_ack")) {
 810                 if (args->verbose)
 811                         fprintf(stderr, "Server supports multi_ack\n");
 812                 multi_ack = 1;
 813         }
 814         if (server_supports("side-band-64k")) {
 815                 if (args->verbose)
 816                         fprintf(stderr, "Server supports side-band-64k\n");
 817                 use_sideband = 2;
 818         }
 819         else if (server_supports("side-band")) {
 820                 if (args->verbose)
 821                         fprintf(stderr, "Server supports side-band\n");
 822                 use_sideband = 1;
 823         }
 824         if (server_supports("allow-tip-sha1-in-want")) {
 825                 if (args->verbose)
 826                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 827                 allow_tip_sha1_in_want = 1;
 828         }
 829         if (!server_supports("thin-pack"))
 830                 args->use_thin_pack = 0;
 831         if (!server_supports("no-progress"))
 832                 args->no_progress = 0;
 833         if (!server_supports("include-tag"))
 834                 args->include_tag = 0;
 835         if (server_supports("ofs-delta")) {
 836                 if (args->verbose)
 837                         fprintf(stderr, "Server supports ofs-delta\n");
 838         } else
 839                 prefer_ofs_delta = 0;
 840
 841         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 842                 agent_supported = 1;
 843                 if (args->verbose && agent_len)
 844                         fprintf(stderr, "Server version is %.*s\n",
 845                                 agent_len, agent_feature);
 846         }
 847
 848         if (everything_local(args, &ref, sought, nr_sought)) {
 849                 packet_flush(fd[1]);
 850                 goto all_done;
 851         }
 852         if (find_common(args, fd, sha1, ref) < 0)
 853                 if (!args->keep_pack)
 854                         /* When cloning, it is not unusual to have
 855                          * no common commit.
 856                          */
 857                         warning("no common commits");
 858
 859         if (args->stateless_rpc)
 860                 packet_flush(fd[1]);
 861         if (get_pack(args, fd, pack_lockfile))
 862                 die("git fetch-pack: fetch failed.");
 863
 864  all_done:
 865         return ref;
 866 }
 867
 868 static int fetch_pack_config(const char *var, const char *value, void *cb)
 869 {
 870         if (strcmp(var, "fetch.unpacklimit") == 0) {
 871                 fetch_unpack_limit = git_config_int(var, value);
 872                 return 0;
 873         }
 874
 875         if (strcmp(var, "transfer.unpacklimit") == 0) {
 876                 transfer_unpack_limit = git_config_int(var, value);
 877                 return 0;
 878         }
 879
 880         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 881                 prefer_ofs_delta = git_config_bool(var, value);
 882                 return 0;
 883         }
 884
 885         if (!strcmp(var, "fetch.fsckobjects")) {
 886                 fetch_fsck_objects = git_config_bool(var, value);
 887                 return 0;
 888         }
 889
 890         if (!strcmp(var, "transfer.fsckobjects")) {
 891                 transfer_fsck_objects = git_config_bool(var, value);
 892                 return 0;
 893         }
 894
 895         return git_default_config(var, value, cb);
 896 }
 897
 898 static void fetch_pack_setup(void)
 899 {
 900         static int did_setup;
 901         if (did_setup)
 902                 return;
 903         git_config(fetch_pack_config, NULL);
 904         if (0 <= transfer_unpack_limit)
 905                 unpack_limit = transfer_unpack_limit;
 906         else if (0 <= fetch_unpack_limit)
 907                 unpack_limit = fetch_unpack_limit;
 908         did_setup = 1;
 909 }
 910
 911 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 912 {
 913         struct string_list names = STRING_LIST_INIT_NODUP;
 914         int src, dst;
 915
 916         for (src = dst = 0; src < nr; src++) {
 917                 struct string_list_item *item;
 918                 item = string_list_insert(&names, ref[src]->name);
 919                 if (item->util)
 920                         continue; /* already have it */
 921                 item->util = ref[src];
 922                 if (src != dst)
 923                         ref[dst] = ref[src];
 924                 dst++;
 925         }
 926         for (src = dst; src < nr; src++)
 927                 ref[src] = NULL;
 928         string_list_clear(&names, 0);
 929         return dst;
 930 }
 931
 932 struct ref *fetch_pack(struct fetch_pack_args *args,
 933                        int fd[], struct child_process *conn,
 934                        const struct ref *ref,
 935                        const char *dest,
 936                        struct ref **sought, int nr_sought,
 937                        char **pack_lockfile)
 938 {
 939         struct stat st;
 940         struct ref *ref_cpy;
 941
 942         fetch_pack_setup();
 943         if (args->depth > 0) {
 944                 if (stat(git_path("shallow"), &st))
 945                         st.st_mtime = 0;
 946         }
 947
 948         if (nr_sought)
 949                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
 950
 951         if (!ref) {
 952                 packet_flush(fd[1]);
 953                 die("no matching remote head");
 954         }
 955         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile);
 956
 957         if (args->depth > 0) {
 958                 static struct lock_file lock;
 959                 struct cache_time mtime;
 960                 struct strbuf sb = STRBUF_INIT;
 961                 char *shallow = git_path("shallow");
 962                 int fd;
 963
 964                 mtime.sec = st.st_mtime;
 965                 mtime.nsec = ST_MTIME_NSEC(st);
 966                 if (stat(shallow, &st)) {
 967                         if (mtime.sec)
 968                                 die("shallow file was removed during fetch");
 969                 } else if (st.st_mtime != mtime.sec
 970 #ifdef USE_NSEC
 971                                 || ST_MTIME_NSEC(st) != mtime.nsec
 972 #endif
 973                           )
 974                         die("shallow file was changed during fetch");
 975
 976                 fd = hold_lock_file_for_update(&lock, shallow,
 977                                                LOCK_DIE_ON_ERROR);
 978                 if (!write_shallow_commits(&sb, 0)
 979                  || write_in_full(fd, sb.buf, sb.len) != sb.len) {
 980                         unlink_or_warn(shallow);
 981                         rollback_lock_file(&lock);
 982                 } else {
 983                         commit_lock_file(&lock);
 984                 }
 985                 strbuf_release(&sb);
 986         }
 987
 988         reprepare_packed_git();
 989         return ref_cpy;
 990 }