fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "connect.h"
  13 #include "transport.h"
  14 #include "version.h"
  15
  16 static int transfer_unpack_limit = -1;
  17 static int fetch_unpack_limit = -1;
  18 static int unpack_limit = 100;
  19 static int prefer_ofs_delta = 1;
  20 static int no_done;
  21 static int fetch_fsck_objects = -1;
  22 static int transfer_fsck_objects = -1;
  23 static int agent_supported;
  24 static struct lock_file shallow_lock;
  25 static const char *alternate_shallow_file;
  26
  27 #define COMPLETE        (1U << 0)
  28 #define COMMON          (1U << 1)
  29 #define COMMON_REF      (1U << 2)
  30 #define SEEN            (1U << 3)
  31 #define POPPED          (1U << 4)
  32
  33 static int marked;
  34
  35 /*
  36  * After sending this many "have"s if we do not get any new ACK , we
  37  * give up traversing our history.
  38  */
  39 #define MAX_IN_VAIN 256
  40
  41 static struct commit_list *rev_list;
  42 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  43
  44 static void rev_list_push(struct commit *commit, int mark)
  45 {
  46         if (!(commit->object.flags & mark)) {
  47                 commit->object.flags |= mark;
  48
  49                 if (!(commit->object.parsed))
  50                         if (parse_commit(commit))
  51                                 return;
  52
  53                 commit_list_insert_by_date(commit, &rev_list);
  54
  55                 if (!(commit->object.flags & COMMON))
  56                         non_common_revs++;
  57         }
  58 }
  59
  60 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  61 {
  62         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  63
  64         if (o && o->type == OBJ_COMMIT)
  65                 rev_list_push((struct commit *)o, SEEN);
  66
  67         return 0;
  68 }
  69
  70 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  71 {
  72         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  73
  74         if (o && o->type == OBJ_COMMIT)
  75                 clear_commit_marks((struct commit *)o,
  76                                    COMMON | COMMON_REF | SEEN | POPPED);
  77         return 0;
  78 }
  79
  80 /*
  81    This function marks a rev and its ancestors as common.
  82    In some cases, it is desirable to mark only the ancestors (for example
  83    when only the server does not yet know that they are common).
  84 */
  85
  86 static void mark_common(struct commit *commit,
  87                 int ancestors_only, int dont_parse)
  88 {
  89         if (commit != NULL && !(commit->object.flags & COMMON)) {
  90                 struct object *o = (struct object *)commit;
  91
  92                 if (!ancestors_only)
  93                         o->flags |= COMMON;
  94
  95                 if (!(o->flags & SEEN))
  96                         rev_list_push(commit, SEEN);
  97                 else {
  98                         struct commit_list *parents;
  99
 100                         if (!ancestors_only && !(o->flags & POPPED))
 101                                 non_common_revs--;
 102                         if (!o->parsed && !dont_parse)
 103                                 if (parse_commit(commit))
 104                                         return;
 105
 106                         for (parents = commit->parents;
 107                                         parents;
 108                                         parents = parents->next)
 109                                 mark_common(parents->item, 0, dont_parse);
 110                 }
 111         }
 112 }
 113
 114 /*
 115   Get the next rev to send, ignoring the common.
 116 */
 117
 118 static const unsigned char *get_rev(void)
 119 {
 120         struct commit *commit = NULL;
 121
 122         while (commit == NULL) {
 123                 unsigned int mark;
 124                 struct commit_list *parents;
 125
 126                 if (rev_list == NULL || non_common_revs == 0)
 127                         return NULL;
 128
 129                 commit = rev_list->item;
 130                 if (!commit->object.parsed)
 131                         parse_commit(commit);
 132                 parents = commit->parents;
 133
 134                 commit->object.flags |= POPPED;
 135                 if (!(commit->object.flags & COMMON))
 136                         non_common_revs--;
 137
 138                 if (commit->object.flags & COMMON) {
 139                         /* do not send "have", and ignore ancestors */
 140                         commit = NULL;
 141                         mark = COMMON | SEEN;
 142                 } else if (commit->object.flags & COMMON_REF)
 143                         /* send "have", and ignore ancestors */
 144                         mark = COMMON | SEEN;
 145                 else
 146                         /* send "have", also for its ancestors */
 147                         mark = SEEN;
 148
 149                 while (parents) {
 150                         if (!(parents->item->object.flags & SEEN))
 151                                 rev_list_push(parents->item, mark);
 152                         if (mark & COMMON)
 153                                 mark_common(parents->item, 1, 0);
 154                         parents = parents->next;
 155                 }
 156
 157                 rev_list = rev_list->next;
 158         }
 159
 160         return commit->object.sha1;
 161 }
 162
 163 enum ack_type {
 164         NAK = 0,
 165         ACK,
 166         ACK_continue,
 167         ACK_common,
 168         ACK_ready
 169 };
 170
 171 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 172 {
 173         if (args->stateless_rpc && args->depth > 0) {
 174                 /* If we sent a depth we will get back "duplicate"
 175                  * shallow and unshallow commands every time there
 176                  * is a block of have lines exchanged.
 177                  */
 178                 char *line;
 179                 while ((line = packet_read_line(fd, NULL))) {
 180                         if (!prefixcmp(line, "shallow "))
 181                                 continue;
 182                         if (!prefixcmp(line, "unshallow "))
 183                                 continue;
 184                         die("git fetch-pack: expected shallow list");
 185                 }
 186         }
 187 }
 188
 189 struct write_shallow_data {
 190         struct strbuf *out;
 191         int use_pack_protocol;
 192         int count;
 193 };
 194
 195 static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
 196 {
 197         struct write_shallow_data *data = cb_data;
 198         const char *hex = sha1_to_hex(graft->sha1);
 199         data->count++;
 200         if (data->use_pack_protocol)
 201                 packet_buf_write(data->out, "shallow %s", hex);
 202         else {
 203                 strbuf_addstr(data->out, hex);
 204                 strbuf_addch(data->out, '\n');
 205         }
 206         return 0;
 207 }
 208
 209 static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
 210 {
 211         struct write_shallow_data data;
 212         data.out = out;
 213         data.use_pack_protocol = use_pack_protocol;
 214         data.count = 0;
 215         for_each_commit_graft(write_one_shallow, &data);
 216         return data.count;
 217 }
 218
 219 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 220 {
 221         int len;
 222         char *line = packet_read_line(fd, &len);
 223
 224         if (!len)
 225                 die("git fetch-pack: expected ACK/NAK, got EOF");
 226         if (!strcmp(line, "NAK"))
 227                 return NAK;
 228         if (!prefixcmp(line, "ACK ")) {
 229                 if (!get_sha1_hex(line+4, result_sha1)) {
 230                         if (len < 45)
 231                                 return ACK;
 232                         if (strstr(line+45, "continue"))
 233                                 return ACK_continue;
 234                         if (strstr(line+45, "common"))
 235                                 return ACK_common;
 236                         if (strstr(line+45, "ready"))
 237                                 return ACK_ready;
 238                         return ACK;
 239                 }
 240         }
 241         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 242 }
 243
 244 static void send_request(struct fetch_pack_args *args,
 245                          int fd, struct strbuf *buf)
 246 {
 247         if (args->stateless_rpc) {
 248                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 249                 packet_flush(fd);
 250         } else
 251                 write_or_die(fd, buf->buf, buf->len);
 252 }
 253
 254 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 255 {
 256         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 257 }
 258
 259 #define INITIAL_FLUSH 16
 260 #define PIPESAFE_FLUSH 32
 261 #define LARGE_FLUSH 1024
 262
 263 static int next_flush(struct fetch_pack_args *args, int count)
 264 {
 265         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 266
 267         if (count < flush_limit)
 268                 count <<= 1;
 269         else
 270                 count += flush_limit;
 271         return count;
 272 }
 273
 274 static int find_common(struct fetch_pack_args *args,
 275                        int fd[2], unsigned char *result_sha1,
 276                        struct ref *refs)
 277 {
 278         int fetching;
 279         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 280         const unsigned char *sha1;
 281         unsigned in_vain = 0;
 282         int got_continue = 0;
 283         int got_ready = 0;
 284         struct strbuf req_buf = STRBUF_INIT;
 285         size_t state_len = 0;
 286
 287         if (args->stateless_rpc && multi_ack == 1)
 288                 die("--stateless-rpc requires multi_ack_detailed");
 289         if (marked)
 290                 for_each_ref(clear_marks, NULL);
 291         marked = 1;
 292
 293         for_each_ref(rev_list_insert_ref, NULL);
 294         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 295
 296         fetching = 0;
 297         for ( ; refs ; refs = refs->next) {
 298                 unsigned char *remote = refs->old_sha1;
 299                 const char *remote_hex;
 300                 struct object *o;
 301
 302                 /*
 303                  * If that object is complete (i.e. it is an ancestor of a
 304                  * local ref), we tell them we have it but do not have to
 305                  * tell them about its ancestors, which they already know
 306                  * about.
 307                  *
 308                  * We use lookup_object here because we are only
 309                  * interested in the case we *know* the object is
 310                  * reachable and we have already scanned it.
 311                  */
 312                 if (((o = lookup_object(remote)) != NULL) &&
 313                                 (o->flags & COMPLETE)) {
 314                         continue;
 315                 }
 316
 317                 remote_hex = sha1_to_hex(remote);
 318                 if (!fetching) {
 319                         struct strbuf c = STRBUF_INIT;
 320                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 321                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 322                         if (no_done)            strbuf_addstr(&c, " no-done");
 323                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 324                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 325                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 326                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 327                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 328                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 329                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 330                                                             git_user_agent_sanitized());
 331                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 332                         strbuf_release(&c);
 333                 } else
 334                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 335                 fetching++;
 336         }
 337
 338         if (!fetching) {
 339                 strbuf_release(&req_buf);
 340                 packet_flush(fd[1]);
 341                 return 1;
 342         }
 343
 344         if (is_repository_shallow())
 345                 write_shallow_commits(&req_buf, 1);
 346         if (args->depth > 0)
 347                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 348         packet_buf_flush(&req_buf);
 349         state_len = req_buf.len;
 350
 351         if (args->depth > 0) {
 352                 char *line;
 353                 unsigned char sha1[20];
 354
 355                 send_request(args, fd[1], &req_buf);
 356                 while ((line = packet_read_line(fd[0], NULL))) {
 357                         if (!prefixcmp(line, "shallow ")) {
 358                                 if (get_sha1_hex(line + 8, sha1))
 359                                         die("invalid shallow line: %s", line);
 360                                 register_shallow(sha1);
 361                                 continue;
 362                         }
 363                         if (!prefixcmp(line, "unshallow ")) {
 364                                 if (get_sha1_hex(line + 10, sha1))
 365                                         die("invalid unshallow line: %s", line);
 366                                 if (!lookup_object(sha1))
 367                                         die("object not found: %s", line);
 368                                 /* make sure that it is parsed as shallow */
 369                                 if (!parse_object(sha1))
 370                                         die("error in object: %s", line);
 371                                 if (unregister_shallow(sha1))
 372                                         die("no shallow found: %s", line);
 373                                 continue;
 374                         }
 375                         die("expected shallow/unshallow, got %s", line);
 376                 }
 377         } else if (!args->stateless_rpc)
 378                 send_request(args, fd[1], &req_buf);
 379
 380         if (!args->stateless_rpc) {
 381                 /* If we aren't using the stateless-rpc interface
 382                  * we don't need to retain the headers.
 383                  */
 384                 strbuf_setlen(&req_buf, 0);
 385                 state_len = 0;
 386         }
 387
 388         flushes = 0;
 389         retval = -1;
 390         while ((sha1 = get_rev())) {
 391                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 392                 if (args->verbose)
 393                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 394                 in_vain++;
 395                 if (flush_at <= ++count) {
 396                         int ack;
 397
 398                         packet_buf_flush(&req_buf);
 399                         send_request(args, fd[1], &req_buf);
 400                         strbuf_setlen(&req_buf, state_len);
 401                         flushes++;
 402                         flush_at = next_flush(args, count);
 403
 404                         /*
 405                          * We keep one window "ahead" of the other side, and
 406                          * will wait for an ACK only on the next one
 407                          */
 408                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 409                                 continue;
 410
 411                         consume_shallow_list(args, fd[0]);
 412                         do {
 413                                 ack = get_ack(fd[0], result_sha1);
 414                                 if (args->verbose && ack)
 415                                         fprintf(stderr, "got ack %d %s\n", ack,
 416                                                         sha1_to_hex(result_sha1));
 417                                 switch (ack) {
 418                                 case ACK:
 419                                         flushes = 0;
 420                                         multi_ack = 0;
 421                                         retval = 0;
 422                                         goto done;
 423                                 case ACK_common:
 424                                 case ACK_ready:
 425                                 case ACK_continue: {
 426                                         struct commit *commit =
 427                                                 lookup_commit(result_sha1);
 428                                         if (!commit)
 429                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 430                                         if (args->stateless_rpc
 431                                          && ack == ACK_common
 432                                          && !(commit->object.flags & COMMON)) {
 433                                                 /* We need to replay the have for this object
 434                                                  * on the next RPC request so the peer knows
 435                                                  * it is in common with us.
 436                                                  */
 437                                                 const char *hex = sha1_to_hex(result_sha1);
 438                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 439                                                 state_len = req_buf.len;
 440                                         }
 441                                         mark_common(commit, 0, 1);
 442                                         retval = 0;
 443                                         in_vain = 0;
 444                                         got_continue = 1;
 445                                         if (ack == ACK_ready) {
 446                                                 rev_list = NULL;
 447                                                 got_ready = 1;
 448                                         }
 449                                         break;
 450                                         }
 451                                 }
 452                         } while (ack);
 453                         flushes--;
 454                         if (got_continue && MAX_IN_VAIN < in_vain) {
 455                                 if (args->verbose)
 456                                         fprintf(stderr, "giving up\n");
 457                                 break; /* give up */
 458                         }
 459                 }
 460         }
 461 done:
 462         if (!got_ready || !no_done) {
 463                 packet_buf_write(&req_buf, "done\n");
 464                 send_request(args, fd[1], &req_buf);
 465         }
 466         if (args->verbose)
 467                 fprintf(stderr, "done\n");
 468         if (retval != 0) {
 469                 multi_ack = 0;
 470                 flushes++;
 471         }
 472         strbuf_release(&req_buf);
 473
 474         consume_shallow_list(args, fd[0]);
 475         while (flushes || multi_ack) {
 476                 int ack = get_ack(fd[0], result_sha1);
 477                 if (ack) {
 478                         if (args->verbose)
 479                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 480                                         sha1_to_hex(result_sha1));
 481                         if (ack == ACK)
 482                                 return 0;
 483                         multi_ack = 1;
 484                         continue;
 485                 }
 486                 flushes--;
 487         }
 488         /* it is no error to fetch into a completely empty repo */
 489         return count ? retval : 0;
 490 }
 491
 492 static struct commit_list *complete;
 493
 494 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 495 {
 496         struct object *o = parse_object(sha1);
 497
 498         while (o && o->type == OBJ_TAG) {
 499                 struct tag *t = (struct tag *) o;
 500                 if (!t->tagged)
 501                         break; /* broken repository */
 502                 o->flags |= COMPLETE;
 503                 o = parse_object(t->tagged->sha1);
 504         }
 505         if (o && o->type == OBJ_COMMIT) {
 506                 struct commit *commit = (struct commit *)o;
 507                 if (!(commit->object.flags & COMPLETE)) {
 508                         commit->object.flags |= COMPLETE;
 509                         commit_list_insert_by_date(commit, &complete);
 510                 }
 511         }
 512         return 0;
 513 }
 514
 515 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 516                                          unsigned long cutoff)
 517 {
 518         while (complete && cutoff <= complete->item->date) {
 519                 if (args->verbose)
 520                         fprintf(stderr, "Marking %s as complete\n",
 521                                 sha1_to_hex(complete->item->object.sha1));
 522                 pop_most_recent_commit(&complete, COMPLETE);
 523         }
 524 }
 525
 526 static void filter_refs(struct fetch_pack_args *args,
 527                         struct ref **refs,
 528                         struct ref **sought, int nr_sought)
 529 {
 530         struct ref *newlist = NULL;
 531         struct ref **newtail = &newlist;
 532         struct ref *ref, *next;
 533         int i;
 534
 535         i = 0;
 536         for (ref = *refs; ref; ref = next) {
 537                 int keep = 0;
 538                 next = ref->next;
 539
 540                 if (!memcmp(ref->name, "refs/", 5) &&
 541                     check_refname_format(ref->name + 5, 0))
 542                         ; /* trash */
 543                 else {
 544                         while (i < nr_sought) {
 545                                 int cmp = strcmp(ref->name, sought[i]->name);
 546                                 if (cmp < 0)
 547                                         break; /* definitely do not have it */
 548                                 else if (cmp == 0) {
 549                                         keep = 1; /* definitely have it */
 550                                         sought[i]->matched = 1;
 551                                 }
 552                                 i++;
 553                         }
 554                 }
 555
 556                 if (!keep && args->fetch_all &&
 557                     (!args->depth || prefixcmp(ref->name, "refs/tags/")))
 558                         keep = 1;
 559
 560                 if (keep) {
 561                         *newtail = ref;
 562                         ref->next = NULL;
 563                         newtail = &ref->next;
 564                 } else {
 565                         free(ref);
 566                 }
 567         }
 568
 569         /* Append unmatched requests to the list */
 570         if (allow_tip_sha1_in_want) {
 571                 for (i = 0; i < nr_sought; i++) {
 572                         ref = sought[i];
 573                         if (ref->matched)
 574                                 continue;
 575                         if (get_sha1_hex(ref->name, ref->old_sha1))
 576                                 continue;
 577
 578                         ref->matched = 1;
 579                         *newtail = ref;
 580                         ref->next = NULL;
 581                         newtail = &ref->next;
 582                 }
 583         }
 584         *refs = newlist;
 585 }
 586
 587 static void mark_alternate_complete(const struct ref *ref, void *unused)
 588 {
 589         mark_complete(NULL, ref->old_sha1, 0, NULL);
 590 }
 591
 592 static int everything_local(struct fetch_pack_args *args,
 593                             struct ref **refs,
 594                             struct ref **sought, int nr_sought)
 595 {
 596         struct ref *ref;
 597         int retval;
 598         unsigned long cutoff = 0;
 599
 600         save_commit_buffer = 0;
 601
 602         for (ref = *refs; ref; ref = ref->next) {
 603                 struct object *o;
 604
 605                 if (!has_sha1_file(ref->old_sha1))
 606                         continue;
 607
 608                 o = parse_object(ref->old_sha1);
 609                 if (!o)
 610                         continue;
 611
 612                 /* We already have it -- which may mean that we were
 613                  * in sync with the other side at some time after
 614                  * that (it is OK if we guess wrong here).
 615                  */
 616                 if (o->type == OBJ_COMMIT) {
 617                         struct commit *commit = (struct commit *)o;
 618                         if (!cutoff || cutoff < commit->date)
 619                                 cutoff = commit->date;
 620                 }
 621         }
 622
 623         if (!args->depth) {
 624                 for_each_ref(mark_complete, NULL);
 625                 for_each_alternate_ref(mark_alternate_complete, NULL);
 626                 if (cutoff)
 627                         mark_recent_complete_commits(args, cutoff);
 628         }
 629
 630         /*
 631          * Mark all complete remote refs as common refs.
 632          * Don't mark them common yet; the server has to be told so first.
 633          */
 634         for (ref = *refs; ref; ref = ref->next) {
 635                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 636                                              NULL, 0);
 637
 638                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 639                         continue;
 640
 641                 if (!(o->flags & SEEN)) {
 642                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 643
 644                         mark_common((struct commit *)o, 1, 1);
 645                 }
 646         }
 647
 648         filter_refs(args, refs, sought, nr_sought);
 649
 650         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 651                 const unsigned char *remote = ref->old_sha1;
 652                 unsigned char local[20];
 653                 struct object *o;
 654
 655                 o = lookup_object(remote);
 656                 if (!o || !(o->flags & COMPLETE)) {
 657                         retval = 0;
 658                         if (!args->verbose)
 659                                 continue;
 660                         fprintf(stderr,
 661                                 "want %s (%s)\n", sha1_to_hex(remote),
 662                                 ref->name);
 663                         continue;
 664                 }
 665
 666                 hashcpy(ref->new_sha1, local);
 667                 if (!args->verbose)
 668                         continue;
 669                 fprintf(stderr,
 670                         "already have %s (%s)\n", sha1_to_hex(remote),
 671                         ref->name);
 672         }
 673         return retval;
 674 }
 675
 676 static int sideband_demux(int in, int out, void *data)
 677 {
 678         int *xd = data;
 679
 680         int ret = recv_sideband("fetch-pack", xd[0], out);
 681         close(out);
 682         return ret;
 683 }
 684
 685 static int get_pack(struct fetch_pack_args *args,
 686                     int xd[2], char **pack_lockfile)
 687 {
 688         struct async demux;
 689         const char *argv[22];
 690         char keep_arg[256];
 691         char hdr_arg[256];
 692         const char **av;
 693         int do_keep = args->keep_pack;
 694         struct child_process cmd;
 695         int ret;
 696
 697         memset(&demux, 0, sizeof(demux));
 698         if (use_sideband) {
 699                 /* xd[] is talking with upload-pack; subprocess reads from
 700                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 701                  * through demux->out.
 702                  */
 703                 demux.proc = sideband_demux;
 704                 demux.data = xd;
 705                 demux.out = -1;
 706                 if (start_async(&demux))
 707                         die("fetch-pack: unable to fork off sideband"
 708                             " demultiplexer");
 709         }
 710         else
 711                 demux.out = xd[0];
 712
 713         memset(&cmd, 0, sizeof(cmd));
 714         cmd.argv = argv;
 715         av = argv;
 716         *hdr_arg = 0;
 717         if (!args->keep_pack && unpack_limit) {
 718                 struct pack_header header;
 719
 720                 if (read_pack_header(demux.out, &header))
 721                         die("protocol error: bad pack header");
 722                 snprintf(hdr_arg, sizeof(hdr_arg),
 723                          "--pack_header=%"PRIu32",%"PRIu32,
 724                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 725                 if (ntohl(header.hdr_entries) < unpack_limit)
 726                         do_keep = 0;
 727                 else
 728                         do_keep = 1;
 729         }
 730
 731         if (alternate_shallow_file) {
 732                 *av++ = "--shallow-file";
 733                 *av++ = alternate_shallow_file;
 734         }
 735
 736         if (do_keep) {
 737                 if (pack_lockfile)
 738                         cmd.out = -1;
 739                 *av++ = "index-pack";
 740                 *av++ = "--stdin";
 741                 if (!args->quiet && !args->no_progress)
 742                         *av++ = "-v";
 743                 if (args->use_thin_pack)
 744                         *av++ = "--fix-thin";
 745                 if (args->lock_pack || unpack_limit) {
 746                         int s = sprintf(keep_arg,
 747                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 748                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 749                                 strcpy(keep_arg + s, "localhost");
 750                         *av++ = keep_arg;
 751                 }
 752                 if (args->check_self_contained_and_connected)
 753                         *av++ = "--check-self-contained-and-connected";
 754         }
 755         else {
 756                 *av++ = "unpack-objects";
 757                 if (args->quiet || args->no_progress)
 758                         *av++ = "-q";
 759                 args->check_self_contained_and_connected = 0;
 760         }
 761         if (*hdr_arg)
 762                 *av++ = hdr_arg;
 763         if (fetch_fsck_objects >= 0
 764             ? fetch_fsck_objects
 765             : transfer_fsck_objects >= 0
 766             ? transfer_fsck_objects
 767             : 0)
 768                 *av++ = "--strict";
 769         *av++ = NULL;
 770
 771         cmd.in = demux.out;
 772         cmd.git_cmd = 1;
 773         if (start_command(&cmd))
 774                 die("fetch-pack: unable to fork off %s", argv[0]);
 775         if (do_keep && pack_lockfile) {
 776                 *pack_lockfile = index_pack_lockfile(cmd.out);
 777                 close(cmd.out);
 778         }
 779
 780         ret = finish_command(&cmd);
 781         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 782                 args->self_contained_and_connected =
 783                         args->check_self_contained_and_connected &&
 784                         ret == 0;
 785         else
 786                 die("%s failed", argv[0]);
 787         if (use_sideband && finish_async(&demux))
 788                 die("error in sideband demultiplexer");
 789         return 0;
 790 }
 791
 792 static int cmp_ref_by_name(const void *a_, const void *b_)
 793 {
 794         const struct ref *a = *((const struct ref **)a_);
 795         const struct ref *b = *((const struct ref **)b_);
 796         return strcmp(a->name, b->name);
 797 }
 798
 799 static void setup_alternate_shallow(void)
 800 {
 801         struct strbuf sb = STRBUF_INIT;
 802         int fd;
 803
 804         check_shallow_file_for_update();
 805         fd = hold_lock_file_for_update(&shallow_lock, git_path("shallow"),
 806                                        LOCK_DIE_ON_ERROR);
 807         if (write_shallow_commits(&sb, 0)) {
 808                 if (write_in_full(fd, sb.buf, sb.len) != sb.len)
 809                         die_errno("failed to write to %s", shallow_lock.filename);
 810                 alternate_shallow_file = shallow_lock.filename;
 811         } else
 812                 /*
 813                  * is_repository_shallow() sees empty string as "no
 814                  * shallow file".
 815                  */
 816                 alternate_shallow_file = "";
 817         strbuf_release(&sb);
 818 }
 819
 820 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 821                                  int fd[2],
 822                                  const struct ref *orig_ref,
 823                                  struct ref **sought, int nr_sought,
 824                                  char **pack_lockfile)
 825 {
 826         struct ref *ref = copy_ref_list(orig_ref);
 827         unsigned char sha1[20];
 828         const char *agent_feature;
 829         int agent_len;
 830
 831         sort_ref_list(&ref, ref_compare_name);
 832         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 833
 834         if (is_repository_shallow() && !server_supports("shallow"))
 835                 die("Server does not support shallow clients");
 836         if (server_supports("multi_ack_detailed")) {
 837                 if (args->verbose)
 838                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 839                 multi_ack = 2;
 840                 if (server_supports("no-done")) {
 841                         if (args->verbose)
 842                                 fprintf(stderr, "Server supports no-done\n");
 843                         if (args->stateless_rpc)
 844                                 no_done = 1;
 845                 }
 846         }
 847         else if (server_supports("multi_ack")) {
 848                 if (args->verbose)
 849                         fprintf(stderr, "Server supports multi_ack\n");
 850                 multi_ack = 1;
 851         }
 852         if (server_supports("side-band-64k")) {
 853                 if (args->verbose)
 854                         fprintf(stderr, "Server supports side-band-64k\n");
 855                 use_sideband = 2;
 856         }
 857         else if (server_supports("side-band")) {
 858                 if (args->verbose)
 859                         fprintf(stderr, "Server supports side-band\n");
 860                 use_sideband = 1;
 861         }
 862         if (server_supports("allow-tip-sha1-in-want")) {
 863                 if (args->verbose)
 864                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 865                 allow_tip_sha1_in_want = 1;
 866         }
 867         if (!server_supports("thin-pack"))
 868                 args->use_thin_pack = 0;
 869         if (!server_supports("no-progress"))
 870                 args->no_progress = 0;
 871         if (!server_supports("include-tag"))
 872                 args->include_tag = 0;
 873         if (server_supports("ofs-delta")) {
 874                 if (args->verbose)
 875                         fprintf(stderr, "Server supports ofs-delta\n");
 876         } else
 877                 prefer_ofs_delta = 0;
 878
 879         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 880                 agent_supported = 1;
 881                 if (args->verbose && agent_len)
 882                         fprintf(stderr, "Server version is %.*s\n",
 883                                 agent_len, agent_feature);
 884         }
 885
 886         if (everything_local(args, &ref, sought, nr_sought)) {
 887                 packet_flush(fd[1]);
 888                 goto all_done;
 889         }
 890         if (find_common(args, fd, sha1, ref) < 0)
 891                 if (!args->keep_pack)
 892                         /* When cloning, it is not unusual to have
 893                          * no common commit.
 894                          */
 895                         warning("no common commits");
 896
 897         if (args->stateless_rpc)
 898                 packet_flush(fd[1]);
 899         if (args->depth > 0)
 900                 setup_alternate_shallow();
 901         if (get_pack(args, fd, pack_lockfile))
 902                 die("git fetch-pack: fetch failed.");
 903
 904  all_done:
 905         return ref;
 906 }
 907
 908 static int fetch_pack_config(const char *var, const char *value, void *cb)
 909 {
 910         if (strcmp(var, "fetch.unpacklimit") == 0) {
 911                 fetch_unpack_limit = git_config_int(var, value);
 912                 return 0;
 913         }
 914
 915         if (strcmp(var, "transfer.unpacklimit") == 0) {
 916                 transfer_unpack_limit = git_config_int(var, value);
 917                 return 0;
 918         }
 919
 920         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 921                 prefer_ofs_delta = git_config_bool(var, value);
 922                 return 0;
 923         }
 924
 925         if (!strcmp(var, "fetch.fsckobjects")) {
 926                 fetch_fsck_objects = git_config_bool(var, value);
 927                 return 0;
 928         }
 929
 930         if (!strcmp(var, "transfer.fsckobjects")) {
 931                 transfer_fsck_objects = git_config_bool(var, value);
 932                 return 0;
 933         }
 934
 935         return git_default_config(var, value, cb);
 936 }
 937
 938 static void fetch_pack_setup(void)
 939 {
 940         static int did_setup;
 941         if (did_setup)
 942                 return;
 943         git_config(fetch_pack_config, NULL);
 944         if (0 <= transfer_unpack_limit)
 945                 unpack_limit = transfer_unpack_limit;
 946         else if (0 <= fetch_unpack_limit)
 947                 unpack_limit = fetch_unpack_limit;
 948         did_setup = 1;
 949 }
 950
 951 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 952 {
 953         struct string_list names = STRING_LIST_INIT_NODUP;
 954         int src, dst;
 955
 956         for (src = dst = 0; src < nr; src++) {
 957                 struct string_list_item *item;
 958                 item = string_list_insert(&names, ref[src]->name);
 959                 if (item->util)
 960                         continue; /* already have it */
 961                 item->util = ref[src];
 962                 if (src != dst)
 963                         ref[dst] = ref[src];
 964                 dst++;
 965         }
 966         for (src = dst; src < nr; src++)
 967                 ref[src] = NULL;
 968         string_list_clear(&names, 0);
 969         return dst;
 970 }
 971
 972 struct ref *fetch_pack(struct fetch_pack_args *args,
 973                        int fd[], struct child_process *conn,
 974                        const struct ref *ref,
 975                        const char *dest,
 976                        struct ref **sought, int nr_sought,
 977                        char **pack_lockfile)
 978 {
 979         struct ref *ref_cpy;
 980
 981         fetch_pack_setup();
 982         if (nr_sought)
 983                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
 984
 985         if (!ref) {
 986                 packet_flush(fd[1]);
 987                 die("no matching remote head");
 988         }
 989         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile);
 990
 991         if (alternate_shallow_file) {
 992                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 993                         unlink_or_warn(git_path("shallow"));
 994                         rollback_lock_file(&shallow_lock);
 995                 } else
 996                         commit_lock_file(&shallow_lock);
 997         }
 998
 999         reprepare_packed_git();
1000         return ref_cpy;
1001 }