fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int fetch_fsck_objects = -1;
  25 static int transfer_fsck_objects = -1;
  26 static int agent_supported;
  27 static struct lock_file shallow_lock;
  28 static const char *alternate_shallow_file;
  29
  30 /* Remember to update object flag allocation in object.h */
  31 #define COMPLETE        (1U << 0)
  32 #define COMMON          (1U << 1)
  33 #define COMMON_REF      (1U << 2)
  34 #define SEEN            (1U << 3)
  35 #define POPPED          (1U << 4)
  36
  37 static int marked;
  38
  39 /*
  40  * After sending this many "have"s if we do not get any new ACK , we
  41  * give up traversing our history.
  42  */
  43 #define MAX_IN_VAIN 256
  44
  45 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  46 static int non_common_revs, multi_ack, use_sideband;
  47 /* Allow specifying sha1 if it is a ref tip. */
  48 #define ALLOW_TIP_SHA1  01
  49 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  50 #define ALLOW_REACHABLE_SHA1    02
  51 static unsigned int allow_unadvertised_object_request;
  52
  53 static void rev_list_push(struct commit *commit, int mark)
  54 {
  55         if (!(commit->object.flags & mark)) {
  56                 commit->object.flags |= mark;
  57
  58                 if (parse_commit(commit))
  59                         return;
  60
  61                 prio_queue_put(&rev_list, commit);
  62
  63                 if (!(commit->object.flags & COMMON))
  64                         non_common_revs++;
  65         }
  66 }
  67
  68 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
  69 {
  70         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  71
  72         if (o && o->type == OBJ_COMMIT)
  73                 rev_list_push((struct commit *)o, SEEN);
  74
  75         return 0;
  76 }
  77
  78 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
  79                                    int flag, void *cb_data)
  80 {
  81         return rev_list_insert_ref(refname, oid->hash);
  82 }
  83
  84 static int clear_marks(const char *refname, const struct object_id *oid,
  85                        int flag, void *cb_data)
  86 {
  87         struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
  88
  89         if (o && o->type == OBJ_COMMIT)
  90                 clear_commit_marks((struct commit *)o,
  91                                    COMMON | COMMON_REF | SEEN | POPPED);
  92         return 0;
  93 }
  94
  95 /*
  96    This function marks a rev and its ancestors as common.
  97    In some cases, it is desirable to mark only the ancestors (for example
  98    when only the server does not yet know that they are common).
  99 */
 100
 101 static void mark_common(struct commit *commit,
 102                 int ancestors_only, int dont_parse)
 103 {
 104         if (commit != NULL && !(commit->object.flags & COMMON)) {
 105                 struct object *o = (struct object *)commit;
 106
 107                 if (!ancestors_only)
 108                         o->flags |= COMMON;
 109
 110                 if (!(o->flags & SEEN))
 111                         rev_list_push(commit, SEEN);
 112                 else {
 113                         struct commit_list *parents;
 114
 115                         if (!ancestors_only && !(o->flags & POPPED))
 116                                 non_common_revs--;
 117                         if (!o->parsed && !dont_parse)
 118                                 if (parse_commit(commit))
 119                                         return;
 120
 121                         for (parents = commit->parents;
 122                                         parents;
 123                                         parents = parents->next)
 124                                 mark_common(parents->item, 0, dont_parse);
 125                 }
 126         }
 127 }
 128
 129 /*
 130   Get the next rev to send, ignoring the common.
 131 */
 132
 133 static const unsigned char *get_rev(void)
 134 {
 135         struct commit *commit = NULL;
 136
 137         while (commit == NULL) {
 138                 unsigned int mark;
 139                 struct commit_list *parents;
 140
 141                 if (rev_list.nr == 0 || non_common_revs == 0)
 142                         return NULL;
 143
 144                 commit = prio_queue_get(&rev_list);
 145                 parse_commit(commit);
 146                 parents = commit->parents;
 147
 148                 commit->object.flags |= POPPED;
 149                 if (!(commit->object.flags & COMMON))
 150                         non_common_revs--;
 151
 152                 if (commit->object.flags & COMMON) {
 153                         /* do not send "have", and ignore ancestors */
 154                         commit = NULL;
 155                         mark = COMMON | SEEN;
 156                 } else if (commit->object.flags & COMMON_REF)
 157                         /* send "have", and ignore ancestors */
 158                         mark = COMMON | SEEN;
 159                 else
 160                         /* send "have", also for its ancestors */
 161                         mark = SEEN;
 162
 163                 while (parents) {
 164                         if (!(parents->item->object.flags & SEEN))
 165                                 rev_list_push(parents->item, mark);
 166                         if (mark & COMMON)
 167                                 mark_common(parents->item, 1, 0);
 168                         parents = parents->next;
 169                 }
 170         }
 171
 172         return commit->object.oid.hash;
 173 }
 174
 175 enum ack_type {
 176         NAK = 0,
 177         ACK,
 178         ACK_continue,
 179         ACK_common,
 180         ACK_ready
 181 };
 182
 183 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 184 {
 185         if (args->stateless_rpc && args->depth > 0) {
 186                 /* If we sent a depth we will get back "duplicate"
 187                  * shallow and unshallow commands every time there
 188                  * is a block of have lines exchanged.
 189                  */
 190                 char *line;
 191                 while ((line = packet_read_line(fd, NULL))) {
 192                         if (starts_with(line, "shallow "))
 193                                 continue;
 194                         if (starts_with(line, "unshallow "))
 195                                 continue;
 196                         die("git fetch-pack: expected shallow list");
 197                 }
 198         }
 199 }
 200
 201 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 202 {
 203         int len;
 204         char *line = packet_read_line(fd, &len);
 205         const char *arg;
 206
 207         if (!len)
 208                 die("git fetch-pack: expected ACK/NAK, got EOF");
 209         if (!strcmp(line, "NAK"))
 210                 return NAK;
 211         if (skip_prefix(line, "ACK ", &arg)) {
 212                 if (!get_sha1_hex(arg, result_sha1)) {
 213                         arg += 40;
 214                         len -= arg - line;
 215                         if (len < 1)
 216                                 return ACK;
 217                         if (strstr(arg, "continue"))
 218                                 return ACK_continue;
 219                         if (strstr(arg, "common"))
 220                                 return ACK_common;
 221                         if (strstr(arg, "ready"))
 222                                 return ACK_ready;
 223                         return ACK;
 224                 }
 225         }
 226         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 227 }
 228
 229 static void send_request(struct fetch_pack_args *args,
 230                          int fd, struct strbuf *buf)
 231 {
 232         if (args->stateless_rpc) {
 233                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 234                 packet_flush(fd);
 235         } else
 236                 write_or_die(fd, buf->buf, buf->len);
 237 }
 238
 239 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 240 {
 241         rev_list_insert_ref(NULL, ref->old_oid.hash);
 242 }
 243
 244 #define INITIAL_FLUSH 16
 245 #define PIPESAFE_FLUSH 32
 246 #define LARGE_FLUSH 16384
 247
 248 static int next_flush(struct fetch_pack_args *args, int count)
 249 {
 250         if (args->stateless_rpc) {
 251                 if (count < LARGE_FLUSH)
 252                         count <<= 1;
 253                 else
 254                         count = count * 11 / 10;
 255         } else {
 256                 if (count < PIPESAFE_FLUSH)
 257                         count <<= 1;
 258                 else
 259                         count += PIPESAFE_FLUSH;
 260         }
 261         return count;
 262 }
 263
 264 static int find_common(struct fetch_pack_args *args,
 265                        int fd[2], unsigned char *result_sha1,
 266                        struct ref *refs)
 267 {
 268         int fetching;
 269         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 270         const unsigned char *sha1;
 271         unsigned in_vain = 0;
 272         int got_continue = 0;
 273         int got_ready = 0;
 274         struct strbuf req_buf = STRBUF_INIT;
 275         size_t state_len = 0;
 276
 277         if (args->stateless_rpc && multi_ack == 1)
 278                 die("--stateless-rpc requires multi_ack_detailed");
 279         if (marked)
 280                 for_each_ref(clear_marks, NULL);
 281         marked = 1;
 282
 283         for_each_ref(rev_list_insert_ref_oid, NULL);
 284         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 285
 286         fetching = 0;
 287         for ( ; refs ; refs = refs->next) {
 288                 unsigned char *remote = refs->old_oid.hash;
 289                 const char *remote_hex;
 290                 struct object *o;
 291
 292                 /*
 293                  * If that object is complete (i.e. it is an ancestor of a
 294                  * local ref), we tell them we have it but do not have to
 295                  * tell them about its ancestors, which they already know
 296                  * about.
 297                  *
 298                  * We use lookup_object here because we are only
 299                  * interested in the case we *know* the object is
 300                  * reachable and we have already scanned it.
 301                  */
 302                 if (((o = lookup_object(remote)) != NULL) &&
 303                                 (o->flags & COMPLETE)) {
 304                         continue;
 305                 }
 306
 307                 remote_hex = sha1_to_hex(remote);
 308                 if (!fetching) {
 309                         struct strbuf c = STRBUF_INIT;
 310                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 311                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 312                         if (no_done)            strbuf_addstr(&c, " no-done");
 313                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 314                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 315                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 316                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 317                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 318                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 319                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 320                                                             git_user_agent_sanitized());
 321                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 322                         strbuf_release(&c);
 323                 } else
 324                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 325                 fetching++;
 326         }
 327
 328         if (!fetching) {
 329                 strbuf_release(&req_buf);
 330                 packet_flush(fd[1]);
 331                 return 1;
 332         }
 333
 334         if (is_repository_shallow())
 335                 write_shallow_commits(&req_buf, 1, NULL);
 336         if (args->depth > 0)
 337                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 338         packet_buf_flush(&req_buf);
 339         state_len = req_buf.len;
 340
 341         if (args->depth > 0) {
 342                 char *line;
 343                 const char *arg;
 344                 unsigned char sha1[20];
 345
 346                 send_request(args, fd[1], &req_buf);
 347                 while ((line = packet_read_line(fd[0], NULL))) {
 348                         if (skip_prefix(line, "shallow ", &arg)) {
 349                                 if (get_sha1_hex(arg, sha1))
 350                                         die("invalid shallow line: %s", line);
 351                                 register_shallow(sha1);
 352                                 continue;
 353                         }
 354                         if (skip_prefix(line, "unshallow ", &arg)) {
 355                                 if (get_sha1_hex(arg, sha1))
 356                                         die("invalid unshallow line: %s", line);
 357                                 if (!lookup_object(sha1))
 358                                         die("object not found: %s", line);
 359                                 /* make sure that it is parsed as shallow */
 360                                 if (!parse_object(sha1))
 361                                         die("error in object: %s", line);
 362                                 if (unregister_shallow(sha1))
 363                                         die("no shallow found: %s", line);
 364                                 continue;
 365                         }
 366                         die("expected shallow/unshallow, got %s", line);
 367                 }
 368         } else if (!args->stateless_rpc)
 369                 send_request(args, fd[1], &req_buf);
 370
 371         if (!args->stateless_rpc) {
 372                 /* If we aren't using the stateless-rpc interface
 373                  * we don't need to retain the headers.
 374                  */
 375                 strbuf_setlen(&req_buf, 0);
 376                 state_len = 0;
 377         }
 378
 379         flushes = 0;
 380         retval = -1;
 381         while ((sha1 = get_rev())) {
 382                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 383                 if (args->verbose)
 384                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 385                 in_vain++;
 386                 if (flush_at <= ++count) {
 387                         int ack;
 388
 389                         packet_buf_flush(&req_buf);
 390                         send_request(args, fd[1], &req_buf);
 391                         strbuf_setlen(&req_buf, state_len);
 392                         flushes++;
 393                         flush_at = next_flush(args, count);
 394
 395                         /*
 396                          * We keep one window "ahead" of the other side, and
 397                          * will wait for an ACK only on the next one
 398                          */
 399                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 400                                 continue;
 401
 402                         consume_shallow_list(args, fd[0]);
 403                         do {
 404                                 ack = get_ack(fd[0], result_sha1);
 405                                 if (args->verbose && ack)
 406                                         fprintf(stderr, "got ack %d %s\n", ack,
 407                                                         sha1_to_hex(result_sha1));
 408                                 switch (ack) {
 409                                 case ACK:
 410                                         flushes = 0;
 411                                         multi_ack = 0;
 412                                         retval = 0;
 413                                         goto done;
 414                                 case ACK_common:
 415                                 case ACK_ready:
 416                                 case ACK_continue: {
 417                                         struct commit *commit =
 418                                                 lookup_commit(result_sha1);
 419                                         if (!commit)
 420                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 421                                         if (args->stateless_rpc
 422                                          && ack == ACK_common
 423                                          && !(commit->object.flags & COMMON)) {
 424                                                 /* We need to replay the have for this object
 425                                                  * on the next RPC request so the peer knows
 426                                                  * it is in common with us.
 427                                                  */
 428                                                 const char *hex = sha1_to_hex(result_sha1);
 429                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 430                                                 state_len = req_buf.len;
 431                                         }
 432                                         mark_common(commit, 0, 1);
 433                                         retval = 0;
 434                                         in_vain = 0;
 435                                         got_continue = 1;
 436                                         if (ack == ACK_ready) {
 437                                                 clear_prio_queue(&rev_list);
 438                                                 got_ready = 1;
 439                                         }
 440                                         break;
 441                                         }
 442                                 }
 443                         } while (ack);
 444                         flushes--;
 445                         if (got_continue && MAX_IN_VAIN < in_vain) {
 446                                 if (args->verbose)
 447                                         fprintf(stderr, "giving up\n");
 448                                 break; /* give up */
 449                         }
 450                 }
 451         }
 452 done:
 453         if (!got_ready || !no_done) {
 454                 packet_buf_write(&req_buf, "done\n");
 455                 send_request(args, fd[1], &req_buf);
 456         }
 457         if (args->verbose)
 458                 fprintf(stderr, "done\n");
 459         if (retval != 0) {
 460                 multi_ack = 0;
 461                 flushes++;
 462         }
 463         strbuf_release(&req_buf);
 464
 465         if (!got_ready || !no_done)
 466                 consume_shallow_list(args, fd[0]);
 467         while (flushes || multi_ack) {
 468                 int ack = get_ack(fd[0], result_sha1);
 469                 if (ack) {
 470                         if (args->verbose)
 471                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 472                                         sha1_to_hex(result_sha1));
 473                         if (ack == ACK)
 474                                 return 0;
 475                         multi_ack = 1;
 476                         continue;
 477                 }
 478                 flushes--;
 479         }
 480         /* it is no error to fetch into a completely empty repo */
 481         return count ? retval : 0;
 482 }
 483
 484 static struct commit_list *complete;
 485
 486 static int mark_complete(const unsigned char *sha1)
 487 {
 488         struct object *o = parse_object(sha1);
 489
 490         while (o && o->type == OBJ_TAG) {
 491                 struct tag *t = (struct tag *) o;
 492                 if (!t->tagged)
 493                         break; /* broken repository */
 494                 o->flags |= COMPLETE;
 495                 o = parse_object(t->tagged->oid.hash);
 496         }
 497         if (o && o->type == OBJ_COMMIT) {
 498                 struct commit *commit = (struct commit *)o;
 499                 if (!(commit->object.flags & COMPLETE)) {
 500                         commit->object.flags |= COMPLETE;
 501                         commit_list_insert(commit, &complete);
 502                 }
 503         }
 504         return 0;
 505 }
 506
 507 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 508                              int flag, void *cb_data)
 509 {
 510         return mark_complete(oid->hash);
 511 }
 512
 513 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 514                                          unsigned long cutoff)
 515 {
 516         while (complete && cutoff <= complete->item->date) {
 517                 if (args->verbose)
 518                         fprintf(stderr, "Marking %s as complete\n",
 519                                 oid_to_hex(&complete->item->object.oid));
 520                 pop_most_recent_commit(&complete, COMPLETE);
 521         }
 522 }
 523
 524 static void filter_refs(struct fetch_pack_args *args,
 525                         struct ref **refs,
 526                         struct ref **sought, int nr_sought)
 527 {
 528         struct ref *newlist = NULL;
 529         struct ref **newtail = &newlist;
 530         struct ref *ref, *next;
 531         int i;
 532
 533         i = 0;
 534         for (ref = *refs; ref; ref = next) {
 535                 int keep = 0;
 536                 next = ref->next;
 537
 538                 if (starts_with(ref->name, "refs/") &&
 539                     check_refname_format(ref->name, 0))
 540                         ; /* trash */
 541                 else {
 542                         while (i < nr_sought) {
 543                                 int cmp = strcmp(ref->name, sought[i]->name);
 544                                 if (cmp < 0)
 545                                         break; /* definitely do not have it */
 546                                 else if (cmp == 0) {
 547                                         keep = 1; /* definitely have it */
 548                                         sought[i]->matched = 1;
 549                                 }
 550                                 i++;
 551                         }
 552                 }
 553
 554                 if (!keep && args->fetch_all &&
 555                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 556                         keep = 1;
 557
 558                 if (keep) {
 559                         *newtail = ref;
 560                         ref->next = NULL;
 561                         newtail = &ref->next;
 562                 } else {
 563                         free(ref);
 564                 }
 565         }
 566
 567         /* Append unmatched requests to the list */
 568         if ((allow_unadvertised_object_request &
 569             (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
 570                 for (i = 0; i < nr_sought; i++) {
 571                         unsigned char sha1[20];
 572
 573                         ref = sought[i];
 574                         if (ref->matched)
 575                                 continue;
 576                         if (get_sha1_hex(ref->name, sha1) ||
 577                             ref->name[40] != '\0' ||
 578                             hashcmp(sha1, ref->old_oid.hash))
 579                                 continue;
 580
 581                         ref->matched = 1;
 582                         *newtail = copy_ref(ref);
 583                         newtail = &(*newtail)->next;
 584                 }
 585         }
 586         *refs = newlist;
 587 }
 588
 589 static void mark_alternate_complete(const struct ref *ref, void *unused)
 590 {
 591         mark_complete(ref->old_oid.hash);
 592 }
 593
 594 static int everything_local(struct fetch_pack_args *args,
 595                             struct ref **refs,
 596                             struct ref **sought, int nr_sought)
 597 {
 598         struct ref *ref;
 599         int retval;
 600         unsigned long cutoff = 0;
 601
 602         save_commit_buffer = 0;
 603
 604         for (ref = *refs; ref; ref = ref->next) {
 605                 struct object *o;
 606
 607                 if (!has_object_file(&ref->old_oid))
 608                         continue;
 609
 610                 o = parse_object(ref->old_oid.hash);
 611                 if (!o)
 612                         continue;
 613
 614                 /* We already have it -- which may mean that we were
 615                  * in sync with the other side at some time after
 616                  * that (it is OK if we guess wrong here).
 617                  */
 618                 if (o->type == OBJ_COMMIT) {
 619                         struct commit *commit = (struct commit *)o;
 620                         if (!cutoff || cutoff < commit->date)
 621                                 cutoff = commit->date;
 622                 }
 623         }
 624
 625         if (!args->depth) {
 626                 for_each_ref(mark_complete_oid, NULL);
 627                 for_each_alternate_ref(mark_alternate_complete, NULL);
 628                 commit_list_sort_by_date(&complete);
 629                 if (cutoff)
 630                         mark_recent_complete_commits(args, cutoff);
 631         }
 632
 633         /*
 634          * Mark all complete remote refs as common refs.
 635          * Don't mark them common yet; the server has to be told so first.
 636          */
 637         for (ref = *refs; ref; ref = ref->next) {
 638                 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 639                                              NULL, 0);
 640
 641                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 642                         continue;
 643
 644                 if (!(o->flags & SEEN)) {
 645                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 646
 647                         mark_common((struct commit *)o, 1, 1);
 648                 }
 649         }
 650
 651         filter_refs(args, refs, sought, nr_sought);
 652
 653         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 654                 const unsigned char *remote = ref->old_oid.hash;
 655                 struct object *o;
 656
 657                 o = lookup_object(remote);
 658                 if (!o || !(o->flags & COMPLETE)) {
 659                         retval = 0;
 660                         if (!args->verbose)
 661                                 continue;
 662                         fprintf(stderr,
 663                                 "want %s (%s)\n", sha1_to_hex(remote),
 664                                 ref->name);
 665                         continue;
 666                 }
 667                 if (!args->verbose)
 668                         continue;
 669                 fprintf(stderr,
 670                         "already have %s (%s)\n", sha1_to_hex(remote),
 671                         ref->name);
 672         }
 673         return retval;
 674 }
 675
 676 static int sideband_demux(int in, int out, void *data)
 677 {
 678         int *xd = data;
 679         int ret;
 680
 681         ret = recv_sideband("fetch-pack", xd[0], out);
 682         close(out);
 683         return ret;
 684 }
 685
 686 static int get_pack(struct fetch_pack_args *args,
 687                     int xd[2], char **pack_lockfile)
 688 {
 689         struct async demux;
 690         int do_keep = args->keep_pack;
 691         const char *cmd_name;
 692         struct pack_header header;
 693         int pass_header = 0;
 694         struct child_process cmd = CHILD_PROCESS_INIT;
 695         int ret;
 696
 697         memset(&demux, 0, sizeof(demux));
 698         if (use_sideband) {
 699                 /* xd[] is talking with upload-pack; subprocess reads from
 700                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 701                  * through demux->out.
 702                  */
 703                 demux.proc = sideband_demux;
 704                 demux.data = xd;
 705                 demux.out = -1;
 706                 demux.isolate_sigpipe = 1;
 707                 if (start_async(&demux))
 708                         die("fetch-pack: unable to fork off sideband"
 709                             " demultiplexer");
 710         }
 711         else
 712                 demux.out = xd[0];
 713
 714         if (!args->keep_pack && unpack_limit) {
 715
 716                 if (read_pack_header(demux.out, &header))
 717                         die("protocol error: bad pack header");
 718                 pass_header = 1;
 719                 if (ntohl(header.hdr_entries) < unpack_limit)
 720                         do_keep = 0;
 721                 else
 722                         do_keep = 1;
 723         }
 724
 725         if (alternate_shallow_file) {
 726                 argv_array_push(&cmd.args, "--shallow-file");
 727                 argv_array_push(&cmd.args, alternate_shallow_file);
 728         }
 729
 730         if (do_keep) {
 731                 if (pack_lockfile)
 732                         cmd.out = -1;
 733                 cmd_name = "index-pack";
 734                 argv_array_push(&cmd.args, cmd_name);
 735                 argv_array_push(&cmd.args, "--stdin");
 736                 if (!args->quiet && !args->no_progress)
 737                         argv_array_push(&cmd.args, "-v");
 738                 if (args->use_thin_pack)
 739                         argv_array_push(&cmd.args, "--fix-thin");
 740                 if (args->lock_pack || unpack_limit) {
 741                         char hostname[256];
 742                         if (gethostname(hostname, sizeof(hostname)))
 743                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 744                         argv_array_pushf(&cmd.args,
 745                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 746                                         (uintmax_t)getpid(), hostname);
 747                 }
 748                 if (args->check_self_contained_and_connected)
 749                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 750         }
 751         else {
 752                 cmd_name = "unpack-objects";
 753                 argv_array_push(&cmd.args, cmd_name);
 754                 if (args->quiet || args->no_progress)
 755                         argv_array_push(&cmd.args, "-q");
 756                 args->check_self_contained_and_connected = 0;
 757         }
 758
 759         if (pass_header)
 760                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 761                                  ntohl(header.hdr_version),
 762                                  ntohl(header.hdr_entries));
 763         if (fetch_fsck_objects >= 0
 764             ? fetch_fsck_objects
 765             : transfer_fsck_objects >= 0
 766             ? transfer_fsck_objects
 767             : 0)
 768                 argv_array_push(&cmd.args, "--strict");
 769
 770         cmd.in = demux.out;
 771         cmd.git_cmd = 1;
 772         if (start_command(&cmd))
 773                 die("fetch-pack: unable to fork off %s", cmd_name);
 774         if (do_keep && pack_lockfile) {
 775                 *pack_lockfile = index_pack_lockfile(cmd.out);
 776                 close(cmd.out);
 777         }
 778
 779         if (!use_sideband)
 780                 /* Closed by start_command() */
 781                 xd[0] = -1;
 782
 783         ret = finish_command(&cmd);
 784         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 785                 args->self_contained_and_connected =
 786                         args->check_self_contained_and_connected &&
 787                         ret == 0;
 788         else
 789                 die("%s failed", cmd_name);
 790         if (use_sideband && finish_async(&demux))
 791                 die("error in sideband demultiplexer");
 792         return 0;
 793 }
 794
 795 static int cmp_ref_by_name(const void *a_, const void *b_)
 796 {
 797         const struct ref *a = *((const struct ref **)a_);
 798         const struct ref *b = *((const struct ref **)b_);
 799         return strcmp(a->name, b->name);
 800 }
 801
 802 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 803                                  int fd[2],
 804                                  const struct ref *orig_ref,
 805                                  struct ref **sought, int nr_sought,
 806                                  struct shallow_info *si,
 807                                  char **pack_lockfile)
 808 {
 809         struct ref *ref = copy_ref_list(orig_ref);
 810         unsigned char sha1[20];
 811         const char *agent_feature;
 812         int agent_len;
 813
 814         sort_ref_list(&ref, ref_compare_name);
 815         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 816
 817         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 818                 die("Server does not support shallow clients");
 819         if (server_supports("multi_ack_detailed")) {
 820                 if (args->verbose)
 821                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 822                 multi_ack = 2;
 823                 if (server_supports("no-done")) {
 824                         if (args->verbose)
 825                                 fprintf(stderr, "Server supports no-done\n");
 826                         if (args->stateless_rpc)
 827                                 no_done = 1;
 828                 }
 829         }
 830         else if (server_supports("multi_ack")) {
 831                 if (args->verbose)
 832                         fprintf(stderr, "Server supports multi_ack\n");
 833                 multi_ack = 1;
 834         }
 835         if (server_supports("side-band-64k")) {
 836                 if (args->verbose)
 837                         fprintf(stderr, "Server supports side-band-64k\n");
 838                 use_sideband = 2;
 839         }
 840         else if (server_supports("side-band")) {
 841                 if (args->verbose)
 842                         fprintf(stderr, "Server supports side-band\n");
 843                 use_sideband = 1;
 844         }
 845         if (server_supports("allow-tip-sha1-in-want")) {
 846                 if (args->verbose)
 847                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 848                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 849         }
 850         if (server_supports("allow-reachable-sha1-in-want")) {
 851                 if (args->verbose)
 852                         fprintf(stderr, "Server supports allow-reachable-sha1-in-want\n");
 853                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 854         }
 855         if (!server_supports("thin-pack"))
 856                 args->use_thin_pack = 0;
 857         if (!server_supports("no-progress"))
 858                 args->no_progress = 0;
 859         if (!server_supports("include-tag"))
 860                 args->include_tag = 0;
 861         if (server_supports("ofs-delta")) {
 862                 if (args->verbose)
 863                         fprintf(stderr, "Server supports ofs-delta\n");
 864         } else
 865                 prefer_ofs_delta = 0;
 866
 867         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 868                 agent_supported = 1;
 869                 if (args->verbose && agent_len)
 870                         fprintf(stderr, "Server version is %.*s\n",
 871                                 agent_len, agent_feature);
 872         }
 873
 874         if (everything_local(args, &ref, sought, nr_sought)) {
 875                 packet_flush(fd[1]);
 876                 goto all_done;
 877         }
 878         if (find_common(args, fd, sha1, ref) < 0)
 879                 if (!args->keep_pack)
 880                         /* When cloning, it is not unusual to have
 881                          * no common commit.
 882                          */
 883                         warning("no common commits");
 884
 885         if (args->stateless_rpc)
 886                 packet_flush(fd[1]);
 887         if (args->depth > 0)
 888                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 889                                         NULL);
 890         else if (si->nr_ours || si->nr_theirs)
 891                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 892         else
 893                 alternate_shallow_file = NULL;
 894         if (get_pack(args, fd, pack_lockfile))
 895                 die("git fetch-pack: fetch failed.");
 896
 897  all_done:
 898         return ref;
 899 }
 900
 901 static void fetch_pack_config(void)
 902 {
 903         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 904         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 905         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 906         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 907         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 908
 909         git_config(git_default_config, NULL);
 910 }
 911
 912 static void fetch_pack_setup(void)
 913 {
 914         static int did_setup;
 915         if (did_setup)
 916                 return;
 917         fetch_pack_config();
 918         if (0 <= transfer_unpack_limit)
 919                 unpack_limit = transfer_unpack_limit;
 920         else if (0 <= fetch_unpack_limit)
 921                 unpack_limit = fetch_unpack_limit;
 922         did_setup = 1;
 923 }
 924
 925 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 926 {
 927         struct string_list names = STRING_LIST_INIT_NODUP;
 928         int src, dst;
 929
 930         for (src = dst = 0; src < nr; src++) {
 931                 struct string_list_item *item;
 932                 item = string_list_insert(&names, ref[src]->name);
 933                 if (item->util)
 934                         continue; /* already have it */
 935                 item->util = ref[src];
 936                 if (src != dst)
 937                         ref[dst] = ref[src];
 938                 dst++;
 939         }
 940         for (src = dst; src < nr; src++)
 941                 ref[src] = NULL;
 942         string_list_clear(&names, 0);
 943         return dst;
 944 }
 945
 946 static void update_shallow(struct fetch_pack_args *args,
 947                            struct ref **sought, int nr_sought,
 948                            struct shallow_info *si)
 949 {
 950         struct sha1_array ref = SHA1_ARRAY_INIT;
 951         int *status;
 952         int i;
 953
 954         if (args->depth > 0 && alternate_shallow_file) {
 955                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 956                         unlink_or_warn(git_path_shallow());
 957                         rollback_lock_file(&shallow_lock);
 958                 } else
 959                         commit_lock_file(&shallow_lock);
 960                 return;
 961         }
 962
 963         if (!si->shallow || !si->shallow->nr)
 964                 return;
 965
 966         if (args->cloning) {
 967                 /*
 968                  * remote is shallow, but this is a clone, there are
 969                  * no objects in repo to worry about. Accept any
 970                  * shallow points that exist in the pack (iow in repo
 971                  * after get_pack() and reprepare_packed_git())
 972                  */
 973                 struct sha1_array extra = SHA1_ARRAY_INIT;
 974                 unsigned char (*sha1)[20] = si->shallow->sha1;
 975                 for (i = 0; i < si->shallow->nr; i++)
 976                         if (has_sha1_file(sha1[i]))
 977                                 sha1_array_append(&extra, sha1[i]);
 978                 if (extra.nr) {
 979                         setup_alternate_shallow(&shallow_lock,
 980                                                 &alternate_shallow_file,
 981                                                 &extra);
 982                         commit_lock_file(&shallow_lock);
 983                 }
 984                 sha1_array_clear(&extra);
 985                 return;
 986         }
 987
 988         if (!si->nr_ours && !si->nr_theirs)
 989                 return;
 990
 991         remove_nonexistent_theirs_shallow(si);
 992         if (!si->nr_ours && !si->nr_theirs)
 993                 return;
 994         for (i = 0; i < nr_sought; i++)
 995                 sha1_array_append(&ref, sought[i]->old_oid.hash);
 996         si->ref = &ref;
 997
 998         if (args->update_shallow) {
 999                 /*
1000                  * remote is also shallow, .git/shallow may be updated
1001                  * so all refs can be accepted. Make sure we only add
1002                  * shallow roots that are actually reachable from new
1003                  * refs.
1004                  */
1005                 struct sha1_array extra = SHA1_ARRAY_INIT;
1006                 unsigned char (*sha1)[20] = si->shallow->sha1;
1007                 assign_shallow_commits_to_refs(si, NULL, NULL);
1008                 if (!si->nr_ours && !si->nr_theirs) {
1009                         sha1_array_clear(&ref);
1010                         return;
1011                 }
1012                 for (i = 0; i < si->nr_ours; i++)
1013                         sha1_array_append(&extra, sha1[si->ours[i]]);
1014                 for (i = 0; i < si->nr_theirs; i++)
1015                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1016                 setup_alternate_shallow(&shallow_lock,
1017                                         &alternate_shallow_file,
1018                                         &extra);
1019                 commit_lock_file(&shallow_lock);
1020                 sha1_array_clear(&extra);
1021                 sha1_array_clear(&ref);
1022                 return;
1023         }
1024
1025         /*
1026          * remote is also shallow, check what ref is safe to update
1027          * without updating .git/shallow
1028          */
1029         status = xcalloc(nr_sought, sizeof(*status));
1030         assign_shallow_commits_to_refs(si, NULL, status);
1031         if (si->nr_ours || si->nr_theirs) {
1032                 for (i = 0; i < nr_sought; i++)
1033                         if (status[i])
1034                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1035         }
1036         free(status);
1037         sha1_array_clear(&ref);
1038 }
1039
1040 struct ref *fetch_pack(struct fetch_pack_args *args,
1041                        int fd[], struct child_process *conn,
1042                        const struct ref *ref,
1043                        const char *dest,
1044                        struct ref **sought, int nr_sought,
1045                        struct sha1_array *shallow,
1046                        char **pack_lockfile)
1047 {
1048         struct ref *ref_cpy;
1049         struct shallow_info si;
1050
1051         fetch_pack_setup();
1052         if (nr_sought)
1053                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1054
1055         if (!ref) {
1056                 packet_flush(fd[1]);
1057                 die("no matching remote head");
1058         }
1059         prepare_shallow_info(&si, shallow);
1060         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1061                                 &si, pack_lockfile);
1062         reprepare_packed_git();
1063         update_shallow(args, sought, nr_sought, &si);
1064         clear_shallow_info(&si);
1065         return ref_cpy;
1066 }