fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int fetch_fsck_objects = -1;
  25 static int transfer_fsck_objects = -1;
  26 static int agent_supported;
  27 static struct lock_file shallow_lock;
  28 static const char *alternate_shallow_file;
  29
  30 /* Remember to update object flag allocation in object.h */
  31 #define COMPLETE        (1U << 0)
  32 #define COMMON          (1U << 1)
  33 #define COMMON_REF      (1U << 2)
  34 #define SEEN            (1U << 3)
  35 #define POPPED          (1U << 4)
  36
  37 static int marked;
  38
  39 /*
  40  * After sending this many "have"s if we do not get any new ACK , we
  41  * give up traversing our history.
  42  */
  43 #define MAX_IN_VAIN 256
  44
  45 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  46 static int non_common_revs, multi_ack, use_sideband;
  47 /* Allow specifying sha1 if it is a ref tip. */
  48 #define ALLOW_TIP_SHA1  01
  49 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  50 #define ALLOW_REACHABLE_SHA1    02
  51 static unsigned int allow_unadvertised_object_request;
  52
  53 __attribute__((format (printf, 2, 3)))
  54 static inline void print_verbose(const struct fetch_pack_args *args,
  55                                  const char *fmt, ...)
  56 {
  57         va_list params;
  58
  59         if (!args->verbose)
  60                 return;
  61
  62         va_start(params, fmt);
  63         vfprintf(stderr, fmt, params);
  64         va_end(params);
  65         fputc('\n', stderr);
  66 }
  67
  68 static void rev_list_push(struct commit *commit, int mark)
  69 {
  70         if (!(commit->object.flags & mark)) {
  71                 commit->object.flags |= mark;
  72
  73                 if (parse_commit(commit))
  74                         return;
  75
  76                 prio_queue_put(&rev_list, commit);
  77
  78                 if (!(commit->object.flags & COMMON))
  79                         non_common_revs++;
  80         }
  81 }
  82
  83 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
  84 {
  85         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  86
  87         if (o && o->type == OBJ_COMMIT)
  88                 rev_list_push((struct commit *)o, SEEN);
  89
  90         return 0;
  91 }
  92
  93 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
  94                                    int flag, void *cb_data)
  95 {
  96         return rev_list_insert_ref(refname, oid->hash);
  97 }
  98
  99 static int clear_marks(const char *refname, const struct object_id *oid,
 100                        int flag, void *cb_data)
 101 {
 102         struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
 103
 104         if (o && o->type == OBJ_COMMIT)
 105                 clear_commit_marks((struct commit *)o,
 106                                    COMMON | COMMON_REF | SEEN | POPPED);
 107         return 0;
 108 }
 109
 110 /*
 111    This function marks a rev and its ancestors as common.
 112    In some cases, it is desirable to mark only the ancestors (for example
 113    when only the server does not yet know that they are common).
 114 */
 115
 116 static void mark_common(struct commit *commit,
 117                 int ancestors_only, int dont_parse)
 118 {
 119         if (commit != NULL && !(commit->object.flags & COMMON)) {
 120                 struct object *o = (struct object *)commit;
 121
 122                 if (!ancestors_only)
 123                         o->flags |= COMMON;
 124
 125                 if (!(o->flags & SEEN))
 126                         rev_list_push(commit, SEEN);
 127                 else {
 128                         struct commit_list *parents;
 129
 130                         if (!ancestors_only && !(o->flags & POPPED))
 131                                 non_common_revs--;
 132                         if (!o->parsed && !dont_parse)
 133                                 if (parse_commit(commit))
 134                                         return;
 135
 136                         for (parents = commit->parents;
 137                                         parents;
 138                                         parents = parents->next)
 139                                 mark_common(parents->item, 0, dont_parse);
 140                 }
 141         }
 142 }
 143
 144 /*
 145   Get the next rev to send, ignoring the common.
 146 */
 147
 148 static const unsigned char *get_rev(void)
 149 {
 150         struct commit *commit = NULL;
 151
 152         while (commit == NULL) {
 153                 unsigned int mark;
 154                 struct commit_list *parents;
 155
 156                 if (rev_list.nr == 0 || non_common_revs == 0)
 157                         return NULL;
 158
 159                 commit = prio_queue_get(&rev_list);
 160                 parse_commit(commit);
 161                 parents = commit->parents;
 162
 163                 commit->object.flags |= POPPED;
 164                 if (!(commit->object.flags & COMMON))
 165                         non_common_revs--;
 166
 167                 if (commit->object.flags & COMMON) {
 168                         /* do not send "have", and ignore ancestors */
 169                         commit = NULL;
 170                         mark = COMMON | SEEN;
 171                 } else if (commit->object.flags & COMMON_REF)
 172                         /* send "have", and ignore ancestors */
 173                         mark = COMMON | SEEN;
 174                 else
 175                         /* send "have", also for its ancestors */
 176                         mark = SEEN;
 177
 178                 while (parents) {
 179                         if (!(parents->item->object.flags & SEEN))
 180                                 rev_list_push(parents->item, mark);
 181                         if (mark & COMMON)
 182                                 mark_common(parents->item, 1, 0);
 183                         parents = parents->next;
 184                 }
 185         }
 186
 187         return commit->object.oid.hash;
 188 }
 189
 190 enum ack_type {
 191         NAK = 0,
 192         ACK,
 193         ACK_continue,
 194         ACK_common,
 195         ACK_ready
 196 };
 197
 198 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 199 {
 200         if (args->stateless_rpc && args->depth > 0) {
 201                 /* If we sent a depth we will get back "duplicate"
 202                  * shallow and unshallow commands every time there
 203                  * is a block of have lines exchanged.
 204                  */
 205                 char *line;
 206                 while ((line = packet_read_line(fd, NULL))) {
 207                         if (starts_with(line, "shallow "))
 208                                 continue;
 209                         if (starts_with(line, "unshallow "))
 210                                 continue;
 211                         die("git fetch-pack: expected shallow list");
 212                 }
 213         }
 214 }
 215
 216 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 217 {
 218         int len;
 219         char *line = packet_read_line(fd, &len);
 220         const char *arg;
 221
 222         if (!len)
 223                 die("git fetch-pack: expected ACK/NAK, got EOF");
 224         if (!strcmp(line, "NAK"))
 225                 return NAK;
 226         if (skip_prefix(line, "ACK ", &arg)) {
 227                 if (!get_sha1_hex(arg, result_sha1)) {
 228                         arg += 40;
 229                         len -= arg - line;
 230                         if (len < 1)
 231                                 return ACK;
 232                         if (strstr(arg, "continue"))
 233                                 return ACK_continue;
 234                         if (strstr(arg, "common"))
 235                                 return ACK_common;
 236                         if (strstr(arg, "ready"))
 237                                 return ACK_ready;
 238                         return ACK;
 239                 }
 240         }
 241         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 242 }
 243
 244 static void send_request(struct fetch_pack_args *args,
 245                          int fd, struct strbuf *buf)
 246 {
 247         if (args->stateless_rpc) {
 248                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 249                 packet_flush(fd);
 250         } else
 251                 write_or_die(fd, buf->buf, buf->len);
 252 }
 253
 254 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 255 {
 256         rev_list_insert_ref(NULL, ref->old_oid.hash);
 257 }
 258
 259 #define INITIAL_FLUSH 16
 260 #define PIPESAFE_FLUSH 32
 261 #define LARGE_FLUSH 1024
 262
 263 static int next_flush(struct fetch_pack_args *args, int count)
 264 {
 265         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 266
 267         if (count < flush_limit)
 268                 count <<= 1;
 269         else
 270                 count += flush_limit;
 271         return count;
 272 }
 273
 274 static int find_common(struct fetch_pack_args *args,
 275                        int fd[2], unsigned char *result_sha1,
 276                        struct ref *refs)
 277 {
 278         int fetching;
 279         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 280         const unsigned char *sha1;
 281         unsigned in_vain = 0;
 282         int got_continue = 0;
 283         int got_ready = 0;
 284         struct strbuf req_buf = STRBUF_INIT;
 285         size_t state_len = 0;
 286
 287         if (args->stateless_rpc && multi_ack == 1)
 288                 die("--stateless-rpc requires multi_ack_detailed");
 289         if (marked)
 290                 for_each_ref(clear_marks, NULL);
 291         marked = 1;
 292
 293         for_each_ref(rev_list_insert_ref_oid, NULL);
 294         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 295
 296         fetching = 0;
 297         for ( ; refs ; refs = refs->next) {
 298                 unsigned char *remote = refs->old_oid.hash;
 299                 const char *remote_hex;
 300                 struct object *o;
 301
 302                 /*
 303                  * If that object is complete (i.e. it is an ancestor of a
 304                  * local ref), we tell them we have it but do not have to
 305                  * tell them about its ancestors, which they already know
 306                  * about.
 307                  *
 308                  * We use lookup_object here because we are only
 309                  * interested in the case we *know* the object is
 310                  * reachable and we have already scanned it.
 311                  */
 312                 if (((o = lookup_object(remote)) != NULL) &&
 313                                 (o->flags & COMPLETE)) {
 314                         continue;
 315                 }
 316
 317                 remote_hex = sha1_to_hex(remote);
 318                 if (!fetching) {
 319                         struct strbuf c = STRBUF_INIT;
 320                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 321                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 322                         if (no_done)            strbuf_addstr(&c, " no-done");
 323                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 324                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 325                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 326                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 327                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 328                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 329                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 330                                                             git_user_agent_sanitized());
 331                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 332                         strbuf_release(&c);
 333                 } else
 334                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 335                 fetching++;
 336         }
 337
 338         if (!fetching) {
 339                 strbuf_release(&req_buf);
 340                 packet_flush(fd[1]);
 341                 return 1;
 342         }
 343
 344         if (is_repository_shallow())
 345                 write_shallow_commits(&req_buf, 1, NULL);
 346         if (args->depth > 0)
 347                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 348         packet_buf_flush(&req_buf);
 349         state_len = req_buf.len;
 350
 351         if (args->depth > 0) {
 352                 char *line;
 353                 const char *arg;
 354                 unsigned char sha1[20];
 355
 356                 send_request(args, fd[1], &req_buf);
 357                 while ((line = packet_read_line(fd[0], NULL))) {
 358                         if (skip_prefix(line, "shallow ", &arg)) {
 359                                 if (get_sha1_hex(arg, sha1))
 360                                         die("invalid shallow line: %s", line);
 361                                 register_shallow(sha1);
 362                                 continue;
 363                         }
 364                         if (skip_prefix(line, "unshallow ", &arg)) {
 365                                 if (get_sha1_hex(arg, sha1))
 366                                         die("invalid unshallow line: %s", line);
 367                                 if (!lookup_object(sha1))
 368                                         die("object not found: %s", line);
 369                                 /* make sure that it is parsed as shallow */
 370                                 if (!parse_object(sha1))
 371                                         die("error in object: %s", line);
 372                                 if (unregister_shallow(sha1))
 373                                         die("no shallow found: %s", line);
 374                                 continue;
 375                         }
 376                         die("expected shallow/unshallow, got %s", line);
 377                 }
 378         } else if (!args->stateless_rpc)
 379                 send_request(args, fd[1], &req_buf);
 380
 381         if (!args->stateless_rpc) {
 382                 /* If we aren't using the stateless-rpc interface
 383                  * we don't need to retain the headers.
 384                  */
 385                 strbuf_setlen(&req_buf, 0);
 386                 state_len = 0;
 387         }
 388
 389         flushes = 0;
 390         retval = -1;
 391         while ((sha1 = get_rev())) {
 392                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 393                 print_verbose(args, "have %s", sha1_to_hex(sha1));
 394                 in_vain++;
 395                 if (flush_at <= ++count) {
 396                         int ack;
 397
 398                         packet_buf_flush(&req_buf);
 399                         send_request(args, fd[1], &req_buf);
 400                         strbuf_setlen(&req_buf, state_len);
 401                         flushes++;
 402                         flush_at = next_flush(args, count);
 403
 404                         /*
 405                          * We keep one window "ahead" of the other side, and
 406                          * will wait for an ACK only on the next one
 407                          */
 408                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 409                                 continue;
 410
 411                         consume_shallow_list(args, fd[0]);
 412                         do {
 413                                 ack = get_ack(fd[0], result_sha1);
 414                                 if (ack)
 415                                         print_verbose(args, "got ack %d %s", ack,
 416                                                       sha1_to_hex(result_sha1));
 417                                 switch (ack) {
 418                                 case ACK:
 419                                         flushes = 0;
 420                                         multi_ack = 0;
 421                                         retval = 0;
 422                                         goto done;
 423                                 case ACK_common:
 424                                 case ACK_ready:
 425                                 case ACK_continue: {
 426                                         struct commit *commit =
 427                                                 lookup_commit(result_sha1);
 428                                         if (!commit)
 429                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 430                                         if (args->stateless_rpc
 431                                          && ack == ACK_common
 432                                          && !(commit->object.flags & COMMON)) {
 433                                                 /* We need to replay the have for this object
 434                                                  * on the next RPC request so the peer knows
 435                                                  * it is in common with us.
 436                                                  */
 437                                                 const char *hex = sha1_to_hex(result_sha1);
 438                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 439                                                 state_len = req_buf.len;
 440                                         }
 441                                         mark_common(commit, 0, 1);
 442                                         retval = 0;
 443                                         in_vain = 0;
 444                                         got_continue = 1;
 445                                         if (ack == ACK_ready) {
 446                                                 clear_prio_queue(&rev_list);
 447                                                 got_ready = 1;
 448                                         }
 449                                         break;
 450                                         }
 451                                 }
 452                         } while (ack);
 453                         flushes--;
 454                         if (got_continue && MAX_IN_VAIN < in_vain) {
 455                                 print_verbose(args, "giving up");
 456                                 break; /* give up */
 457                         }
 458                 }
 459         }
 460 done:
 461         if (!got_ready || !no_done) {
 462                 packet_buf_write(&req_buf, "done\n");
 463                 send_request(args, fd[1], &req_buf);
 464         }
 465         print_verbose(args, "done");
 466         if (retval != 0) {
 467                 multi_ack = 0;
 468                 flushes++;
 469         }
 470         strbuf_release(&req_buf);
 471
 472         if (!got_ready || !no_done)
 473                 consume_shallow_list(args, fd[0]);
 474         while (flushes || multi_ack) {
 475                 int ack = get_ack(fd[0], result_sha1);
 476                 if (ack) {
 477                         print_verbose(args, "got ack (%d) %s", ack,
 478                                       sha1_to_hex(result_sha1));
 479                         if (ack == ACK)
 480                                 return 0;
 481                         multi_ack = 1;
 482                         continue;
 483                 }
 484                 flushes--;
 485         }
 486         /* it is no error to fetch into a completely empty repo */
 487         return count ? retval : 0;
 488 }
 489
 490 static struct commit_list *complete;
 491
 492 static int mark_complete(const unsigned char *sha1)
 493 {
 494         struct object *o = parse_object(sha1);
 495
 496         while (o && o->type == OBJ_TAG) {
 497                 struct tag *t = (struct tag *) o;
 498                 if (!t->tagged)
 499                         break; /* broken repository */
 500                 o->flags |= COMPLETE;
 501                 o = parse_object(t->tagged->oid.hash);
 502         }
 503         if (o && o->type == OBJ_COMMIT) {
 504                 struct commit *commit = (struct commit *)o;
 505                 if (!(commit->object.flags & COMPLETE)) {
 506                         commit->object.flags |= COMPLETE;
 507                         commit_list_insert(commit, &complete);
 508                 }
 509         }
 510         return 0;
 511 }
 512
 513 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 514                              int flag, void *cb_data)
 515 {
 516         return mark_complete(oid->hash);
 517 }
 518
 519 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 520                                          unsigned long cutoff)
 521 {
 522         while (complete && cutoff <= complete->item->date) {
 523                 print_verbose(args, "Marking %s as complete",
 524                               oid_to_hex(&complete->item->object.oid));
 525                 pop_most_recent_commit(&complete, COMPLETE);
 526         }
 527 }
 528
 529 static void filter_refs(struct fetch_pack_args *args,
 530                         struct ref **refs,
 531                         struct ref **sought, int nr_sought)
 532 {
 533         struct ref *newlist = NULL;
 534         struct ref **newtail = &newlist;
 535         struct ref *ref, *next;
 536         int i;
 537
 538         i = 0;
 539         for (ref = *refs; ref; ref = next) {
 540                 int keep = 0;
 541                 next = ref->next;
 542
 543                 if (starts_with(ref->name, "refs/") &&
 544                     check_refname_format(ref->name, 0))
 545                         ; /* trash */
 546                 else {
 547                         while (i < nr_sought) {
 548                                 int cmp = strcmp(ref->name, sought[i]->name);
 549                                 if (cmp < 0)
 550                                         break; /* definitely do not have it */
 551                                 else if (cmp == 0) {
 552                                         keep = 1; /* definitely have it */
 553                                         sought[i]->matched = 1;
 554                                 }
 555                                 i++;
 556                         }
 557                 }
 558
 559                 if (!keep && args->fetch_all &&
 560                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 561                         keep = 1;
 562
 563                 if (keep) {
 564                         *newtail = ref;
 565                         ref->next = NULL;
 566                         newtail = &ref->next;
 567                 } else {
 568                         free(ref);
 569                 }
 570         }
 571
 572         /* Append unmatched requests to the list */
 573         if ((allow_unadvertised_object_request &
 574             (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
 575                 for (i = 0; i < nr_sought; i++) {
 576                         unsigned char sha1[20];
 577
 578                         ref = sought[i];
 579                         if (ref->matched)
 580                                 continue;
 581                         if (get_sha1_hex(ref->name, sha1) ||
 582                             ref->name[40] != '\0' ||
 583                             hashcmp(sha1, ref->old_oid.hash))
 584                                 continue;
 585
 586                         ref->matched = 1;
 587                         *newtail = copy_ref(ref);
 588                         newtail = &(*newtail)->next;
 589                 }
 590         }
 591         *refs = newlist;
 592 }
 593
 594 static void mark_alternate_complete(const struct ref *ref, void *unused)
 595 {
 596         mark_complete(ref->old_oid.hash);
 597 }
 598
 599 static int everything_local(struct fetch_pack_args *args,
 600                             struct ref **refs,
 601                             struct ref **sought, int nr_sought)
 602 {
 603         struct ref *ref;
 604         int retval;
 605         unsigned long cutoff = 0;
 606
 607         save_commit_buffer = 0;
 608
 609         for (ref = *refs; ref; ref = ref->next) {
 610                 struct object *o;
 611
 612                 if (!has_object_file(&ref->old_oid))
 613                         continue;
 614
 615                 o = parse_object(ref->old_oid.hash);
 616                 if (!o)
 617                         continue;
 618
 619                 /* We already have it -- which may mean that we were
 620                  * in sync with the other side at some time after
 621                  * that (it is OK if we guess wrong here).
 622                  */
 623                 if (o->type == OBJ_COMMIT) {
 624                         struct commit *commit = (struct commit *)o;
 625                         if (!cutoff || cutoff < commit->date)
 626                                 cutoff = commit->date;
 627                 }
 628         }
 629
 630         if (!args->depth) {
 631                 for_each_ref(mark_complete_oid, NULL);
 632                 for_each_alternate_ref(mark_alternate_complete, NULL);
 633                 commit_list_sort_by_date(&complete);
 634                 if (cutoff)
 635                         mark_recent_complete_commits(args, cutoff);
 636         }
 637
 638         /*
 639          * Mark all complete remote refs as common refs.
 640          * Don't mark them common yet; the server has to be told so first.
 641          */
 642         for (ref = *refs; ref; ref = ref->next) {
 643                 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 644                                              NULL, 0);
 645
 646                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 647                         continue;
 648
 649                 if (!(o->flags & SEEN)) {
 650                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 651
 652                         mark_common((struct commit *)o, 1, 1);
 653                 }
 654         }
 655
 656         filter_refs(args, refs, sought, nr_sought);
 657
 658         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 659                 const unsigned char *remote = ref->old_oid.hash;
 660                 struct object *o;
 661
 662                 o = lookup_object(remote);
 663                 if (!o || !(o->flags & COMPLETE)) {
 664                         retval = 0;
 665                         print_verbose(args, "want %s (%s)", sha1_to_hex(remote),
 666                                       ref->name);
 667                         continue;
 668                 }
 669                 print_verbose(args, "already have %s (%s)", sha1_to_hex(remote),
 670                               ref->name);
 671         }
 672         return retval;
 673 }
 674
 675 static int sideband_demux(int in, int out, void *data)
 676 {
 677         int *xd = data;
 678
 679         int ret = recv_sideband("fetch-pack", xd[0], out);
 680         close(out);
 681         return ret;
 682 }
 683
 684 static int get_pack(struct fetch_pack_args *args,
 685                     int xd[2], char **pack_lockfile)
 686 {
 687         struct async demux;
 688         int do_keep = args->keep_pack;
 689         const char *cmd_name;
 690         struct pack_header header;
 691         int pass_header = 0;
 692         struct child_process cmd = CHILD_PROCESS_INIT;
 693         int ret;
 694
 695         memset(&demux, 0, sizeof(demux));
 696         if (use_sideband) {
 697                 /* xd[] is talking with upload-pack; subprocess reads from
 698                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 699                  * through demux->out.
 700                  */
 701                 demux.proc = sideband_demux;
 702                 demux.data = xd;
 703                 demux.out = -1;
 704                 if (start_async(&demux))
 705                         die("fetch-pack: unable to fork off sideband"
 706                             " demultiplexer");
 707         }
 708         else
 709                 demux.out = xd[0];
 710
 711         if (!args->keep_pack && unpack_limit) {
 712
 713                 if (read_pack_header(demux.out, &header))
 714                         die("protocol error: bad pack header");
 715                 pass_header = 1;
 716                 if (ntohl(header.hdr_entries) < unpack_limit)
 717                         do_keep = 0;
 718                 else
 719                         do_keep = 1;
 720         }
 721
 722         if (alternate_shallow_file) {
 723                 argv_array_push(&cmd.args, "--shallow-file");
 724                 argv_array_push(&cmd.args, alternate_shallow_file);
 725         }
 726
 727         if (do_keep) {
 728                 if (pack_lockfile)
 729                         cmd.out = -1;
 730                 cmd_name = "index-pack";
 731                 argv_array_push(&cmd.args, cmd_name);
 732                 argv_array_push(&cmd.args, "--stdin");
 733                 if (!args->quiet && !args->no_progress)
 734                         argv_array_push(&cmd.args, "-v");
 735                 if (args->use_thin_pack)
 736                         argv_array_push(&cmd.args, "--fix-thin");
 737                 if (args->lock_pack || unpack_limit) {
 738                         char hostname[256];
 739                         if (gethostname(hostname, sizeof(hostname)))
 740                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 741                         argv_array_pushf(&cmd.args,
 742                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 743                                         (uintmax_t)getpid(), hostname);
 744                 }
 745                 if (args->check_self_contained_and_connected)
 746                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 747         }
 748         else {
 749                 cmd_name = "unpack-objects";
 750                 argv_array_push(&cmd.args, cmd_name);
 751                 if (args->quiet || args->no_progress)
 752                         argv_array_push(&cmd.args, "-q");
 753                 args->check_self_contained_and_connected = 0;
 754         }
 755
 756         if (pass_header)
 757                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 758                                  ntohl(header.hdr_version),
 759                                  ntohl(header.hdr_entries));
 760         if (fetch_fsck_objects >= 0
 761             ? fetch_fsck_objects
 762             : transfer_fsck_objects >= 0
 763             ? transfer_fsck_objects
 764             : 0)
 765                 argv_array_push(&cmd.args, "--strict");
 766
 767         cmd.in = demux.out;
 768         cmd.git_cmd = 1;
 769         if (start_command(&cmd))
 770                 die("fetch-pack: unable to fork off %s", cmd_name);
 771         if (do_keep && pack_lockfile) {
 772                 *pack_lockfile = index_pack_lockfile(cmd.out);
 773                 close(cmd.out);
 774         }
 775
 776         if (!use_sideband)
 777                 /* Closed by start_command() */
 778                 xd[0] = -1;
 779
 780         ret = finish_command(&cmd);
 781         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 782                 args->self_contained_and_connected =
 783                         args->check_self_contained_and_connected &&
 784                         ret == 0;
 785         else
 786                 die("%s failed", cmd_name);
 787         if (use_sideband && finish_async(&demux))
 788                 die("error in sideband demultiplexer");
 789         return 0;
 790 }
 791
 792 static int cmp_ref_by_name(const void *a_, const void *b_)
 793 {
 794         const struct ref *a = *((const struct ref **)a_);
 795         const struct ref *b = *((const struct ref **)b_);
 796         return strcmp(a->name, b->name);
 797 }
 798
 799 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 800                                  int fd[2],
 801                                  const struct ref *orig_ref,
 802                                  struct ref **sought, int nr_sought,
 803                                  struct shallow_info *si,
 804                                  char **pack_lockfile)
 805 {
 806         struct ref *ref = copy_ref_list(orig_ref);
 807         unsigned char sha1[20];
 808         const char *agent_feature;
 809         int agent_len;
 810
 811         sort_ref_list(&ref, ref_compare_name);
 812         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 813
 814         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 815                 die("Server does not support shallow clients");
 816         if (server_supports("multi_ack_detailed")) {
 817                 print_verbose(args, "Server supports multi_ack_detailed");
 818                 multi_ack = 2;
 819                 if (server_supports("no-done")) {
 820                         print_verbose(args, "Server supports no-done");
 821                         if (args->stateless_rpc)
 822                                 no_done = 1;
 823                 }
 824         }
 825         else if (server_supports("multi_ack")) {
 826                 print_verbose(args, "Server supports multi_ack");
 827                 multi_ack = 1;
 828         }
 829         if (server_supports("side-band-64k")) {
 830                 print_verbose(args, "Server supports side-band-64k");
 831                 use_sideband = 2;
 832         }
 833         else if (server_supports("side-band")) {
 834                 print_verbose(args, "Server supports side-band");
 835                 use_sideband = 1;
 836         }
 837         if (server_supports("allow-tip-sha1-in-want")) {
 838                 print_verbose(args, "Server supports allow-tip-sha1-in-want");
 839                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 840         }
 841         if (server_supports("allow-reachable-sha1-in-want")) {
 842                 print_verbose(args, "Server supports allow-reachable-sha1-in-want");
 843                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 844         }
 845         if (!server_supports("thin-pack"))
 846                 args->use_thin_pack = 0;
 847         if (!server_supports("no-progress"))
 848                 args->no_progress = 0;
 849         if (!server_supports("include-tag"))
 850                 args->include_tag = 0;
 851         if (server_supports("ofs-delta"))
 852                 print_verbose(args, "Server supports ofs-delta");
 853         else
 854                 prefer_ofs_delta = 0;
 855
 856         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 857                 agent_supported = 1;
 858                 if (agent_len)
 859                         print_verbose(args, "Server version is %.*s",
 860                                       agent_len, agent_feature);
 861         }
 862
 863         if (everything_local(args, &ref, sought, nr_sought)) {
 864                 packet_flush(fd[1]);
 865                 goto all_done;
 866         }
 867         if (find_common(args, fd, sha1, ref) < 0)
 868                 if (!args->keep_pack)
 869                         /* When cloning, it is not unusual to have
 870                          * no common commit.
 871                          */
 872                         warning("no common commits");
 873
 874         if (args->stateless_rpc)
 875                 packet_flush(fd[1]);
 876         if (args->depth > 0)
 877                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 878                                         NULL);
 879         else if (si->nr_ours || si->nr_theirs)
 880                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 881         else
 882                 alternate_shallow_file = NULL;
 883         if (get_pack(args, fd, pack_lockfile))
 884                 die("git fetch-pack: fetch failed.");
 885
 886  all_done:
 887         return ref;
 888 }
 889
 890 static void fetch_pack_config(void)
 891 {
 892         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 893         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 894         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 895         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 896         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 897
 898         git_config(git_default_config, NULL);
 899 }
 900
 901 static void fetch_pack_setup(void)
 902 {
 903         static int did_setup;
 904         if (did_setup)
 905                 return;
 906         fetch_pack_config();
 907         if (0 <= transfer_unpack_limit)
 908                 unpack_limit = transfer_unpack_limit;
 909         else if (0 <= fetch_unpack_limit)
 910                 unpack_limit = fetch_unpack_limit;
 911         did_setup = 1;
 912 }
 913
 914 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 915 {
 916         struct string_list names = STRING_LIST_INIT_NODUP;
 917         int src, dst;
 918
 919         for (src = dst = 0; src < nr; src++) {
 920                 struct string_list_item *item;
 921                 item = string_list_insert(&names, ref[src]->name);
 922                 if (item->util)
 923                         continue; /* already have it */
 924                 item->util = ref[src];
 925                 if (src != dst)
 926                         ref[dst] = ref[src];
 927                 dst++;
 928         }
 929         for (src = dst; src < nr; src++)
 930                 ref[src] = NULL;
 931         string_list_clear(&names, 0);
 932         return dst;
 933 }
 934
 935 static void update_shallow(struct fetch_pack_args *args,
 936                            struct ref **sought, int nr_sought,
 937                            struct shallow_info *si)
 938 {
 939         struct sha1_array ref = SHA1_ARRAY_INIT;
 940         int *status;
 941         int i;
 942
 943         if (args->depth > 0 && alternate_shallow_file) {
 944                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 945                         unlink_or_warn(git_path_shallow());
 946                         rollback_lock_file(&shallow_lock);
 947                 } else
 948                         commit_lock_file(&shallow_lock);
 949                 return;
 950         }
 951
 952         if (!si->shallow || !si->shallow->nr)
 953                 return;
 954
 955         if (args->cloning) {
 956                 /*
 957                  * remote is shallow, but this is a clone, there are
 958                  * no objects in repo to worry about. Accept any
 959                  * shallow points that exist in the pack (iow in repo
 960                  * after get_pack() and reprepare_packed_git())
 961                  */
 962                 struct sha1_array extra = SHA1_ARRAY_INIT;
 963                 unsigned char (*sha1)[20] = si->shallow->sha1;
 964                 for (i = 0; i < si->shallow->nr; i++)
 965                         if (has_sha1_file(sha1[i]))
 966                                 sha1_array_append(&extra, sha1[i]);
 967                 if (extra.nr) {
 968                         setup_alternate_shallow(&shallow_lock,
 969                                                 &alternate_shallow_file,
 970                                                 &extra);
 971                         commit_lock_file(&shallow_lock);
 972                 }
 973                 sha1_array_clear(&extra);
 974                 return;
 975         }
 976
 977         if (!si->nr_ours && !si->nr_theirs)
 978                 return;
 979
 980         remove_nonexistent_theirs_shallow(si);
 981         if (!si->nr_ours && !si->nr_theirs)
 982                 return;
 983         for (i = 0; i < nr_sought; i++)
 984                 sha1_array_append(&ref, sought[i]->old_oid.hash);
 985         si->ref = &ref;
 986
 987         if (args->update_shallow) {
 988                 /*
 989                  * remote is also shallow, .git/shallow may be updated
 990                  * so all refs can be accepted. Make sure we only add
 991                  * shallow roots that are actually reachable from new
 992                  * refs.
 993                  */
 994                 struct sha1_array extra = SHA1_ARRAY_INIT;
 995                 unsigned char (*sha1)[20] = si->shallow->sha1;
 996                 assign_shallow_commits_to_refs(si, NULL, NULL);
 997                 if (!si->nr_ours && !si->nr_theirs) {
 998                         sha1_array_clear(&ref);
 999                         return;
1000                 }
1001                 for (i = 0; i < si->nr_ours; i++)
1002                         sha1_array_append(&extra, sha1[si->ours[i]]);
1003                 for (i = 0; i < si->nr_theirs; i++)
1004                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1005                 setup_alternate_shallow(&shallow_lock,
1006                                         &alternate_shallow_file,
1007                                         &extra);
1008                 commit_lock_file(&shallow_lock);
1009                 sha1_array_clear(&extra);
1010                 sha1_array_clear(&ref);
1011                 return;
1012         }
1013
1014         /*
1015          * remote is also shallow, check what ref is safe to update
1016          * without updating .git/shallow
1017          */
1018         status = xcalloc(nr_sought, sizeof(*status));
1019         assign_shallow_commits_to_refs(si, NULL, status);
1020         if (si->nr_ours || si->nr_theirs) {
1021                 for (i = 0; i < nr_sought; i++)
1022                         if (status[i])
1023                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1024         }
1025         free(status);
1026         sha1_array_clear(&ref);
1027 }
1028
1029 struct ref *fetch_pack(struct fetch_pack_args *args,
1030                        int fd[], struct child_process *conn,
1031                        const struct ref *ref,
1032                        const char *dest,
1033                        struct ref **sought, int nr_sought,
1034                        struct sha1_array *shallow,
1035                        char **pack_lockfile)
1036 {
1037         struct ref *ref_cpy;
1038         struct shallow_info si;
1039
1040         fetch_pack_setup();
1041         if (nr_sought)
1042                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1043
1044         if (!ref) {
1045                 packet_flush(fd[1]);
1046                 die("no matching remote head");
1047         }
1048         prepare_shallow_info(&si, shallow);
1049         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1050                                 &si, pack_lockfile);
1051         reprepare_packed_git();
1052         update_shallow(args, sought, nr_sought, &si);
1053         clear_shallow_info(&si);
1054         return ref_cpy;
1055 }