fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int deepen_since_ok;
  25 static int fetch_fsck_objects = -1;
  26 static int transfer_fsck_objects = -1;
  27 static int agent_supported;
  28 static struct lock_file shallow_lock;
  29 static const char *alternate_shallow_file;
  30
  31 /* Remember to update object flag allocation in object.h */
  32 #define COMPLETE        (1U << 0)
  33 #define COMMON          (1U << 1)
  34 #define COMMON_REF      (1U << 2)
  35 #define SEEN            (1U << 3)
  36 #define POPPED          (1U << 4)
  37
  38 static int marked;
  39
  40 /*
  41  * After sending this many "have"s if we do not get any new ACK , we
  42  * give up traversing our history.
  43  */
  44 #define MAX_IN_VAIN 256
  45
  46 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  47 static int non_common_revs, multi_ack, use_sideband;
  48 /* Allow specifying sha1 if it is a ref tip. */
  49 #define ALLOW_TIP_SHA1  01
  50 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  51 #define ALLOW_REACHABLE_SHA1    02
  52 static unsigned int allow_unadvertised_object_request;
  53
  54 __attribute__((format (printf, 2, 3)))
  55 static inline void print_verbose(const struct fetch_pack_args *args,
  56                                  const char *fmt, ...)
  57 {
  58         va_list params;
  59
  60         if (!args->verbose)
  61                 return;
  62
  63         va_start(params, fmt);
  64         vfprintf(stderr, fmt, params);
  65         va_end(params);
  66         fputc('\n', stderr);
  67 }
  68
  69 static void rev_list_push(struct commit *commit, int mark)
  70 {
  71         if (!(commit->object.flags & mark)) {
  72                 commit->object.flags |= mark;
  73
  74                 if (parse_commit(commit))
  75                         return;
  76
  77                 prio_queue_put(&rev_list, commit);
  78
  79                 if (!(commit->object.flags & COMMON))
  80                         non_common_revs++;
  81         }
  82 }
  83
  84 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
  85 {
  86         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  87
  88         if (o && o->type == OBJ_COMMIT)
  89                 rev_list_push((struct commit *)o, SEEN);
  90
  91         return 0;
  92 }
  93
  94 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
  95                                    int flag, void *cb_data)
  96 {
  97         return rev_list_insert_ref(refname, oid->hash);
  98 }
  99
 100 static int clear_marks(const char *refname, const struct object_id *oid,
 101                        int flag, void *cb_data)
 102 {
 103         struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
 104
 105         if (o && o->type == OBJ_COMMIT)
 106                 clear_commit_marks((struct commit *)o,
 107                                    COMMON | COMMON_REF | SEEN | POPPED);
 108         return 0;
 109 }
 110
 111 /*
 112    This function marks a rev and its ancestors as common.
 113    In some cases, it is desirable to mark only the ancestors (for example
 114    when only the server does not yet know that they are common).
 115 */
 116
 117 static void mark_common(struct commit *commit,
 118                 int ancestors_only, int dont_parse)
 119 {
 120         if (commit != NULL && !(commit->object.flags & COMMON)) {
 121                 struct object *o = (struct object *)commit;
 122
 123                 if (!ancestors_only)
 124                         o->flags |= COMMON;
 125
 126                 if (!(o->flags & SEEN))
 127                         rev_list_push(commit, SEEN);
 128                 else {
 129                         struct commit_list *parents;
 130
 131                         if (!ancestors_only && !(o->flags & POPPED))
 132                                 non_common_revs--;
 133                         if (!o->parsed && !dont_parse)
 134                                 if (parse_commit(commit))
 135                                         return;
 136
 137                         for (parents = commit->parents;
 138                                         parents;
 139                                         parents = parents->next)
 140                                 mark_common(parents->item, 0, dont_parse);
 141                 }
 142         }
 143 }
 144
 145 /*
 146   Get the next rev to send, ignoring the common.
 147 */
 148
 149 static const unsigned char *get_rev(void)
 150 {
 151         struct commit *commit = NULL;
 152
 153         while (commit == NULL) {
 154                 unsigned int mark;
 155                 struct commit_list *parents;
 156
 157                 if (rev_list.nr == 0 || non_common_revs == 0)
 158                         return NULL;
 159
 160                 commit = prio_queue_get(&rev_list);
 161                 parse_commit(commit);
 162                 parents = commit->parents;
 163
 164                 commit->object.flags |= POPPED;
 165                 if (!(commit->object.flags & COMMON))
 166                         non_common_revs--;
 167
 168                 if (commit->object.flags & COMMON) {
 169                         /* do not send "have", and ignore ancestors */
 170                         commit = NULL;
 171                         mark = COMMON | SEEN;
 172                 } else if (commit->object.flags & COMMON_REF)
 173                         /* send "have", and ignore ancestors */
 174                         mark = COMMON | SEEN;
 175                 else
 176                         /* send "have", also for its ancestors */
 177                         mark = SEEN;
 178
 179                 while (parents) {
 180                         if (!(parents->item->object.flags & SEEN))
 181                                 rev_list_push(parents->item, mark);
 182                         if (mark & COMMON)
 183                                 mark_common(parents->item, 1, 0);
 184                         parents = parents->next;
 185                 }
 186         }
 187
 188         return commit->object.oid.hash;
 189 }
 190
 191 enum ack_type {
 192         NAK = 0,
 193         ACK,
 194         ACK_continue,
 195         ACK_common,
 196         ACK_ready
 197 };
 198
 199 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 200 {
 201         if (args->stateless_rpc && args->deepen) {
 202                 /* If we sent a depth we will get back "duplicate"
 203                  * shallow and unshallow commands every time there
 204                  * is a block of have lines exchanged.
 205                  */
 206                 char *line;
 207                 while ((line = packet_read_line(fd, NULL))) {
 208                         if (starts_with(line, "shallow "))
 209                                 continue;
 210                         if (starts_with(line, "unshallow "))
 211                                 continue;
 212                         die(_("git fetch-pack: expected shallow list"));
 213                 }
 214         }
 215 }
 216
 217 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 218 {
 219         int len;
 220         char *line = packet_read_line(fd, &len);
 221         const char *arg;
 222
 223         if (!len)
 224                 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
 225         if (!strcmp(line, "NAK"))
 226                 return NAK;
 227         if (skip_prefix(line, "ACK ", &arg)) {
 228                 if (!get_sha1_hex(arg, result_sha1)) {
 229                         arg += 40;
 230                         len -= arg - line;
 231                         if (len < 1)
 232                                 return ACK;
 233                         if (strstr(arg, "continue"))
 234                                 return ACK_continue;
 235                         if (strstr(arg, "common"))
 236                                 return ACK_common;
 237                         if (strstr(arg, "ready"))
 238                                 return ACK_ready;
 239                         return ACK;
 240                 }
 241         }
 242         die(_("git fetch_pack: expected ACK/NAK, got '%s'"), line);
 243 }
 244
 245 static void send_request(struct fetch_pack_args *args,
 246                          int fd, struct strbuf *buf)
 247 {
 248         if (args->stateless_rpc) {
 249                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 250                 packet_flush(fd);
 251         } else
 252                 write_or_die(fd, buf->buf, buf->len);
 253 }
 254
 255 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 256 {
 257         rev_list_insert_ref(NULL, ref->old_oid.hash);
 258 }
 259
 260 #define INITIAL_FLUSH 16
 261 #define PIPESAFE_FLUSH 32
 262 #define LARGE_FLUSH 1024
 263
 264 static int next_flush(struct fetch_pack_args *args, int count)
 265 {
 266         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 267
 268         if (count < flush_limit)
 269                 count <<= 1;
 270         else
 271                 count += flush_limit;
 272         return count;
 273 }
 274
 275 static int find_common(struct fetch_pack_args *args,
 276                        int fd[2], unsigned char *result_sha1,
 277                        struct ref *refs)
 278 {
 279         int fetching;
 280         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 281         const unsigned char *sha1;
 282         unsigned in_vain = 0;
 283         int got_continue = 0;
 284         int got_ready = 0;
 285         struct strbuf req_buf = STRBUF_INIT;
 286         size_t state_len = 0;
 287
 288         if (args->stateless_rpc && multi_ack == 1)
 289                 die(_("--stateless-rpc requires multi_ack_detailed"));
 290         if (marked)
 291                 for_each_ref(clear_marks, NULL);
 292         marked = 1;
 293
 294         for_each_ref(rev_list_insert_ref_oid, NULL);
 295         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 296
 297         fetching = 0;
 298         for ( ; refs ; refs = refs->next) {
 299                 unsigned char *remote = refs->old_oid.hash;
 300                 const char *remote_hex;
 301                 struct object *o;
 302
 303                 /*
 304                  * If that object is complete (i.e. it is an ancestor of a
 305                  * local ref), we tell them we have it but do not have to
 306                  * tell them about its ancestors, which they already know
 307                  * about.
 308                  *
 309                  * We use lookup_object here because we are only
 310                  * interested in the case we *know* the object is
 311                  * reachable and we have already scanned it.
 312                  */
 313                 if (((o = lookup_object(remote)) != NULL) &&
 314                                 (o->flags & COMPLETE)) {
 315                         continue;
 316                 }
 317
 318                 remote_hex = sha1_to_hex(remote);
 319                 if (!fetching) {
 320                         struct strbuf c = STRBUF_INIT;
 321                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 322                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 323                         if (no_done)            strbuf_addstr(&c, " no-done");
 324                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 325                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 326                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 327                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 328                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 329                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 330                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 331                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 332                                                             git_user_agent_sanitized());
 333                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 334                         strbuf_release(&c);
 335                 } else
 336                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 337                 fetching++;
 338         }
 339
 340         if (!fetching) {
 341                 strbuf_release(&req_buf);
 342                 packet_flush(fd[1]);
 343                 return 1;
 344         }
 345
 346         if (is_repository_shallow())
 347                 write_shallow_commits(&req_buf, 1, NULL);
 348         if (args->depth > 0)
 349                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 350         if (args->deepen_since) {
 351                 unsigned long max_age = approxidate(args->deepen_since);
 352                 packet_buf_write(&req_buf, "deepen-since %lu", max_age);
 353         }
 354         packet_buf_flush(&req_buf);
 355         state_len = req_buf.len;
 356
 357         if (args->deepen) {
 358                 char *line;
 359                 const char *arg;
 360                 unsigned char sha1[20];
 361
 362                 send_request(args, fd[1], &req_buf);
 363                 while ((line = packet_read_line(fd[0], NULL))) {
 364                         if (skip_prefix(line, "shallow ", &arg)) {
 365                                 if (get_sha1_hex(arg, sha1))
 366                                         die(_("invalid shallow line: %s"), line);
 367                                 register_shallow(sha1);
 368                                 continue;
 369                         }
 370                         if (skip_prefix(line, "unshallow ", &arg)) {
 371                                 if (get_sha1_hex(arg, sha1))
 372                                         die(_("invalid unshallow line: %s"), line);
 373                                 if (!lookup_object(sha1))
 374                                         die(_("object not found: %s"), line);
 375                                 /* make sure that it is parsed as shallow */
 376                                 if (!parse_object(sha1))
 377                                         die(_("error in object: %s"), line);
 378                                 if (unregister_shallow(sha1))
 379                                         die(_("no shallow found: %s"), line);
 380                                 continue;
 381                         }
 382                         die(_("expected shallow/unshallow, got %s"), line);
 383                 }
 384         } else if (!args->stateless_rpc)
 385                 send_request(args, fd[1], &req_buf);
 386
 387         if (!args->stateless_rpc) {
 388                 /* If we aren't using the stateless-rpc interface
 389                  * we don't need to retain the headers.
 390                  */
 391                 strbuf_setlen(&req_buf, 0);
 392                 state_len = 0;
 393         }
 394
 395         flushes = 0;
 396         retval = -1;
 397         while ((sha1 = get_rev())) {
 398                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 399                 print_verbose(args, "have %s", sha1_to_hex(sha1));
 400                 in_vain++;
 401                 if (flush_at <= ++count) {
 402                         int ack;
 403
 404                         packet_buf_flush(&req_buf);
 405                         send_request(args, fd[1], &req_buf);
 406                         strbuf_setlen(&req_buf, state_len);
 407                         flushes++;
 408                         flush_at = next_flush(args, count);
 409
 410                         /*
 411                          * We keep one window "ahead" of the other side, and
 412                          * will wait for an ACK only on the next one
 413                          */
 414                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 415                                 continue;
 416
 417                         consume_shallow_list(args, fd[0]);
 418                         do {
 419                                 ack = get_ack(fd[0], result_sha1);
 420                                 if (ack)
 421                                         print_verbose(args, _("got %s %d %s"), "ack",
 422                                                       ack, sha1_to_hex(result_sha1));
 423                                 switch (ack) {
 424                                 case ACK:
 425                                         flushes = 0;
 426                                         multi_ack = 0;
 427                                         retval = 0;
 428                                         goto done;
 429                                 case ACK_common:
 430                                 case ACK_ready:
 431                                 case ACK_continue: {
 432                                         struct commit *commit =
 433                                                 lookup_commit(result_sha1);
 434                                         if (!commit)
 435                                                 die(_("invalid commit %s"), sha1_to_hex(result_sha1));
 436                                         if (args->stateless_rpc
 437                                          && ack == ACK_common
 438                                          && !(commit->object.flags & COMMON)) {
 439                                                 /* We need to replay the have for this object
 440                                                  * on the next RPC request so the peer knows
 441                                                  * it is in common with us.
 442                                                  */
 443                                                 const char *hex = sha1_to_hex(result_sha1);
 444                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 445                                                 state_len = req_buf.len;
 446                                         }
 447                                         mark_common(commit, 0, 1);
 448                                         retval = 0;
 449                                         in_vain = 0;
 450                                         got_continue = 1;
 451                                         if (ack == ACK_ready) {
 452                                                 clear_prio_queue(&rev_list);
 453                                                 got_ready = 1;
 454                                         }
 455                                         break;
 456                                         }
 457                                 }
 458                         } while (ack);
 459                         flushes--;
 460                         if (got_continue && MAX_IN_VAIN < in_vain) {
 461                                 print_verbose(args, _("giving up"));
 462                                 break; /* give up */
 463                         }
 464                 }
 465         }
 466 done:
 467         if (!got_ready || !no_done) {
 468                 packet_buf_write(&req_buf, "done\n");
 469                 send_request(args, fd[1], &req_buf);
 470         }
 471         print_verbose(args, _("done"));
 472         if (retval != 0) {
 473                 multi_ack = 0;
 474                 flushes++;
 475         }
 476         strbuf_release(&req_buf);
 477
 478         if (!got_ready || !no_done)
 479                 consume_shallow_list(args, fd[0]);
 480         while (flushes || multi_ack) {
 481                 int ack = get_ack(fd[0], result_sha1);
 482                 if (ack) {
 483                         print_verbose(args, _("got %s (%d) %s"), "ack",
 484                                       ack, sha1_to_hex(result_sha1));
 485                         if (ack == ACK)
 486                                 return 0;
 487                         multi_ack = 1;
 488                         continue;
 489                 }
 490                 flushes--;
 491         }
 492         /* it is no error to fetch into a completely empty repo */
 493         return count ? retval : 0;
 494 }
 495
 496 static struct commit_list *complete;
 497
 498 static int mark_complete(const unsigned char *sha1)
 499 {
 500         struct object *o = parse_object(sha1);
 501
 502         while (o && o->type == OBJ_TAG) {
 503                 struct tag *t = (struct tag *) o;
 504                 if (!t->tagged)
 505                         break; /* broken repository */
 506                 o->flags |= COMPLETE;
 507                 o = parse_object(t->tagged->oid.hash);
 508         }
 509         if (o && o->type == OBJ_COMMIT) {
 510                 struct commit *commit = (struct commit *)o;
 511                 if (!(commit->object.flags & COMPLETE)) {
 512                         commit->object.flags |= COMPLETE;
 513                         commit_list_insert(commit, &complete);
 514                 }
 515         }
 516         return 0;
 517 }
 518
 519 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 520                              int flag, void *cb_data)
 521 {
 522         return mark_complete(oid->hash);
 523 }
 524
 525 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 526                                          unsigned long cutoff)
 527 {
 528         while (complete && cutoff <= complete->item->date) {
 529                 print_verbose(args, _("Marking %s as complete"),
 530                               oid_to_hex(&complete->item->object.oid));
 531                 pop_most_recent_commit(&complete, COMPLETE);
 532         }
 533 }
 534
 535 static void filter_refs(struct fetch_pack_args *args,
 536                         struct ref **refs,
 537                         struct ref **sought, int nr_sought)
 538 {
 539         struct ref *newlist = NULL;
 540         struct ref **newtail = &newlist;
 541         struct ref *ref, *next;
 542         int i;
 543
 544         i = 0;
 545         for (ref = *refs; ref; ref = next) {
 546                 int keep = 0;
 547                 next = ref->next;
 548
 549                 if (starts_with(ref->name, "refs/") &&
 550                     check_refname_format(ref->name, 0))
 551                         ; /* trash */
 552                 else {
 553                         while (i < nr_sought) {
 554                                 int cmp = strcmp(ref->name, sought[i]->name);
 555                                 if (cmp < 0)
 556                                         break; /* definitely do not have it */
 557                                 else if (cmp == 0) {
 558                                         keep = 1; /* definitely have it */
 559                                         sought[i]->matched = 1;
 560                                 }
 561                                 i++;
 562                         }
 563                 }
 564
 565                 if (!keep && args->fetch_all &&
 566                     (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 567                         keep = 1;
 568
 569                 if (keep) {
 570                         *newtail = ref;
 571                         ref->next = NULL;
 572                         newtail = &ref->next;
 573                 } else {
 574                         free(ref);
 575                 }
 576         }
 577
 578         /* Append unmatched requests to the list */
 579         if ((allow_unadvertised_object_request &
 580             (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
 581                 for (i = 0; i < nr_sought; i++) {
 582                         unsigned char sha1[20];
 583
 584                         ref = sought[i];
 585                         if (ref->matched)
 586                                 continue;
 587                         if (get_sha1_hex(ref->name, sha1) ||
 588                             ref->name[40] != '\0' ||
 589                             hashcmp(sha1, ref->old_oid.hash))
 590                                 continue;
 591
 592                         ref->matched = 1;
 593                         *newtail = copy_ref(ref);
 594                         newtail = &(*newtail)->next;
 595                 }
 596         }
 597         *refs = newlist;
 598 }
 599
 600 static void mark_alternate_complete(const struct ref *ref, void *unused)
 601 {
 602         mark_complete(ref->old_oid.hash);
 603 }
 604
 605 static int everything_local(struct fetch_pack_args *args,
 606                             struct ref **refs,
 607                             struct ref **sought, int nr_sought)
 608 {
 609         struct ref *ref;
 610         int retval;
 611         unsigned long cutoff = 0;
 612
 613         save_commit_buffer = 0;
 614
 615         for (ref = *refs; ref; ref = ref->next) {
 616                 struct object *o;
 617
 618                 if (!has_object_file(&ref->old_oid))
 619                         continue;
 620
 621                 o = parse_object(ref->old_oid.hash);
 622                 if (!o)
 623                         continue;
 624
 625                 /* We already have it -- which may mean that we were
 626                  * in sync with the other side at some time after
 627                  * that (it is OK if we guess wrong here).
 628                  */
 629                 if (o->type == OBJ_COMMIT) {
 630                         struct commit *commit = (struct commit *)o;
 631                         if (!cutoff || cutoff < commit->date)
 632                                 cutoff = commit->date;
 633                 }
 634         }
 635
 636         if (!args->deepen) {
 637                 for_each_ref(mark_complete_oid, NULL);
 638                 for_each_alternate_ref(mark_alternate_complete, NULL);
 639                 commit_list_sort_by_date(&complete);
 640                 if (cutoff)
 641                         mark_recent_complete_commits(args, cutoff);
 642         }
 643
 644         /*
 645          * Mark all complete remote refs as common refs.
 646          * Don't mark them common yet; the server has to be told so first.
 647          */
 648         for (ref = *refs; ref; ref = ref->next) {
 649                 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 650                                              NULL, 0);
 651
 652                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 653                         continue;
 654
 655                 if (!(o->flags & SEEN)) {
 656                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 657
 658                         mark_common((struct commit *)o, 1, 1);
 659                 }
 660         }
 661
 662         filter_refs(args, refs, sought, nr_sought);
 663
 664         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 665                 const unsigned char *remote = ref->old_oid.hash;
 666                 struct object *o;
 667
 668                 o = lookup_object(remote);
 669                 if (!o || !(o->flags & COMPLETE)) {
 670                         retval = 0;
 671                         print_verbose(args, "want %s (%s)", sha1_to_hex(remote),
 672                                       ref->name);
 673                         continue;
 674                 }
 675                 print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote),
 676                               ref->name);
 677         }
 678         return retval;
 679 }
 680
 681 static int sideband_demux(int in, int out, void *data)
 682 {
 683         int *xd = data;
 684
 685         int ret = recv_sideband("fetch-pack", xd[0], out);
 686         close(out);
 687         return ret;
 688 }
 689
 690 static int get_pack(struct fetch_pack_args *args,
 691                     int xd[2], char **pack_lockfile)
 692 {
 693         struct async demux;
 694         int do_keep = args->keep_pack;
 695         const char *cmd_name;
 696         struct pack_header header;
 697         int pass_header = 0;
 698         struct child_process cmd = CHILD_PROCESS_INIT;
 699         int ret;
 700
 701         memset(&demux, 0, sizeof(demux));
 702         if (use_sideband) {
 703                 /* xd[] is talking with upload-pack; subprocess reads from
 704                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 705                  * through demux->out.
 706                  */
 707                 demux.proc = sideband_demux;
 708                 demux.data = xd;
 709                 demux.out = -1;
 710                 if (start_async(&demux))
 711                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 712         }
 713         else
 714                 demux.out = xd[0];
 715
 716         if (!args->keep_pack && unpack_limit) {
 717
 718                 if (read_pack_header(demux.out, &header))
 719                         die(_("protocol error: bad pack header"));
 720                 pass_header = 1;
 721                 if (ntohl(header.hdr_entries) < unpack_limit)
 722                         do_keep = 0;
 723                 else
 724                         do_keep = 1;
 725         }
 726
 727         if (alternate_shallow_file) {
 728                 argv_array_push(&cmd.args, "--shallow-file");
 729                 argv_array_push(&cmd.args, alternate_shallow_file);
 730         }
 731
 732         if (do_keep) {
 733                 if (pack_lockfile)
 734                         cmd.out = -1;
 735                 cmd_name = "index-pack";
 736                 argv_array_push(&cmd.args, cmd_name);
 737                 argv_array_push(&cmd.args, "--stdin");
 738                 if (!args->quiet && !args->no_progress)
 739                         argv_array_push(&cmd.args, "-v");
 740                 if (args->use_thin_pack)
 741                         argv_array_push(&cmd.args, "--fix-thin");
 742                 if (args->lock_pack || unpack_limit) {
 743                         char hostname[256];
 744                         if (gethostname(hostname, sizeof(hostname)))
 745                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 746                         argv_array_pushf(&cmd.args,
 747                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 748                                         (uintmax_t)getpid(), hostname);
 749                 }
 750                 if (args->check_self_contained_and_connected)
 751                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 752         }
 753         else {
 754                 cmd_name = "unpack-objects";
 755                 argv_array_push(&cmd.args, cmd_name);
 756                 if (args->quiet || args->no_progress)
 757                         argv_array_push(&cmd.args, "-q");
 758                 args->check_self_contained_and_connected = 0;
 759         }
 760
 761         if (pass_header)
 762                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 763                                  ntohl(header.hdr_version),
 764                                  ntohl(header.hdr_entries));
 765         if (fetch_fsck_objects >= 0
 766             ? fetch_fsck_objects
 767             : transfer_fsck_objects >= 0
 768             ? transfer_fsck_objects
 769             : 0)
 770                 argv_array_push(&cmd.args, "--strict");
 771
 772         cmd.in = demux.out;
 773         cmd.git_cmd = 1;
 774         if (start_command(&cmd))
 775                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 776         if (do_keep && pack_lockfile) {
 777                 *pack_lockfile = index_pack_lockfile(cmd.out);
 778                 close(cmd.out);
 779         }
 780
 781         if (!use_sideband)
 782                 /* Closed by start_command() */
 783                 xd[0] = -1;
 784
 785         ret = finish_command(&cmd);
 786         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 787                 args->self_contained_and_connected =
 788                         args->check_self_contained_and_connected &&
 789                         ret == 0;
 790         else
 791                 die(_("%s failed"), cmd_name);
 792         if (use_sideband && finish_async(&demux))
 793                 die(_("error in sideband demultiplexer"));
 794         return 0;
 795 }
 796
 797 static int cmp_ref_by_name(const void *a_, const void *b_)
 798 {
 799         const struct ref *a = *((const struct ref **)a_);
 800         const struct ref *b = *((const struct ref **)b_);
 801         return strcmp(a->name, b->name);
 802 }
 803
 804 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 805                                  int fd[2],
 806                                  const struct ref *orig_ref,
 807                                  struct ref **sought, int nr_sought,
 808                                  struct shallow_info *si,
 809                                  char **pack_lockfile)
 810 {
 811         struct ref *ref = copy_ref_list(orig_ref);
 812         unsigned char sha1[20];
 813         const char *agent_feature;
 814         int agent_len;
 815
 816         sort_ref_list(&ref, ref_compare_name);
 817         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 818
 819         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 820                 die(_("Server does not support shallow clients"));
 821         if (args->depth > 0 || args->deepen_since)
 822                 args->deepen = 1;
 823         if (server_supports("multi_ack_detailed")) {
 824                 print_verbose(args, _("Server supports multi_ack_detailed"));
 825                 multi_ack = 2;
 826                 if (server_supports("no-done")) {
 827                         print_verbose(args, _("Server supports no-done"));
 828                         if (args->stateless_rpc)
 829                                 no_done = 1;
 830                 }
 831         }
 832         else if (server_supports("multi_ack")) {
 833                 print_verbose(args, _("Server supports multi_ack"));
 834                 multi_ack = 1;
 835         }
 836         if (server_supports("side-band-64k")) {
 837                 print_verbose(args, _("Server supports side-band-64k"));
 838                 use_sideband = 2;
 839         }
 840         else if (server_supports("side-band")) {
 841                 print_verbose(args, _("Server supports side-band"));
 842                 use_sideband = 1;
 843         }
 844         if (server_supports("allow-tip-sha1-in-want")) {
 845                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
 846                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 847         }
 848         if (server_supports("allow-reachable-sha1-in-want")) {
 849                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
 850                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 851         }
 852         if (!server_supports("thin-pack"))
 853                 args->use_thin_pack = 0;
 854         if (!server_supports("no-progress"))
 855                 args->no_progress = 0;
 856         if (!server_supports("include-tag"))
 857                 args->include_tag = 0;
 858         if (server_supports("ofs-delta"))
 859                 print_verbose(args, _("Server supports ofs-delta"));
 860         else
 861                 prefer_ofs_delta = 0;
 862
 863         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 864                 agent_supported = 1;
 865                 if (agent_len)
 866                         print_verbose(args, _("Server version is %.*s"),
 867                                       agent_len, agent_feature);
 868         }
 869         if (server_supports("deepen-since"))
 870                 deepen_since_ok = 1;
 871         else if (args->deepen_since)
 872                 die(_("Server does not support --shallow-since"));
 873
 874         if (everything_local(args, &ref, sought, nr_sought)) {
 875                 packet_flush(fd[1]);
 876                 goto all_done;
 877         }
 878         if (find_common(args, fd, sha1, ref) < 0)
 879                 if (!args->keep_pack)
 880                         /* When cloning, it is not unusual to have
 881                          * no common commit.
 882                          */
 883                         warning(_("no common commits"));
 884
 885         if (args->stateless_rpc)
 886                 packet_flush(fd[1]);
 887         if (args->deepen)
 888                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 889                                         NULL);
 890         else if (si->nr_ours || si->nr_theirs)
 891                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 892         else
 893                 alternate_shallow_file = NULL;
 894         if (get_pack(args, fd, pack_lockfile))
 895                 die(_("git fetch-pack: fetch failed."));
 896
 897  all_done:
 898         return ref;
 899 }
 900
 901 static void fetch_pack_config(void)
 902 {
 903         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 904         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 905         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 906         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 907         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 908
 909         git_config(git_default_config, NULL);
 910 }
 911
 912 static void fetch_pack_setup(void)
 913 {
 914         static int did_setup;
 915         if (did_setup)
 916                 return;
 917         fetch_pack_config();
 918         if (0 <= transfer_unpack_limit)
 919                 unpack_limit = transfer_unpack_limit;
 920         else if (0 <= fetch_unpack_limit)
 921                 unpack_limit = fetch_unpack_limit;
 922         did_setup = 1;
 923 }
 924
 925 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 926 {
 927         struct string_list names = STRING_LIST_INIT_NODUP;
 928         int src, dst;
 929
 930         for (src = dst = 0; src < nr; src++) {
 931                 struct string_list_item *item;
 932                 item = string_list_insert(&names, ref[src]->name);
 933                 if (item->util)
 934                         continue; /* already have it */
 935                 item->util = ref[src];
 936                 if (src != dst)
 937                         ref[dst] = ref[src];
 938                 dst++;
 939         }
 940         for (src = dst; src < nr; src++)
 941                 ref[src] = NULL;
 942         string_list_clear(&names, 0);
 943         return dst;
 944 }
 945
 946 static void update_shallow(struct fetch_pack_args *args,
 947                            struct ref **sought, int nr_sought,
 948                            struct shallow_info *si)
 949 {
 950         struct sha1_array ref = SHA1_ARRAY_INIT;
 951         int *status;
 952         int i;
 953
 954         if (args->deepen && alternate_shallow_file) {
 955                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 956                         unlink_or_warn(git_path_shallow());
 957                         rollback_lock_file(&shallow_lock);
 958                 } else
 959                         commit_lock_file(&shallow_lock);
 960                 return;
 961         }
 962
 963         if (!si->shallow || !si->shallow->nr)
 964                 return;
 965
 966         if (args->cloning) {
 967                 /*
 968                  * remote is shallow, but this is a clone, there are
 969                  * no objects in repo to worry about. Accept any
 970                  * shallow points that exist in the pack (iow in repo
 971                  * after get_pack() and reprepare_packed_git())
 972                  */
 973                 struct sha1_array extra = SHA1_ARRAY_INIT;
 974                 unsigned char (*sha1)[20] = si->shallow->sha1;
 975                 for (i = 0; i < si->shallow->nr; i++)
 976                         if (has_sha1_file(sha1[i]))
 977                                 sha1_array_append(&extra, sha1[i]);
 978                 if (extra.nr) {
 979                         setup_alternate_shallow(&shallow_lock,
 980                                                 &alternate_shallow_file,
 981                                                 &extra);
 982                         commit_lock_file(&shallow_lock);
 983                 }
 984                 sha1_array_clear(&extra);
 985                 return;
 986         }
 987
 988         if (!si->nr_ours && !si->nr_theirs)
 989                 return;
 990
 991         remove_nonexistent_theirs_shallow(si);
 992         if (!si->nr_ours && !si->nr_theirs)
 993                 return;
 994         for (i = 0; i < nr_sought; i++)
 995                 sha1_array_append(&ref, sought[i]->old_oid.hash);
 996         si->ref = &ref;
 997
 998         if (args->update_shallow) {
 999                 /*
1000                  * remote is also shallow, .git/shallow may be updated
1001                  * so all refs can be accepted. Make sure we only add
1002                  * shallow roots that are actually reachable from new
1003                  * refs.
1004                  */
1005                 struct sha1_array extra = SHA1_ARRAY_INIT;
1006                 unsigned char (*sha1)[20] = si->shallow->sha1;
1007                 assign_shallow_commits_to_refs(si, NULL, NULL);
1008                 if (!si->nr_ours && !si->nr_theirs) {
1009                         sha1_array_clear(&ref);
1010                         return;
1011                 }
1012                 for (i = 0; i < si->nr_ours; i++)
1013                         sha1_array_append(&extra, sha1[si->ours[i]]);
1014                 for (i = 0; i < si->nr_theirs; i++)
1015                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1016                 setup_alternate_shallow(&shallow_lock,
1017                                         &alternate_shallow_file,
1018                                         &extra);
1019                 commit_lock_file(&shallow_lock);
1020                 sha1_array_clear(&extra);
1021                 sha1_array_clear(&ref);
1022                 return;
1023         }
1024
1025         /*
1026          * remote is also shallow, check what ref is safe to update
1027          * without updating .git/shallow
1028          */
1029         status = xcalloc(nr_sought, sizeof(*status));
1030         assign_shallow_commits_to_refs(si, NULL, status);
1031         if (si->nr_ours || si->nr_theirs) {
1032                 for (i = 0; i < nr_sought; i++)
1033                         if (status[i])
1034                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1035         }
1036         free(status);
1037         sha1_array_clear(&ref);
1038 }
1039
1040 struct ref *fetch_pack(struct fetch_pack_args *args,
1041                        int fd[], struct child_process *conn,
1042                        const struct ref *ref,
1043                        const char *dest,
1044                        struct ref **sought, int nr_sought,
1045                        struct sha1_array *shallow,
1046                        char **pack_lockfile)
1047 {
1048         struct ref *ref_cpy;
1049         struct shallow_info si;
1050
1051         fetch_pack_setup();
1052         if (nr_sought)
1053                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1054
1055         if (!ref) {
1056                 packet_flush(fd[1]);
1057                 die(_("no matching remote head"));
1058         }
1059         prepare_shallow_info(&si, shallow);
1060         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1061                                 &si, pack_lockfile);
1062         reprepare_packed_git();
1063         update_shallow(args, sought, nr_sought, &si);
1064         clear_shallow_info(&si);
1065         return ref_cpy;
1066 }