fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int deepen_since_ok;
  25 static int deepen_not_ok;
  26 static int fetch_fsck_objects = -1;
  27 static int transfer_fsck_objects = -1;
  28 static int agent_supported;
  29 static struct lock_file shallow_lock;
  30 static const char *alternate_shallow_file;
  31
  32 /* Remember to update object flag allocation in object.h */
  33 #define COMPLETE        (1U << 0)
  34 #define COMMON          (1U << 1)
  35 #define COMMON_REF      (1U << 2)
  36 #define SEEN            (1U << 3)
  37 #define POPPED          (1U << 4)
  38
  39 static int marked;
  40
  41 /*
  42  * After sending this many "have"s if we do not get any new ACK , we
  43  * give up traversing our history.
  44  */
  45 #define MAX_IN_VAIN 256
  46
  47 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  48 static int non_common_revs, multi_ack, use_sideband;
  49 /* Allow specifying sha1 if it is a ref tip. */
  50 #define ALLOW_TIP_SHA1  01
  51 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  52 #define ALLOW_REACHABLE_SHA1    02
  53 static unsigned int allow_unadvertised_object_request;
  54
  55 __attribute__((format (printf, 2, 3)))
  56 static inline void print_verbose(const struct fetch_pack_args *args,
  57                                  const char *fmt, ...)
  58 {
  59         va_list params;
  60
  61         if (!args->verbose)
  62                 return;
  63
  64         va_start(params, fmt);
  65         vfprintf(stderr, fmt, params);
  66         va_end(params);
  67         fputc('\n', stderr);
  68 }
  69
  70 static void rev_list_push(struct commit *commit, int mark)
  71 {
  72         if (!(commit->object.flags & mark)) {
  73                 commit->object.flags |= mark;
  74
  75                 if (parse_commit(commit))
  76                         return;
  77
  78                 prio_queue_put(&rev_list, commit);
  79
  80                 if (!(commit->object.flags & COMMON))
  81                         non_common_revs++;
  82         }
  83 }
  84
  85 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
  86 {
  87         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  88
  89         if (o && o->type == OBJ_COMMIT)
  90                 rev_list_push((struct commit *)o, SEEN);
  91
  92         return 0;
  93 }
  94
  95 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
  96                                    int flag, void *cb_data)
  97 {
  98         return rev_list_insert_ref(refname, oid->hash);
  99 }
 100
 101 static int clear_marks(const char *refname, const struct object_id *oid,
 102                        int flag, void *cb_data)
 103 {
 104         struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
 105
 106         if (o && o->type == OBJ_COMMIT)
 107                 clear_commit_marks((struct commit *)o,
 108                                    COMMON | COMMON_REF | SEEN | POPPED);
 109         return 0;
 110 }
 111
 112 /*
 113    This function marks a rev and its ancestors as common.
 114    In some cases, it is desirable to mark only the ancestors (for example
 115    when only the server does not yet know that they are common).
 116 */
 117
 118 static void mark_common(struct commit *commit,
 119                 int ancestors_only, int dont_parse)
 120 {
 121         if (commit != NULL && !(commit->object.flags & COMMON)) {
 122                 struct object *o = (struct object *)commit;
 123
 124                 if (!ancestors_only)
 125                         o->flags |= COMMON;
 126
 127                 if (!(o->flags & SEEN))
 128                         rev_list_push(commit, SEEN);
 129                 else {
 130                         struct commit_list *parents;
 131
 132                         if (!ancestors_only && !(o->flags & POPPED))
 133                                 non_common_revs--;
 134                         if (!o->parsed && !dont_parse)
 135                                 if (parse_commit(commit))
 136                                         return;
 137
 138                         for (parents = commit->parents;
 139                                         parents;
 140                                         parents = parents->next)
 141                                 mark_common(parents->item, 0, dont_parse);
 142                 }
 143         }
 144 }
 145
 146 /*
 147   Get the next rev to send, ignoring the common.
 148 */
 149
 150 static const unsigned char *get_rev(void)
 151 {
 152         struct commit *commit = NULL;
 153
 154         while (commit == NULL) {
 155                 unsigned int mark;
 156                 struct commit_list *parents;
 157
 158                 if (rev_list.nr == 0 || non_common_revs == 0)
 159                         return NULL;
 160
 161                 commit = prio_queue_get(&rev_list);
 162                 parse_commit(commit);
 163                 parents = commit->parents;
 164
 165                 commit->object.flags |= POPPED;
 166                 if (!(commit->object.flags & COMMON))
 167                         non_common_revs--;
 168
 169                 if (commit->object.flags & COMMON) {
 170                         /* do not send "have", and ignore ancestors */
 171                         commit = NULL;
 172                         mark = COMMON | SEEN;
 173                 } else if (commit->object.flags & COMMON_REF)
 174                         /* send "have", and ignore ancestors */
 175                         mark = COMMON | SEEN;
 176                 else
 177                         /* send "have", also for its ancestors */
 178                         mark = SEEN;
 179
 180                 while (parents) {
 181                         if (!(parents->item->object.flags & SEEN))
 182                                 rev_list_push(parents->item, mark);
 183                         if (mark & COMMON)
 184                                 mark_common(parents->item, 1, 0);
 185                         parents = parents->next;
 186                 }
 187         }
 188
 189         return commit->object.oid.hash;
 190 }
 191
 192 enum ack_type {
 193         NAK = 0,
 194         ACK,
 195         ACK_continue,
 196         ACK_common,
 197         ACK_ready
 198 };
 199
 200 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 201 {
 202         if (args->stateless_rpc && args->deepen) {
 203                 /* If we sent a depth we will get back "duplicate"
 204                  * shallow and unshallow commands every time there
 205                  * is a block of have lines exchanged.
 206                  */
 207                 char *line;
 208                 while ((line = packet_read_line(fd, NULL))) {
 209                         if (starts_with(line, "shallow "))
 210                                 continue;
 211                         if (starts_with(line, "unshallow "))
 212                                 continue;
 213                         die(_("git fetch-pack: expected shallow list"));
 214                 }
 215         }
 216 }
 217
 218 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 219 {
 220         int len;
 221         char *line = packet_read_line(fd, &len);
 222         const char *arg;
 223
 224         if (!len)
 225                 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
 226         if (!strcmp(line, "NAK"))
 227                 return NAK;
 228         if (skip_prefix(line, "ACK ", &arg)) {
 229                 if (!get_sha1_hex(arg, result_sha1)) {
 230                         arg += 40;
 231                         len -= arg - line;
 232                         if (len < 1)
 233                                 return ACK;
 234                         if (strstr(arg, "continue"))
 235                                 return ACK_continue;
 236                         if (strstr(arg, "common"))
 237                                 return ACK_common;
 238                         if (strstr(arg, "ready"))
 239                                 return ACK_ready;
 240                         return ACK;
 241                 }
 242         }
 243         die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 244 }
 245
 246 static void send_request(struct fetch_pack_args *args,
 247                          int fd, struct strbuf *buf)
 248 {
 249         if (args->stateless_rpc) {
 250                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 251                 packet_flush(fd);
 252         } else
 253                 write_or_die(fd, buf->buf, buf->len);
 254 }
 255
 256 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 257 {
 258         rev_list_insert_ref(NULL, ref->old_oid.hash);
 259 }
 260
 261 #define INITIAL_FLUSH 16
 262 #define PIPESAFE_FLUSH 32
 263 #define LARGE_FLUSH 16384
 264
 265 static int next_flush(struct fetch_pack_args *args, int count)
 266 {
 267         if (args->stateless_rpc) {
 268                 if (count < LARGE_FLUSH)
 269                         count <<= 1;
 270                 else
 271                         count = count * 11 / 10;
 272         } else {
 273                 if (count < PIPESAFE_FLUSH)
 274                         count <<= 1;
 275                 else
 276                         count += PIPESAFE_FLUSH;
 277         }
 278         return count;
 279 }
 280
 281 static int find_common(struct fetch_pack_args *args,
 282                        int fd[2], unsigned char *result_sha1,
 283                        struct ref *refs)
 284 {
 285         int fetching;
 286         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 287         const unsigned char *sha1;
 288         unsigned in_vain = 0;
 289         int got_continue = 0;
 290         int got_ready = 0;
 291         struct strbuf req_buf = STRBUF_INIT;
 292         size_t state_len = 0;
 293
 294         if (args->stateless_rpc && multi_ack == 1)
 295                 die(_("--stateless-rpc requires multi_ack_detailed"));
 296         if (marked)
 297                 for_each_ref(clear_marks, NULL);
 298         marked = 1;
 299
 300         for_each_ref(rev_list_insert_ref_oid, NULL);
 301         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 302
 303         fetching = 0;
 304         for ( ; refs ; refs = refs->next) {
 305                 unsigned char *remote = refs->old_oid.hash;
 306                 const char *remote_hex;
 307                 struct object *o;
 308
 309                 /*
 310                  * If that object is complete (i.e. it is an ancestor of a
 311                  * local ref), we tell them we have it but do not have to
 312                  * tell them about its ancestors, which they already know
 313                  * about.
 314                  *
 315                  * We use lookup_object here because we are only
 316                  * interested in the case we *know* the object is
 317                  * reachable and we have already scanned it.
 318                  */
 319                 if (((o = lookup_object(remote)) != NULL) &&
 320                                 (o->flags & COMPLETE)) {
 321                         continue;
 322                 }
 323
 324                 remote_hex = sha1_to_hex(remote);
 325                 if (!fetching) {
 326                         struct strbuf c = STRBUF_INIT;
 327                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 328                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 329                         if (no_done)            strbuf_addstr(&c, " no-done");
 330                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 331                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 332                         if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 333                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 334                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 335                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 336                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 337                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 338                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 339                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 340                                                             git_user_agent_sanitized());
 341                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 342                         strbuf_release(&c);
 343                 } else
 344                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 345                 fetching++;
 346         }
 347
 348         if (!fetching) {
 349                 strbuf_release(&req_buf);
 350                 packet_flush(fd[1]);
 351                 return 1;
 352         }
 353
 354         if (is_repository_shallow())
 355                 write_shallow_commits(&req_buf, 1, NULL);
 356         if (args->depth > 0)
 357                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 358         if (args->deepen_since) {
 359                 unsigned long max_age = approxidate(args->deepen_since);
 360                 packet_buf_write(&req_buf, "deepen-since %lu", max_age);
 361         }
 362         if (args->deepen_not) {
 363                 int i;
 364                 for (i = 0; i < args->deepen_not->nr; i++) {
 365                         struct string_list_item *s = args->deepen_not->items + i;
 366                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
 367                 }
 368         }
 369         packet_buf_flush(&req_buf);
 370         state_len = req_buf.len;
 371
 372         if (args->deepen) {
 373                 char *line;
 374                 const char *arg;
 375                 unsigned char sha1[20];
 376
 377                 send_request(args, fd[1], &req_buf);
 378                 while ((line = packet_read_line(fd[0], NULL))) {
 379                         if (skip_prefix(line, "shallow ", &arg)) {
 380                                 if (get_sha1_hex(arg, sha1))
 381                                         die(_("invalid shallow line: %s"), line);
 382                                 register_shallow(sha1);
 383                                 continue;
 384                         }
 385                         if (skip_prefix(line, "unshallow ", &arg)) {
 386                                 if (get_sha1_hex(arg, sha1))
 387                                         die(_("invalid unshallow line: %s"), line);
 388                                 if (!lookup_object(sha1))
 389                                         die(_("object not found: %s"), line);
 390                                 /* make sure that it is parsed as shallow */
 391                                 if (!parse_object(sha1))
 392                                         die(_("error in object: %s"), line);
 393                                 if (unregister_shallow(sha1))
 394                                         die(_("no shallow found: %s"), line);
 395                                 continue;
 396                         }
 397                         die(_("expected shallow/unshallow, got %s"), line);
 398                 }
 399         } else if (!args->stateless_rpc)
 400                 send_request(args, fd[1], &req_buf);
 401
 402         if (!args->stateless_rpc) {
 403                 /* If we aren't using the stateless-rpc interface
 404                  * we don't need to retain the headers.
 405                  */
 406                 strbuf_setlen(&req_buf, 0);
 407                 state_len = 0;
 408         }
 409
 410         flushes = 0;
 411         retval = -1;
 412         while ((sha1 = get_rev())) {
 413                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 414                 print_verbose(args, "have %s", sha1_to_hex(sha1));
 415                 in_vain++;
 416                 if (flush_at <= ++count) {
 417                         int ack;
 418
 419                         packet_buf_flush(&req_buf);
 420                         send_request(args, fd[1], &req_buf);
 421                         strbuf_setlen(&req_buf, state_len);
 422                         flushes++;
 423                         flush_at = next_flush(args, count);
 424
 425                         /*
 426                          * We keep one window "ahead" of the other side, and
 427                          * will wait for an ACK only on the next one
 428                          */
 429                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 430                                 continue;
 431
 432                         consume_shallow_list(args, fd[0]);
 433                         do {
 434                                 ack = get_ack(fd[0], result_sha1);
 435                                 if (ack)
 436                                         print_verbose(args, _("got %s %d %s"), "ack",
 437                                                       ack, sha1_to_hex(result_sha1));
 438                                 switch (ack) {
 439                                 case ACK:
 440                                         flushes = 0;
 441                                         multi_ack = 0;
 442                                         retval = 0;
 443                                         goto done;
 444                                 case ACK_common:
 445                                 case ACK_ready:
 446                                 case ACK_continue: {
 447                                         struct commit *commit =
 448                                                 lookup_commit(result_sha1);
 449                                         if (!commit)
 450                                                 die(_("invalid commit %s"), sha1_to_hex(result_sha1));
 451                                         if (args->stateless_rpc
 452                                          && ack == ACK_common
 453                                          && !(commit->object.flags & COMMON)) {
 454                                                 /* We need to replay the have for this object
 455                                                  * on the next RPC request so the peer knows
 456                                                  * it is in common with us.
 457                                                  */
 458                                                 const char *hex = sha1_to_hex(result_sha1);
 459                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 460                                                 state_len = req_buf.len;
 461                                                 /*
 462                                                  * Reset in_vain because an ack
 463                                                  * for this commit has not been
 464                                                  * seen.
 465                                                  */
 466                                                 in_vain = 0;
 467                                         } else if (!args->stateless_rpc
 468                                                    || ack != ACK_common)
 469                                                 in_vain = 0;
 470                                         mark_common(commit, 0, 1);
 471                                         retval = 0;
 472                                         got_continue = 1;
 473                                         if (ack == ACK_ready) {
 474                                                 clear_prio_queue(&rev_list);
 475                                                 got_ready = 1;
 476                                         }
 477                                         break;
 478                                         }
 479                                 }
 480                         } while (ack);
 481                         flushes--;
 482                         if (got_continue && MAX_IN_VAIN < in_vain) {
 483                                 print_verbose(args, _("giving up"));
 484                                 break; /* give up */
 485                         }
 486                 }
 487         }
 488 done:
 489         if (!got_ready || !no_done) {
 490                 packet_buf_write(&req_buf, "done\n");
 491                 send_request(args, fd[1], &req_buf);
 492         }
 493         print_verbose(args, _("done"));
 494         if (retval != 0) {
 495                 multi_ack = 0;
 496                 flushes++;
 497         }
 498         strbuf_release(&req_buf);
 499
 500         if (!got_ready || !no_done)
 501                 consume_shallow_list(args, fd[0]);
 502         while (flushes || multi_ack) {
 503                 int ack = get_ack(fd[0], result_sha1);
 504                 if (ack) {
 505                         print_verbose(args, _("got %s (%d) %s"), "ack",
 506                                       ack, sha1_to_hex(result_sha1));
 507                         if (ack == ACK)
 508                                 return 0;
 509                         multi_ack = 1;
 510                         continue;
 511                 }
 512                 flushes--;
 513         }
 514         /* it is no error to fetch into a completely empty repo */
 515         return count ? retval : 0;
 516 }
 517
 518 static struct commit_list *complete;
 519
 520 static int mark_complete(const unsigned char *sha1)
 521 {
 522         struct object *o = parse_object(sha1);
 523
 524         while (o && o->type == OBJ_TAG) {
 525                 struct tag *t = (struct tag *) o;
 526                 if (!t->tagged)
 527                         break; /* broken repository */
 528                 o->flags |= COMPLETE;
 529                 o = parse_object(t->tagged->oid.hash);
 530         }
 531         if (o && o->type == OBJ_COMMIT) {
 532                 struct commit *commit = (struct commit *)o;
 533                 if (!(commit->object.flags & COMPLETE)) {
 534                         commit->object.flags |= COMPLETE;
 535                         commit_list_insert(commit, &complete);
 536                 }
 537         }
 538         return 0;
 539 }
 540
 541 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 542                              int flag, void *cb_data)
 543 {
 544         return mark_complete(oid->hash);
 545 }
 546
 547 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 548                                          unsigned long cutoff)
 549 {
 550         while (complete && cutoff <= complete->item->date) {
 551                 print_verbose(args, _("Marking %s as complete"),
 552                               oid_to_hex(&complete->item->object.oid));
 553                 pop_most_recent_commit(&complete, COMPLETE);
 554         }
 555 }
 556
 557 static void filter_refs(struct fetch_pack_args *args,
 558                         struct ref **refs,
 559                         struct ref **sought, int nr_sought)
 560 {
 561         struct ref *newlist = NULL;
 562         struct ref **newtail = &newlist;
 563         struct ref *ref, *next;
 564         int i;
 565
 566         i = 0;
 567         for (ref = *refs; ref; ref = next) {
 568                 int keep = 0;
 569                 next = ref->next;
 570
 571                 if (starts_with(ref->name, "refs/") &&
 572                     check_refname_format(ref->name, 0))
 573                         ; /* trash */
 574                 else {
 575                         while (i < nr_sought) {
 576                                 int cmp = strcmp(ref->name, sought[i]->name);
 577                                 if (cmp < 0)
 578                                         break; /* definitely do not have it */
 579                                 else if (cmp == 0) {
 580                                         keep = 1; /* definitely have it */
 581                                         sought[i]->matched = 1;
 582                                 }
 583                                 i++;
 584                         }
 585                 }
 586
 587                 if (!keep && args->fetch_all &&
 588                     (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 589                         keep = 1;
 590
 591                 if (keep) {
 592                         *newtail = ref;
 593                         ref->next = NULL;
 594                         newtail = &ref->next;
 595                 } else {
 596                         free(ref);
 597                 }
 598         }
 599
 600         /* Append unmatched requests to the list */
 601         if ((allow_unadvertised_object_request &
 602             (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
 603                 for (i = 0; i < nr_sought; i++) {
 604                         unsigned char sha1[20];
 605
 606                         ref = sought[i];
 607                         if (ref->matched)
 608                                 continue;
 609                         if (get_sha1_hex(ref->name, sha1) ||
 610                             ref->name[40] != '\0' ||
 611                             hashcmp(sha1, ref->old_oid.hash))
 612                                 continue;
 613
 614                         ref->matched = 1;
 615                         *newtail = copy_ref(ref);
 616                         newtail = &(*newtail)->next;
 617                 }
 618         }
 619         *refs = newlist;
 620 }
 621
 622 static void mark_alternate_complete(const struct ref *ref, void *unused)
 623 {
 624         mark_complete(ref->old_oid.hash);
 625 }
 626
 627 static int everything_local(struct fetch_pack_args *args,
 628                             struct ref **refs,
 629                             struct ref **sought, int nr_sought)
 630 {
 631         struct ref *ref;
 632         int retval;
 633         unsigned long cutoff = 0;
 634
 635         save_commit_buffer = 0;
 636
 637         for (ref = *refs; ref; ref = ref->next) {
 638                 struct object *o;
 639
 640                 if (!has_object_file(&ref->old_oid))
 641                         continue;
 642
 643                 o = parse_object(ref->old_oid.hash);
 644                 if (!o)
 645                         continue;
 646
 647                 /* We already have it -- which may mean that we were
 648                  * in sync with the other side at some time after
 649                  * that (it is OK if we guess wrong here).
 650                  */
 651                 if (o->type == OBJ_COMMIT) {
 652                         struct commit *commit = (struct commit *)o;
 653                         if (!cutoff || cutoff < commit->date)
 654                                 cutoff = commit->date;
 655                 }
 656         }
 657
 658         if (!args->deepen) {
 659                 for_each_ref(mark_complete_oid, NULL);
 660                 for_each_alternate_ref(mark_alternate_complete, NULL);
 661                 commit_list_sort_by_date(&complete);
 662                 if (cutoff)
 663                         mark_recent_complete_commits(args, cutoff);
 664         }
 665
 666         /*
 667          * Mark all complete remote refs as common refs.
 668          * Don't mark them common yet; the server has to be told so first.
 669          */
 670         for (ref = *refs; ref; ref = ref->next) {
 671                 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 672                                              NULL, 0);
 673
 674                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 675                         continue;
 676
 677                 if (!(o->flags & SEEN)) {
 678                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 679
 680                         mark_common((struct commit *)o, 1, 1);
 681                 }
 682         }
 683
 684         filter_refs(args, refs, sought, nr_sought);
 685
 686         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 687                 const unsigned char *remote = ref->old_oid.hash;
 688                 struct object *o;
 689
 690                 o = lookup_object(remote);
 691                 if (!o || !(o->flags & COMPLETE)) {
 692                         retval = 0;
 693                         print_verbose(args, "want %s (%s)", sha1_to_hex(remote),
 694                                       ref->name);
 695                         continue;
 696                 }
 697                 print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote),
 698                               ref->name);
 699         }
 700         return retval;
 701 }
 702
 703 static int sideband_demux(int in, int out, void *data)
 704 {
 705         int *xd = data;
 706         int ret;
 707
 708         ret = recv_sideband("fetch-pack", xd[0], out);
 709         close(out);
 710         return ret;
 711 }
 712
 713 static int get_pack(struct fetch_pack_args *args,
 714                     int xd[2], char **pack_lockfile)
 715 {
 716         struct async demux;
 717         int do_keep = args->keep_pack;
 718         const char *cmd_name;
 719         struct pack_header header;
 720         int pass_header = 0;
 721         struct child_process cmd = CHILD_PROCESS_INIT;
 722         int ret;
 723
 724         memset(&demux, 0, sizeof(demux));
 725         if (use_sideband) {
 726                 /* xd[] is talking with upload-pack; subprocess reads from
 727                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 728                  * through demux->out.
 729                  */
 730                 demux.proc = sideband_demux;
 731                 demux.data = xd;
 732                 demux.out = -1;
 733                 demux.isolate_sigpipe = 1;
 734                 if (start_async(&demux))
 735                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 736         }
 737         else
 738                 demux.out = xd[0];
 739
 740         if (!args->keep_pack && unpack_limit) {
 741
 742                 if (read_pack_header(demux.out, &header))
 743                         die(_("protocol error: bad pack header"));
 744                 pass_header = 1;
 745                 if (ntohl(header.hdr_entries) < unpack_limit)
 746                         do_keep = 0;
 747                 else
 748                         do_keep = 1;
 749         }
 750
 751         if (alternate_shallow_file) {
 752                 argv_array_push(&cmd.args, "--shallow-file");
 753                 argv_array_push(&cmd.args, alternate_shallow_file);
 754         }
 755
 756         if (do_keep) {
 757                 if (pack_lockfile)
 758                         cmd.out = -1;
 759                 cmd_name = "index-pack";
 760                 argv_array_push(&cmd.args, cmd_name);
 761                 argv_array_push(&cmd.args, "--stdin");
 762                 if (!args->quiet && !args->no_progress)
 763                         argv_array_push(&cmd.args, "-v");
 764                 if (args->use_thin_pack)
 765                         argv_array_push(&cmd.args, "--fix-thin");
 766                 if (args->lock_pack || unpack_limit) {
 767                         char hostname[256];
 768                         if (gethostname(hostname, sizeof(hostname)))
 769                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 770                         argv_array_pushf(&cmd.args,
 771                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 772                                         (uintmax_t)getpid(), hostname);
 773                 }
 774                 if (args->check_self_contained_and_connected)
 775                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 776         }
 777         else {
 778                 cmd_name = "unpack-objects";
 779                 argv_array_push(&cmd.args, cmd_name);
 780                 if (args->quiet || args->no_progress)
 781                         argv_array_push(&cmd.args, "-q");
 782                 args->check_self_contained_and_connected = 0;
 783         }
 784
 785         if (pass_header)
 786                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 787                                  ntohl(header.hdr_version),
 788                                  ntohl(header.hdr_entries));
 789         if (fetch_fsck_objects >= 0
 790             ? fetch_fsck_objects
 791             : transfer_fsck_objects >= 0
 792             ? transfer_fsck_objects
 793             : 0)
 794                 argv_array_push(&cmd.args, "--strict");
 795
 796         cmd.in = demux.out;
 797         cmd.git_cmd = 1;
 798         if (start_command(&cmd))
 799                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 800         if (do_keep && pack_lockfile) {
 801                 *pack_lockfile = index_pack_lockfile(cmd.out);
 802                 close(cmd.out);
 803         }
 804
 805         if (!use_sideband)
 806                 /* Closed by start_command() */
 807                 xd[0] = -1;
 808
 809         ret = finish_command(&cmd);
 810         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 811                 args->self_contained_and_connected =
 812                         args->check_self_contained_and_connected &&
 813                         ret == 0;
 814         else
 815                 die(_("%s failed"), cmd_name);
 816         if (use_sideband && finish_async(&demux))
 817                 die(_("error in sideband demultiplexer"));
 818         return 0;
 819 }
 820
 821 static int cmp_ref_by_name(const void *a_, const void *b_)
 822 {
 823         const struct ref *a = *((const struct ref **)a_);
 824         const struct ref *b = *((const struct ref **)b_);
 825         return strcmp(a->name, b->name);
 826 }
 827
 828 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 829                                  int fd[2],
 830                                  const struct ref *orig_ref,
 831                                  struct ref **sought, int nr_sought,
 832                                  struct shallow_info *si,
 833                                  char **pack_lockfile)
 834 {
 835         struct ref *ref = copy_ref_list(orig_ref);
 836         unsigned char sha1[20];
 837         const char *agent_feature;
 838         int agent_len;
 839
 840         sort_ref_list(&ref, ref_compare_name);
 841         QSORT(sought, nr_sought, cmp_ref_by_name);
 842
 843         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 844                 die(_("Server does not support shallow clients"));
 845         if (args->depth > 0 || args->deepen_since || args->deepen_not)
 846                 args->deepen = 1;
 847         if (server_supports("multi_ack_detailed")) {
 848                 print_verbose(args, _("Server supports multi_ack_detailed"));
 849                 multi_ack = 2;
 850                 if (server_supports("no-done")) {
 851                         print_verbose(args, _("Server supports no-done"));
 852                         if (args->stateless_rpc)
 853                                 no_done = 1;
 854                 }
 855         }
 856         else if (server_supports("multi_ack")) {
 857                 print_verbose(args, _("Server supports multi_ack"));
 858                 multi_ack = 1;
 859         }
 860         if (server_supports("side-band-64k")) {
 861                 print_verbose(args, _("Server supports side-band-64k"));
 862                 use_sideband = 2;
 863         }
 864         else if (server_supports("side-band")) {
 865                 print_verbose(args, _("Server supports side-band"));
 866                 use_sideband = 1;
 867         }
 868         if (server_supports("allow-tip-sha1-in-want")) {
 869                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
 870                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 871         }
 872         if (server_supports("allow-reachable-sha1-in-want")) {
 873                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
 874                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 875         }
 876         if (!server_supports("thin-pack"))
 877                 args->use_thin_pack = 0;
 878         if (!server_supports("no-progress"))
 879                 args->no_progress = 0;
 880         if (!server_supports("include-tag"))
 881                 args->include_tag = 0;
 882         if (server_supports("ofs-delta"))
 883                 print_verbose(args, _("Server supports ofs-delta"));
 884         else
 885                 prefer_ofs_delta = 0;
 886
 887         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 888                 agent_supported = 1;
 889                 if (agent_len)
 890                         print_verbose(args, _("Server version is %.*s"),
 891                                       agent_len, agent_feature);
 892         }
 893         if (server_supports("deepen-since"))
 894                 deepen_since_ok = 1;
 895         else if (args->deepen_since)
 896                 die(_("Server does not support --shallow-since"));
 897         if (server_supports("deepen-not"))
 898                 deepen_not_ok = 1;
 899         else if (args->deepen_not)
 900                 die(_("Server does not support --shallow-exclude"));
 901         if (!server_supports("deepen-relative") && args->deepen_relative)
 902                 die(_("Server does not support --deepen"));
 903
 904         if (everything_local(args, &ref, sought, nr_sought)) {
 905                 packet_flush(fd[1]);
 906                 goto all_done;
 907         }
 908         if (find_common(args, fd, sha1, ref) < 0)
 909                 if (!args->keep_pack)
 910                         /* When cloning, it is not unusual to have
 911                          * no common commit.
 912                          */
 913                         warning(_("no common commits"));
 914
 915         if (args->stateless_rpc)
 916                 packet_flush(fd[1]);
 917         if (args->deepen)
 918                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 919                                         NULL);
 920         else if (si->nr_ours || si->nr_theirs)
 921                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 922         else
 923                 alternate_shallow_file = NULL;
 924         if (get_pack(args, fd, pack_lockfile))
 925                 die(_("git fetch-pack: fetch failed."));
 926
 927  all_done:
 928         return ref;
 929 }
 930
 931 static void fetch_pack_config(void)
 932 {
 933         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 934         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 935         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 936         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 937         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 938
 939         git_config(git_default_config, NULL);
 940 }
 941
 942 static void fetch_pack_setup(void)
 943 {
 944         static int did_setup;
 945         if (did_setup)
 946                 return;
 947         fetch_pack_config();
 948         if (0 <= transfer_unpack_limit)
 949                 unpack_limit = transfer_unpack_limit;
 950         else if (0 <= fetch_unpack_limit)
 951                 unpack_limit = fetch_unpack_limit;
 952         did_setup = 1;
 953 }
 954
 955 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 956 {
 957         struct string_list names = STRING_LIST_INIT_NODUP;
 958         int src, dst;
 959
 960         for (src = dst = 0; src < nr; src++) {
 961                 struct string_list_item *item;
 962                 item = string_list_insert(&names, ref[src]->name);
 963                 if (item->util)
 964                         continue; /* already have it */
 965                 item->util = ref[src];
 966                 if (src != dst)
 967                         ref[dst] = ref[src];
 968                 dst++;
 969         }
 970         for (src = dst; src < nr; src++)
 971                 ref[src] = NULL;
 972         string_list_clear(&names, 0);
 973         return dst;
 974 }
 975
 976 static void update_shallow(struct fetch_pack_args *args,
 977                            struct ref **sought, int nr_sought,
 978                            struct shallow_info *si)
 979 {
 980         struct sha1_array ref = SHA1_ARRAY_INIT;
 981         int *status;
 982         int i;
 983
 984         if (args->deepen && alternate_shallow_file) {
 985                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 986                         unlink_or_warn(git_path_shallow());
 987                         rollback_lock_file(&shallow_lock);
 988                 } else
 989                         commit_lock_file(&shallow_lock);
 990                 return;
 991         }
 992
 993         if (!si->shallow || !si->shallow->nr)
 994                 return;
 995
 996         if (args->cloning) {
 997                 /*
 998                  * remote is shallow, but this is a clone, there are
 999                  * no objects in repo to worry about. Accept any
1000                  * shallow points that exist in the pack (iow in repo
1001                  * after get_pack() and reprepare_packed_git())
1002                  */
1003                 struct sha1_array extra = SHA1_ARRAY_INIT;
1004                 unsigned char (*sha1)[20] = si->shallow->sha1;
1005                 for (i = 0; i < si->shallow->nr; i++)
1006                         if (has_sha1_file(sha1[i]))
1007                                 sha1_array_append(&extra, sha1[i]);
1008                 if (extra.nr) {
1009                         setup_alternate_shallow(&shallow_lock,
1010                                                 &alternate_shallow_file,
1011                                                 &extra);
1012                         commit_lock_file(&shallow_lock);
1013                 }
1014                 sha1_array_clear(&extra);
1015                 return;
1016         }
1017
1018         if (!si->nr_ours && !si->nr_theirs)
1019                 return;
1020
1021         remove_nonexistent_theirs_shallow(si);
1022         if (!si->nr_ours && !si->nr_theirs)
1023                 return;
1024         for (i = 0; i < nr_sought; i++)
1025                 sha1_array_append(&ref, sought[i]->old_oid.hash);
1026         si->ref = &ref;
1027
1028         if (args->update_shallow) {
1029                 /*
1030                  * remote is also shallow, .git/shallow may be updated
1031                  * so all refs can be accepted. Make sure we only add
1032                  * shallow roots that are actually reachable from new
1033                  * refs.
1034                  */
1035                 struct sha1_array extra = SHA1_ARRAY_INIT;
1036                 unsigned char (*sha1)[20] = si->shallow->sha1;
1037                 assign_shallow_commits_to_refs(si, NULL, NULL);
1038                 if (!si->nr_ours && !si->nr_theirs) {
1039                         sha1_array_clear(&ref);
1040                         return;
1041                 }
1042                 for (i = 0; i < si->nr_ours; i++)
1043                         sha1_array_append(&extra, sha1[si->ours[i]]);
1044                 for (i = 0; i < si->nr_theirs; i++)
1045                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1046                 setup_alternate_shallow(&shallow_lock,
1047                                         &alternate_shallow_file,
1048                                         &extra);
1049                 commit_lock_file(&shallow_lock);
1050                 sha1_array_clear(&extra);
1051                 sha1_array_clear(&ref);
1052                 return;
1053         }
1054
1055         /*
1056          * remote is also shallow, check what ref is safe to update
1057          * without updating .git/shallow
1058          */
1059         status = xcalloc(nr_sought, sizeof(*status));
1060         assign_shallow_commits_to_refs(si, NULL, status);
1061         if (si->nr_ours || si->nr_theirs) {
1062                 for (i = 0; i < nr_sought; i++)
1063                         if (status[i])
1064                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1065         }
1066         free(status);
1067         sha1_array_clear(&ref);
1068 }
1069
1070 struct ref *fetch_pack(struct fetch_pack_args *args,
1071                        int fd[], struct child_process *conn,
1072                        const struct ref *ref,
1073                        const char *dest,
1074                        struct ref **sought, int nr_sought,
1075                        struct sha1_array *shallow,
1076                        char **pack_lockfile)
1077 {
1078         struct ref *ref_cpy;
1079         struct shallow_info si;
1080
1081         fetch_pack_setup();
1082         if (nr_sought)
1083                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1084
1085         if (!ref) {
1086                 packet_flush(fd[1]);
1087                 die(_("no matching remote head"));
1088         }
1089         prepare_shallow_info(&si, shallow);
1090         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1091                                 &si, pack_lockfile);
1092         reprepare_packed_git();
1093         update_shallow(args, sought, nr_sought, &si);
1094         clear_shallow_info(&si);
1095         return ref_cpy;
1096 }