builtin/fetch-pack.c

   1 #include "builtin.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "transport.h"
  13 #include "version.h"
  14
  15 static int transfer_unpack_limit = -1;
  16 static int fetch_unpack_limit = -1;
  17 static int unpack_limit = 100;
  18 static int prefer_ofs_delta = 1;
  19 static int no_done;
  20 static int fetch_fsck_objects = -1;
  21 static int transfer_fsck_objects = -1;
  22 static struct fetch_pack_args args = {
  23         /* .uploadpack = */ "git-upload-pack",
  24 };
  25
  26 static const char fetch_pack_usage[] =
  27 "git fetch-pack [--all] [--stdin] [--quiet|-q] [--keep|-k] [--thin] "
  28 "[--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] "
  29 "[--no-progress] [-v] [<host>:]<directory> [<refs>...]";
  30
  31 #define COMPLETE        (1U << 0)
  32 #define COMMON          (1U << 1)
  33 #define COMMON_REF      (1U << 2)
  34 #define SEEN            (1U << 3)
  35 #define POPPED          (1U << 4)
  36
  37 static int marked;
  38
  39 /*
  40  * After sending this many "have"s if we do not get any new ACK , we
  41  * give up traversing our history.
  42  */
  43 #define MAX_IN_VAIN 256
  44
  45 static struct commit_list *rev_list;
  46 static int non_common_revs, multi_ack, use_sideband;
  47
  48 static void rev_list_push(struct commit *commit, int mark)
  49 {
  50         if (!(commit->object.flags & mark)) {
  51                 commit->object.flags |= mark;
  52
  53                 if (!(commit->object.parsed))
  54                         if (parse_commit(commit))
  55                                 return;
  56
  57                 commit_list_insert_by_date(commit, &rev_list);
  58
  59                 if (!(commit->object.flags & COMMON))
  60                         non_common_revs++;
  61         }
  62 }
  63
  64 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  65 {
  66         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  67
  68         if (o && o->type == OBJ_COMMIT)
  69                 rev_list_push((struct commit *)o, SEEN);
  70
  71         return 0;
  72 }
  73
  74 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  75 {
  76         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  77
  78         if (o && o->type == OBJ_COMMIT)
  79                 clear_commit_marks((struct commit *)o,
  80                                    COMMON | COMMON_REF | SEEN | POPPED);
  81         return 0;
  82 }
  83
  84 /*
  85    This function marks a rev and its ancestors as common.
  86    In some cases, it is desirable to mark only the ancestors (for example
  87    when only the server does not yet know that they are common).
  88 */
  89
  90 static void mark_common(struct commit *commit,
  91                 int ancestors_only, int dont_parse)
  92 {
  93         if (commit != NULL && !(commit->object.flags & COMMON)) {
  94                 struct object *o = (struct object *)commit;
  95
  96                 if (!ancestors_only)
  97                         o->flags |= COMMON;
  98
  99                 if (!(o->flags & SEEN))
 100                         rev_list_push(commit, SEEN);
 101                 else {
 102                         struct commit_list *parents;
 103
 104                         if (!ancestors_only && !(o->flags & POPPED))
 105                                 non_common_revs--;
 106                         if (!o->parsed && !dont_parse)
 107                                 if (parse_commit(commit))
 108                                         return;
 109
 110                         for (parents = commit->parents;
 111                                         parents;
 112                                         parents = parents->next)
 113                                 mark_common(parents->item, 0, dont_parse);
 114                 }
 115         }
 116 }
 117
 118 /*
 119   Get the next rev to send, ignoring the common.
 120 */
 121
 122 static const unsigned char *get_rev(void)
 123 {
 124         struct commit *commit = NULL;
 125
 126         while (commit == NULL) {
 127                 unsigned int mark;
 128                 struct commit_list *parents;
 129
 130                 if (rev_list == NULL || non_common_revs == 0)
 131                         return NULL;
 132
 133                 commit = rev_list->item;
 134                 if (!commit->object.parsed)
 135                         parse_commit(commit);
 136                 parents = commit->parents;
 137
 138                 commit->object.flags |= POPPED;
 139                 if (!(commit->object.flags & COMMON))
 140                         non_common_revs--;
 141
 142                 if (commit->object.flags & COMMON) {
 143                         /* do not send "have", and ignore ancestors */
 144                         commit = NULL;
 145                         mark = COMMON | SEEN;
 146                 } else if (commit->object.flags & COMMON_REF)
 147                         /* send "have", and ignore ancestors */
 148                         mark = COMMON | SEEN;
 149                 else
 150                         /* send "have", also for its ancestors */
 151                         mark = SEEN;
 152
 153                 while (parents) {
 154                         if (!(parents->item->object.flags & SEEN))
 155                                 rev_list_push(parents->item, mark);
 156                         if (mark & COMMON)
 157                                 mark_common(parents->item, 1, 0);
 158                         parents = parents->next;
 159                 }
 160
 161                 rev_list = rev_list->next;
 162         }
 163
 164         return commit->object.sha1;
 165 }
 166
 167 enum ack_type {
 168         NAK = 0,
 169         ACK,
 170         ACK_continue,
 171         ACK_common,
 172         ACK_ready
 173 };
 174
 175 static void consume_shallow_list(int fd)
 176 {
 177         if (args.stateless_rpc && args.depth > 0) {
 178                 /* If we sent a depth we will get back "duplicate"
 179                  * shallow and unshallow commands every time there
 180                  * is a block of have lines exchanged.
 181                  */
 182                 char line[1000];
 183                 while (packet_read_line(fd, line, sizeof(line))) {
 184                         if (!prefixcmp(line, "shallow "))
 185                                 continue;
 186                         if (!prefixcmp(line, "unshallow "))
 187                                 continue;
 188                         die("git fetch-pack: expected shallow list");
 189                 }
 190         }
 191 }
 192
 193 struct write_shallow_data {
 194         struct strbuf *out;
 195         int use_pack_protocol;
 196         int count;
 197 };
 198
 199 static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
 200 {
 201         struct write_shallow_data *data = cb_data;
 202         const char *hex = sha1_to_hex(graft->sha1);
 203         data->count++;
 204         if (data->use_pack_protocol)
 205                 packet_buf_write(data->out, "shallow %s", hex);
 206         else {
 207                 strbuf_addstr(data->out, hex);
 208                 strbuf_addch(data->out, '\n');
 209         }
 210         return 0;
 211 }
 212
 213 static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
 214 {
 215         struct write_shallow_data data;
 216         data.out = out;
 217         data.use_pack_protocol = use_pack_protocol;
 218         data.count = 0;
 219         for_each_commit_graft(write_one_shallow, &data);
 220         return data.count;
 221 }
 222
 223 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 224 {
 225         static char line[1000];
 226         int len = packet_read_line(fd, line, sizeof(line));
 227
 228         if (!len)
 229                 die("git fetch-pack: expected ACK/NAK, got EOF");
 230         if (line[len-1] == '\n')
 231                 line[--len] = 0;
 232         if (!strcmp(line, "NAK"))
 233                 return NAK;
 234         if (!prefixcmp(line, "ACK ")) {
 235                 if (!get_sha1_hex(line+4, result_sha1)) {
 236                         if (strstr(line+45, "continue"))
 237                                 return ACK_continue;
 238                         if (strstr(line+45, "common"))
 239                                 return ACK_common;
 240                         if (strstr(line+45, "ready"))
 241                                 return ACK_ready;
 242                         return ACK;
 243                 }
 244         }
 245         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 246 }
 247
 248 static void send_request(int fd, struct strbuf *buf)
 249 {
 250         if (args.stateless_rpc) {
 251                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 252                 packet_flush(fd);
 253         } else
 254                 safe_write(fd, buf->buf, buf->len);
 255 }
 256
 257 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 258 {
 259         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 260 }
 261
 262 #define INITIAL_FLUSH 16
 263 #define PIPESAFE_FLUSH 32
 264 #define LARGE_FLUSH 1024
 265
 266 static int next_flush(int count)
 267 {
 268         int flush_limit = args.stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 269
 270         if (count < flush_limit)
 271                 count <<= 1;
 272         else
 273                 count += flush_limit;
 274         return count;
 275 }
 276
 277 static int find_common(int fd[2], unsigned char *result_sha1,
 278                        struct ref *refs)
 279 {
 280         int fetching;
 281         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 282         const unsigned char *sha1;
 283         unsigned in_vain = 0;
 284         int got_continue = 0;
 285         int got_ready = 0;
 286         struct strbuf req_buf = STRBUF_INIT;
 287         size_t state_len = 0;
 288
 289         if (args.stateless_rpc && multi_ack == 1)
 290                 die("--stateless-rpc requires multi_ack_detailed");
 291         if (marked)
 292                 for_each_ref(clear_marks, NULL);
 293         marked = 1;
 294
 295         for_each_ref(rev_list_insert_ref, NULL);
 296         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 297
 298         fetching = 0;
 299         for ( ; refs ; refs = refs->next) {
 300                 unsigned char *remote = refs->old_sha1;
 301                 const char *remote_hex;
 302                 struct object *o;
 303
 304                 /*
 305                  * If that object is complete (i.e. it is an ancestor of a
 306                  * local ref), we tell them we have it but do not have to
 307                  * tell them about its ancestors, which they already know
 308                  * about.
 309                  *
 310                  * We use lookup_object here because we are only
 311                  * interested in the case we *know* the object is
 312                  * reachable and we have already scanned it.
 313                  */
 314                 if (((o = lookup_object(remote)) != NULL) &&
 315                                 (o->flags & COMPLETE)) {
 316                         continue;
 317                 }
 318
 319                 remote_hex = sha1_to_hex(remote);
 320                 if (!fetching) {
 321                         struct strbuf c = STRBUF_INIT;
 322                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 323                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 324                         if (no_done)            strbuf_addstr(&c, " no-done");
 325                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 326                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 327                         if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack");
 328                         if (args.no_progress)   strbuf_addstr(&c, " no-progress");
 329                         if (args.include_tag)   strbuf_addstr(&c, " include-tag");
 330                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 331                         strbuf_addf(&c, " agent=%s", git_user_agent_sanitized());
 332                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 333                         strbuf_release(&c);
 334                 } else
 335                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 336                 fetching++;
 337         }
 338
 339         if (!fetching) {
 340                 strbuf_release(&req_buf);
 341                 packet_flush(fd[1]);
 342                 return 1;
 343         }
 344
 345         if (is_repository_shallow())
 346                 write_shallow_commits(&req_buf, 1);
 347         if (args.depth > 0)
 348                 packet_buf_write(&req_buf, "deepen %d", args.depth);
 349         packet_buf_flush(&req_buf);
 350         state_len = req_buf.len;
 351
 352         if (args.depth > 0) {
 353                 char line[1024];
 354                 unsigned char sha1[20];
 355
 356                 send_request(fd[1], &req_buf);
 357                 while (packet_read_line(fd[0], line, sizeof(line))) {
 358                         if (!prefixcmp(line, "shallow ")) {
 359                                 if (get_sha1_hex(line + 8, sha1))
 360                                         die("invalid shallow line: %s", line);
 361                                 register_shallow(sha1);
 362                                 continue;
 363                         }
 364                         if (!prefixcmp(line, "unshallow ")) {
 365                                 if (get_sha1_hex(line + 10, sha1))
 366                                         die("invalid unshallow line: %s", line);
 367                                 if (!lookup_object(sha1))
 368                                         die("object not found: %s", line);
 369                                 /* make sure that it is parsed as shallow */
 370                                 if (!parse_object(sha1))
 371                                         die("error in object: %s", line);
 372                                 if (unregister_shallow(sha1))
 373                                         die("no shallow found: %s", line);
 374                                 continue;
 375                         }
 376                         die("expected shallow/unshallow, got %s", line);
 377                 }
 378         } else if (!args.stateless_rpc)
 379                 send_request(fd[1], &req_buf);
 380
 381         if (!args.stateless_rpc) {
 382                 /* If we aren't using the stateless-rpc interface
 383                  * we don't need to retain the headers.
 384                  */
 385                 strbuf_setlen(&req_buf, 0);
 386                 state_len = 0;
 387         }
 388
 389         flushes = 0;
 390         retval = -1;
 391         while ((sha1 = get_rev())) {
 392                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 393                 if (args.verbose)
 394                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 395                 in_vain++;
 396                 if (flush_at <= ++count) {
 397                         int ack;
 398
 399                         packet_buf_flush(&req_buf);
 400                         send_request(fd[1], &req_buf);
 401                         strbuf_setlen(&req_buf, state_len);
 402                         flushes++;
 403                         flush_at = next_flush(count);
 404
 405                         /*
 406                          * We keep one window "ahead" of the other side, and
 407                          * will wait for an ACK only on the next one
 408                          */
 409                         if (!args.stateless_rpc && count == INITIAL_FLUSH)
 410                                 continue;
 411
 412                         consume_shallow_list(fd[0]);
 413                         do {
 414                                 ack = get_ack(fd[0], result_sha1);
 415                                 if (args.verbose && ack)
 416                                         fprintf(stderr, "got ack %d %s\n", ack,
 417                                                         sha1_to_hex(result_sha1));
 418                                 switch (ack) {
 419                                 case ACK:
 420                                         flushes = 0;
 421                                         multi_ack = 0;
 422                                         retval = 0;
 423                                         goto done;
 424                                 case ACK_common:
 425                                 case ACK_ready:
 426                                 case ACK_continue: {
 427                                         struct commit *commit =
 428                                                 lookup_commit(result_sha1);
 429                                         if (!commit)
 430                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 431                                         if (args.stateless_rpc
 432                                          && ack == ACK_common
 433                                          && !(commit->object.flags & COMMON)) {
 434                                                 /* We need to replay the have for this object
 435                                                  * on the next RPC request so the peer knows
 436                                                  * it is in common with us.
 437                                                  */
 438                                                 const char *hex = sha1_to_hex(result_sha1);
 439                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 440                                                 state_len = req_buf.len;
 441                                         }
 442                                         mark_common(commit, 0, 1);
 443                                         retval = 0;
 444                                         in_vain = 0;
 445                                         got_continue = 1;
 446                                         if (ack == ACK_ready) {
 447                                                 rev_list = NULL;
 448                                                 got_ready = 1;
 449                                         }
 450                                         break;
 451                                         }
 452                                 }
 453                         } while (ack);
 454                         flushes--;
 455                         if (got_continue && MAX_IN_VAIN < in_vain) {
 456                                 if (args.verbose)
 457                                         fprintf(stderr, "giving up\n");
 458                                 break; /* give up */
 459                         }
 460                 }
 461         }
 462 done:
 463         if (!got_ready || !no_done) {
 464                 packet_buf_write(&req_buf, "done\n");
 465                 send_request(fd[1], &req_buf);
 466         }
 467         if (args.verbose)
 468                 fprintf(stderr, "done\n");
 469         if (retval != 0) {
 470                 multi_ack = 0;
 471                 flushes++;
 472         }
 473         strbuf_release(&req_buf);
 474
 475         consume_shallow_list(fd[0]);
 476         while (flushes || multi_ack) {
 477                 int ack = get_ack(fd[0], result_sha1);
 478                 if (ack) {
 479                         if (args.verbose)
 480                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 481                                         sha1_to_hex(result_sha1));
 482                         if (ack == ACK)
 483                                 return 0;
 484                         multi_ack = 1;
 485                         continue;
 486                 }
 487                 flushes--;
 488         }
 489         /* it is no error to fetch into a completely empty repo */
 490         return count ? retval : 0;
 491 }
 492
 493 static struct commit_list *complete;
 494
 495 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 496 {
 497         struct object *o = parse_object(sha1);
 498
 499         while (o && o->type == OBJ_TAG) {
 500                 struct tag *t = (struct tag *) o;
 501                 if (!t->tagged)
 502                         break; /* broken repository */
 503                 o->flags |= COMPLETE;
 504                 o = parse_object(t->tagged->sha1);
 505         }
 506         if (o && o->type == OBJ_COMMIT) {
 507                 struct commit *commit = (struct commit *)o;
 508                 if (!(commit->object.flags & COMPLETE)) {
 509                         commit->object.flags |= COMPLETE;
 510                         commit_list_insert_by_date(commit, &complete);
 511                 }
 512         }
 513         return 0;
 514 }
 515
 516 static void mark_recent_complete_commits(unsigned long cutoff)
 517 {
 518         while (complete && cutoff <= complete->item->date) {
 519                 if (args.verbose)
 520                         fprintf(stderr, "Marking %s as complete\n",
 521                                 sha1_to_hex(complete->item->object.sha1));
 522                 pop_most_recent_commit(&complete, COMPLETE);
 523         }
 524 }
 525
 526 static void filter_refs(struct ref **refs, int nr_match, char **match)
 527 {
 528         struct ref **return_refs;
 529         struct ref *newlist = NULL;
 530         struct ref **newtail = &newlist;
 531         struct ref *ref, *next;
 532         struct ref *fastarray[32];
 533         int match_pos;
 534
 535         if (nr_match && !args.fetch_all) {
 536                 if (ARRAY_SIZE(fastarray) < nr_match)
 537                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 538                 else {
 539                         return_refs = fastarray;
 540                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 541                 }
 542         }
 543         else
 544                 return_refs = NULL;
 545
 546         match_pos = 0;
 547         for (ref = *refs; ref; ref = next) {
 548                 next = ref->next;
 549                 if (!memcmp(ref->name, "refs/", 5) &&
 550                     check_refname_format(ref->name + 5, 0))
 551                         ; /* trash */
 552                 else if (args.fetch_all &&
 553                          (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
 554                         *newtail = ref;
 555                         ref->next = NULL;
 556                         newtail = &ref->next;
 557                         continue;
 558                 }
 559                 else {
 560                         int cmp = -1;
 561                         while (match_pos < nr_match) {
 562                                 cmp = strcmp(ref->name, match[match_pos]);
 563                                 if (cmp < 0) /* definitely do not have it */
 564                                         break;
 565                                 else if (cmp == 0) { /* definitely have it */
 566                                         match[match_pos][0] = '\0';
 567                                         return_refs[match_pos] = ref;
 568                                         break;
 569                                 }
 570                                 else /* might have it; keep looking */
 571                                         match_pos++;
 572                         }
 573                         if (!cmp)
 574                                 continue; /* we will link it later */
 575                 }
 576                 free(ref);
 577         }
 578
 579         if (!args.fetch_all) {
 580                 int i;
 581                 for (i = 0; i < nr_match; i++) {
 582                         ref = return_refs[i];
 583                         if (ref) {
 584                                 *newtail = ref;
 585                                 ref->next = NULL;
 586                                 newtail = &ref->next;
 587                         }
 588                 }
 589                 if (return_refs != fastarray)
 590                         free(return_refs);
 591         }
 592         *refs = newlist;
 593 }
 594
 595 static void mark_alternate_complete(const struct ref *ref, void *unused)
 596 {
 597         mark_complete(NULL, ref->old_sha1, 0, NULL);
 598 }
 599
 600 static int everything_local(struct ref **refs, int nr_match, char **match)
 601 {
 602         struct ref *ref;
 603         int retval;
 604         unsigned long cutoff = 0;
 605
 606         save_commit_buffer = 0;
 607
 608         for (ref = *refs; ref; ref = ref->next) {
 609                 struct object *o;
 610
 611                 o = parse_object(ref->old_sha1);
 612                 if (!o)
 613                         continue;
 614
 615                 /* We already have it -- which may mean that we were
 616                  * in sync with the other side at some time after
 617                  * that (it is OK if we guess wrong here).
 618                  */
 619                 if (o->type == OBJ_COMMIT) {
 620                         struct commit *commit = (struct commit *)o;
 621                         if (!cutoff || cutoff < commit->date)
 622                                 cutoff = commit->date;
 623                 }
 624         }
 625
 626         if (!args.depth) {
 627                 for_each_ref(mark_complete, NULL);
 628                 for_each_alternate_ref(mark_alternate_complete, NULL);
 629                 if (cutoff)
 630                         mark_recent_complete_commits(cutoff);
 631         }
 632
 633         /*
 634          * Mark all complete remote refs as common refs.
 635          * Don't mark them common yet; the server has to be told so first.
 636          */
 637         for (ref = *refs; ref; ref = ref->next) {
 638                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 639                                              NULL, 0);
 640
 641                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 642                         continue;
 643
 644                 if (!(o->flags & SEEN)) {
 645                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 646
 647                         mark_common((struct commit *)o, 1, 1);
 648                 }
 649         }
 650
 651         filter_refs(refs, nr_match, match);
 652
 653         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 654                 const unsigned char *remote = ref->old_sha1;
 655                 unsigned char local[20];
 656                 struct object *o;
 657
 658                 o = lookup_object(remote);
 659                 if (!o || !(o->flags & COMPLETE)) {
 660                         retval = 0;
 661                         if (!args.verbose)
 662                                 continue;
 663                         fprintf(stderr,
 664                                 "want %s (%s)\n", sha1_to_hex(remote),
 665                                 ref->name);
 666                         continue;
 667                 }
 668
 669                 hashcpy(ref->new_sha1, local);
 670                 if (!args.verbose)
 671                         continue;
 672                 fprintf(stderr,
 673                         "already have %s (%s)\n", sha1_to_hex(remote),
 674                         ref->name);
 675         }
 676         return retval;
 677 }
 678
 679 static int sideband_demux(int in, int out, void *data)
 680 {
 681         int *xd = data;
 682
 683         int ret = recv_sideband("fetch-pack", xd[0], out);
 684         close(out);
 685         return ret;
 686 }
 687
 688 static int get_pack(int xd[2], char **pack_lockfile)
 689 {
 690         struct async demux;
 691         const char *argv[20];
 692         char keep_arg[256];
 693         char hdr_arg[256];
 694         const char **av;
 695         int do_keep = args.keep_pack;
 696         struct child_process cmd;
 697
 698         memset(&demux, 0, sizeof(demux));
 699         if (use_sideband) {
 700                 /* xd[] is talking with upload-pack; subprocess reads from
 701                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 702                  * through demux->out.
 703                  */
 704                 demux.proc = sideband_demux;
 705                 demux.data = xd;
 706                 demux.out = -1;
 707                 if (start_async(&demux))
 708                         die("fetch-pack: unable to fork off sideband"
 709                             " demultiplexer");
 710         }
 711         else
 712                 demux.out = xd[0];
 713
 714         memset(&cmd, 0, sizeof(cmd));
 715         cmd.argv = argv;
 716         av = argv;
 717         *hdr_arg = 0;
 718         if (!args.keep_pack && unpack_limit) {
 719                 struct pack_header header;
 720
 721                 if (read_pack_header(demux.out, &header))
 722                         die("protocol error: bad pack header");
 723                 snprintf(hdr_arg, sizeof(hdr_arg),
 724                          "--pack_header=%"PRIu32",%"PRIu32,
 725                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 726                 if (ntohl(header.hdr_entries) < unpack_limit)
 727                         do_keep = 0;
 728                 else
 729                         do_keep = 1;
 730         }
 731
 732         if (do_keep) {
 733                 if (pack_lockfile)
 734                         cmd.out = -1;
 735                 *av++ = "index-pack";
 736                 *av++ = "--stdin";
 737                 if (!args.quiet && !args.no_progress)
 738                         *av++ = "-v";
 739                 if (args.use_thin_pack)
 740                         *av++ = "--fix-thin";
 741                 if (args.lock_pack || unpack_limit) {
 742                         int s = sprintf(keep_arg,
 743                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 744                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 745                                 strcpy(keep_arg + s, "localhost");
 746                         *av++ = keep_arg;
 747                 }
 748         }
 749         else {
 750                 *av++ = "unpack-objects";
 751                 if (args.quiet || args.no_progress)
 752                         *av++ = "-q";
 753         }
 754         if (*hdr_arg)
 755                 *av++ = hdr_arg;
 756         if (fetch_fsck_objects >= 0
 757             ? fetch_fsck_objects
 758             : transfer_fsck_objects >= 0
 759             ? transfer_fsck_objects
 760             : 0)
 761                 *av++ = "--strict";
 762         *av++ = NULL;
 763
 764         cmd.in = demux.out;
 765         cmd.git_cmd = 1;
 766         if (start_command(&cmd))
 767                 die("fetch-pack: unable to fork off %s", argv[0]);
 768         if (do_keep && pack_lockfile) {
 769                 *pack_lockfile = index_pack_lockfile(cmd.out);
 770                 close(cmd.out);
 771         }
 772
 773         if (finish_command(&cmd))
 774                 die("%s failed", argv[0]);
 775         if (use_sideband && finish_async(&demux))
 776                 die("error in sideband demultiplexer");
 777         return 0;
 778 }
 779
 780 static struct ref *do_fetch_pack(int fd[2],
 781                 const struct ref *orig_ref,
 782                 int nr_match,
 783                 char **match,
 784                 char **pack_lockfile)
 785 {
 786         struct ref *ref = copy_ref_list(orig_ref);
 787         unsigned char sha1[20];
 788
 789         sort_ref_list(&ref, ref_compare_name);
 790
 791         if (is_repository_shallow() && !server_supports("shallow"))
 792                 die("Server does not support shallow clients");
 793         if (server_supports("multi_ack_detailed")) {
 794                 if (args.verbose)
 795                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 796                 multi_ack = 2;
 797                 if (server_supports("no-done")) {
 798                         if (args.verbose)
 799                                 fprintf(stderr, "Server supports no-done\n");
 800                         if (args.stateless_rpc)
 801                                 no_done = 1;
 802                 }
 803         }
 804         else if (server_supports("multi_ack")) {
 805                 if (args.verbose)
 806                         fprintf(stderr, "Server supports multi_ack\n");
 807                 multi_ack = 1;
 808         }
 809         if (server_supports("side-band-64k")) {
 810                 if (args.verbose)
 811                         fprintf(stderr, "Server supports side-band-64k\n");
 812                 use_sideband = 2;
 813         }
 814         else if (server_supports("side-band")) {
 815                 if (args.verbose)
 816                         fprintf(stderr, "Server supports side-band\n");
 817                 use_sideband = 1;
 818         }
 819         if (server_supports("ofs-delta")) {
 820                 if (args.verbose)
 821                         fprintf(stderr, "Server supports ofs-delta\n");
 822         } else
 823                 prefer_ofs_delta = 0;
 824         if (everything_local(&ref, nr_match, match)) {
 825                 packet_flush(fd[1]);
 826                 goto all_done;
 827         }
 828         if (find_common(fd, sha1, ref) < 0)
 829                 if (!args.keep_pack)
 830                         /* When cloning, it is not unusual to have
 831                          * no common commit.
 832                          */
 833                         warning("no common commits");
 834
 835         if (args.stateless_rpc)
 836                 packet_flush(fd[1]);
 837         if (get_pack(fd, pack_lockfile))
 838                 die("git fetch-pack: fetch failed.");
 839
 840  all_done:
 841         return ref;
 842 }
 843
 844 static int remove_duplicates(int nr_heads, char **heads)
 845 {
 846         int src, dst;
 847
 848         if (!nr_heads)
 849                 return 0;
 850
 851         for (src = dst = 1; src < nr_heads; src++)
 852                 if (strcmp(heads[src], heads[dst-1]))
 853                         heads[dst++] = heads[src];
 854         return dst;
 855 }
 856
 857 static int fetch_pack_config(const char *var, const char *value, void *cb)
 858 {
 859         if (strcmp(var, "fetch.unpacklimit") == 0) {
 860                 fetch_unpack_limit = git_config_int(var, value);
 861                 return 0;
 862         }
 863
 864         if (strcmp(var, "transfer.unpacklimit") == 0) {
 865                 transfer_unpack_limit = git_config_int(var, value);
 866                 return 0;
 867         }
 868
 869         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 870                 prefer_ofs_delta = git_config_bool(var, value);
 871                 return 0;
 872         }
 873
 874         if (!strcmp(var, "fetch.fsckobjects")) {
 875                 fetch_fsck_objects = git_config_bool(var, value);
 876                 return 0;
 877         }
 878
 879         if (!strcmp(var, "transfer.fsckobjects")) {
 880                 transfer_fsck_objects = git_config_bool(var, value);
 881                 return 0;
 882         }
 883
 884         return git_default_config(var, value, cb);
 885 }
 886
 887 static struct lock_file lock;
 888
 889 static void fetch_pack_setup(void)
 890 {
 891         static int did_setup;
 892         if (did_setup)
 893                 return;
 894         git_config(fetch_pack_config, NULL);
 895         if (0 <= transfer_unpack_limit)
 896                 unpack_limit = transfer_unpack_limit;
 897         else if (0 <= fetch_unpack_limit)
 898                 unpack_limit = fetch_unpack_limit;
 899         did_setup = 1;
 900 }
 901
 902 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 903 {
 904         int i, ret;
 905         struct ref *ref = NULL;
 906         const char *dest = NULL;
 907         int alloc_heads = 0, nr_heads = 0;
 908         char **heads = NULL;
 909         int fd[2];
 910         char *pack_lockfile = NULL;
 911         char **pack_lockfile_ptr = NULL;
 912         struct child_process *conn;
 913
 914         packet_trace_identity("fetch-pack");
 915
 916         for (i = 1; i < argc && *argv[i] == '-'; i++) {
 917                 const char *arg = argv[i];
 918
 919                 if (!prefixcmp(arg, "--upload-pack=")) {
 920                         args.uploadpack = arg + 14;
 921                         continue;
 922                 }
 923                 if (!prefixcmp(arg, "--exec=")) {
 924                         args.uploadpack = arg + 7;
 925                         continue;
 926                 }
 927                 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 928                         args.quiet = 1;
 929                         continue;
 930                 }
 931                 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 932                         args.lock_pack = args.keep_pack;
 933                         args.keep_pack = 1;
 934                         continue;
 935                 }
 936                 if (!strcmp("--thin", arg)) {
 937                         args.use_thin_pack = 1;
 938                         continue;
 939                 }
 940                 if (!strcmp("--include-tag", arg)) {
 941                         args.include_tag = 1;
 942                         continue;
 943                 }
 944                 if (!strcmp("--all", arg)) {
 945                         args.fetch_all = 1;
 946                         continue;
 947                 }
 948                 if (!strcmp("--stdin", arg)) {
 949                         args.stdin_refs = 1;
 950                         continue;
 951                 }
 952                 if (!strcmp("-v", arg)) {
 953                         args.verbose = 1;
 954                         continue;
 955                 }
 956                 if (!prefixcmp(arg, "--depth=")) {
 957                         args.depth = strtol(arg + 8, NULL, 0);
 958                         continue;
 959                 }
 960                 if (!strcmp("--no-progress", arg)) {
 961                         args.no_progress = 1;
 962                         continue;
 963                 }
 964                 if (!strcmp("--stateless-rpc", arg)) {
 965                         args.stateless_rpc = 1;
 966                         continue;
 967                 }
 968                 if (!strcmp("--lock-pack", arg)) {
 969                         args.lock_pack = 1;
 970                         pack_lockfile_ptr = &pack_lockfile;
 971                         continue;
 972                 }
 973                 usage(fetch_pack_usage);
 974         }
 975
 976         if (i < argc)
 977                 dest = argv[i++];
 978         else
 979                 usage(fetch_pack_usage);
 980
 981         /*
 982          * Copy refs from cmdline to growable list, then append any
 983          * refs from the standard input:
 984          */
 985         ALLOC_GROW(heads, argc - i, alloc_heads);
 986         for (; i < argc; i++)
 987                 heads[nr_heads++] = xstrdup(argv[i]);
 988         if (args.stdin_refs) {
 989                 if (args.stateless_rpc) {
 990                         /* in stateless RPC mode we use pkt-line to read
 991                          * from stdin, until we get a flush packet
 992                          */
 993                         static char line[1000];
 994                         for (;;) {
 995                                 int n = packet_read_line(0, line, sizeof(line));
 996                                 if (!n)
 997                                         break;
 998                                 if (line[n-1] == '\n')
 999                                         n--;
1000                                 ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
1001                                 heads[nr_heads++] = xmemdupz(line, n);
1002                         }
1003                 }
1004                 else {
1005                         /* read from stdin one ref per line, until EOF */
1006                         struct strbuf line = STRBUF_INIT;
1007                         while (strbuf_getline(&line, stdin, '\n') != EOF) {
1008                                 ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
1009                                 heads[nr_heads++] = strbuf_detach(&line, NULL);
1010                         }
1011                         strbuf_release(&line);
1012                 }
1013         }
1014
1015         if (args.stateless_rpc) {
1016                 conn = NULL;
1017                 fd[0] = 0;
1018                 fd[1] = 1;
1019         } else {
1020                 conn = git_connect(fd, dest, args.uploadpack,
1021                                    args.verbose ? CONNECT_VERBOSE : 0);
1022         }
1023
1024         get_remote_heads(fd[0], &ref, 0, NULL);
1025
1026         ref = fetch_pack(&args, fd, conn, ref, dest,
1027                 nr_heads, heads, pack_lockfile_ptr);
1028         if (pack_lockfile) {
1029                 printf("lock %s\n", pack_lockfile);
1030                 fflush(stdout);
1031         }
1032         close(fd[0]);
1033         close(fd[1]);
1034         if (finish_connect(conn))
1035                 ref = NULL;
1036         ret = !ref;
1037
1038         if (!ret && nr_heads) {
1039                 /* If the heads to pull were given, we should have
1040                  * consumed all of them by matching the remote.
1041                  * Otherwise, 'git fetch remote no-such-ref' would
1042                  * silently succeed without issuing an error.
1043                  */
1044                 for (i = 0; i < nr_heads; i++)
1045                         if (heads[i] && heads[i][0]) {
1046                                 error("no such remote ref %s", heads[i]);
1047                                 ret = 1;
1048                         }
1049         }
1050         while (ref) {
1051                 printf("%s %s\n",
1052                        sha1_to_hex(ref->old_sha1), ref->name);
1053                 ref = ref->next;
1054         }
1055
1056         return ret;
1057 }
1058
1059 static int compare_heads(const void *a, const void *b)
1060 {
1061         return strcmp(*(const char **)a, *(const char **)b);
1062 }
1063
1064 struct ref *fetch_pack(struct fetch_pack_args *my_args,
1065                        int fd[], struct child_process *conn,
1066                        const struct ref *ref,
1067                 const char *dest,
1068                 int nr_heads,
1069                 char **heads,
1070                 char **pack_lockfile)
1071 {
1072         struct stat st;
1073         struct ref *ref_cpy;
1074
1075         fetch_pack_setup();
1076         if (&args != my_args)
1077                 memcpy(&args, my_args, sizeof(args));
1078         if (args.depth > 0) {
1079                 if (stat(git_path("shallow"), &st))
1080                         st.st_mtime = 0;
1081         }
1082
1083         if (heads && nr_heads) {
1084                 qsort(heads, nr_heads, sizeof(*heads), compare_heads);
1085                 nr_heads = remove_duplicates(nr_heads, heads);
1086         }
1087
1088         if (!ref) {
1089                 packet_flush(fd[1]);
1090                 die("no matching remote head");
1091         }
1092         ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
1093
1094         if (args.depth > 0) {
1095                 struct cache_time mtime;
1096                 struct strbuf sb = STRBUF_INIT;
1097                 char *shallow = git_path("shallow");
1098                 int fd;
1099
1100                 mtime.sec = st.st_mtime;
1101                 mtime.nsec = ST_MTIME_NSEC(st);
1102                 if (stat(shallow, &st)) {
1103                         if (mtime.sec)
1104                                 die("shallow file was removed during fetch");
1105                 } else if (st.st_mtime != mtime.sec
1106 #ifdef USE_NSEC
1107                                 || ST_MTIME_NSEC(st) != mtime.nsec
1108 #endif
1109                           )
1110                         die("shallow file was changed during fetch");
1111
1112                 fd = hold_lock_file_for_update(&lock, shallow,
1113                                                LOCK_DIE_ON_ERROR);
1114                 if (!write_shallow_commits(&sb, 0)
1115                  || write_in_full(fd, sb.buf, sb.len) != sb.len) {
1116                         unlink_or_warn(shallow);
1117                         rollback_lock_file(&lock);
1118                 } else {
1119                         commit_lock_file(&lock);
1120                 }
1121                 strbuf_release(&sb);
1122         }
1123
1124         reprepare_packed_git();
1125         return ref_cpy;
1126 }