builtin/fetch-pack.c

   1 #include "builtin.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "transport.h"
  13
  14 static int transfer_unpack_limit = -1;
  15 static int fetch_unpack_limit = -1;
  16 static int unpack_limit = 100;
  17 static int prefer_ofs_delta = 1;
  18 static int no_done = 0;
  19 static struct fetch_pack_args args = {
  20         /* .uploadpack = */ "git-upload-pack",
  21 };
  22
  23 static const char fetch_pack_usage[] =
  24 "git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
  25
  26 #define COMPLETE        (1U << 0)
  27 #define COMMON          (1U << 1)
  28 #define COMMON_REF      (1U << 2)
  29 #define SEEN            (1U << 3)
  30 #define POPPED          (1U << 4)
  31
  32 static int marked;
  33
  34 /*
  35  * After sending this many "have"s if we do not get any new ACK , we
  36  * give up traversing our history.
  37  */
  38 #define MAX_IN_VAIN 256
  39
  40 static struct commit_list *rev_list;
  41 static int non_common_revs, multi_ack, use_sideband;
  42
  43 static void rev_list_push(struct commit *commit, int mark)
  44 {
  45         if (!(commit->object.flags & mark)) {
  46                 commit->object.flags |= mark;
  47
  48                 if (!(commit->object.parsed))
  49                         if (parse_commit(commit))
  50                                 return;
  51
  52                 commit_list_insert_by_date(commit, &rev_list);
  53
  54                 if (!(commit->object.flags & COMMON))
  55                         non_common_revs++;
  56         }
  57 }
  58
  59 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  60 {
  61         struct object *o = deref_tag(parse_object(sha1), path, 0);
  62
  63         if (o && o->type == OBJ_COMMIT)
  64                 rev_list_push((struct commit *)o, SEEN);
  65
  66         return 0;
  67 }
  68
  69 static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  70 {
  71         struct object *o = deref_tag(parse_object(sha1), path, 0);
  72
  73         if (o && o->type == OBJ_COMMIT)
  74                 clear_commit_marks((struct commit *)o,
  75                                    COMMON | COMMON_REF | SEEN | POPPED);
  76         return 0;
  77 }
  78
  79 /*
  80    This function marks a rev and its ancestors as common.
  81    In some cases, it is desirable to mark only the ancestors (for example
  82    when only the server does not yet know that they are common).
  83 */
  84
  85 static void mark_common(struct commit *commit,
  86                 int ancestors_only, int dont_parse)
  87 {
  88         if (commit != NULL && !(commit->object.flags & COMMON)) {
  89                 struct object *o = (struct object *)commit;
  90
  91                 if (!ancestors_only)
  92                         o->flags |= COMMON;
  93
  94                 if (!(o->flags & SEEN))
  95                         rev_list_push(commit, SEEN);
  96                 else {
  97                         struct commit_list *parents;
  98
  99                         if (!ancestors_only && !(o->flags & POPPED))
 100                                 non_common_revs--;
 101                         if (!o->parsed && !dont_parse)
 102                                 if (parse_commit(commit))
 103                                         return;
 104
 105                         for (parents = commit->parents;
 106                                         parents;
 107                                         parents = parents->next)
 108                                 mark_common(parents->item, 0, dont_parse);
 109                 }
 110         }
 111 }
 112
 113 /*
 114   Get the next rev to send, ignoring the common.
 115 */
 116
 117 static const unsigned char *get_rev(void)
 118 {
 119         struct commit *commit = NULL;
 120
 121         while (commit == NULL) {
 122                 unsigned int mark;
 123                 struct commit_list *parents;
 124
 125                 if (rev_list == NULL || non_common_revs == 0)
 126                         return NULL;
 127
 128                 commit = rev_list->item;
 129                 if (!commit->object.parsed)
 130                         parse_commit(commit);
 131                 parents = commit->parents;
 132
 133                 commit->object.flags |= POPPED;
 134                 if (!(commit->object.flags & COMMON))
 135                         non_common_revs--;
 136
 137                 if (commit->object.flags & COMMON) {
 138                         /* do not send "have", and ignore ancestors */
 139                         commit = NULL;
 140                         mark = COMMON | SEEN;
 141                 } else if (commit->object.flags & COMMON_REF)
 142                         /* send "have", and ignore ancestors */
 143                         mark = COMMON | SEEN;
 144                 else
 145                         /* send "have", also for its ancestors */
 146                         mark = SEEN;
 147
 148                 while (parents) {
 149                         if (!(parents->item->object.flags & SEEN))
 150                                 rev_list_push(parents->item, mark);
 151                         if (mark & COMMON)
 152                                 mark_common(parents->item, 1, 0);
 153                         parents = parents->next;
 154                 }
 155
 156                 rev_list = rev_list->next;
 157         }
 158
 159         return commit->object.sha1;
 160 }
 161
 162 enum ack_type {
 163         NAK = 0,
 164         ACK,
 165         ACK_continue,
 166         ACK_common,
 167         ACK_ready
 168 };
 169
 170 static void consume_shallow_list(int fd)
 171 {
 172         if (args.stateless_rpc && args.depth > 0) {
 173                 /* If we sent a depth we will get back "duplicate"
 174                  * shallow and unshallow commands every time there
 175                  * is a block of have lines exchanged.
 176                  */
 177                 char line[1000];
 178                 while (packet_read_line(fd, line, sizeof(line))) {
 179                         if (!prefixcmp(line, "shallow "))
 180                                 continue;
 181                         if (!prefixcmp(line, "unshallow "))
 182                                 continue;
 183                         die("git fetch-pack: expected shallow list");
 184                 }
 185         }
 186 }
 187
 188 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 189 {
 190         static char line[1000];
 191         int len = packet_read_line(fd, line, sizeof(line));
 192
 193         if (!len)
 194                 die("git fetch-pack: expected ACK/NAK, got EOF");
 195         if (line[len-1] == '\n')
 196                 line[--len] = 0;
 197         if (!strcmp(line, "NAK"))
 198                 return NAK;
 199         if (!prefixcmp(line, "ACK ")) {
 200                 if (!get_sha1_hex(line+4, result_sha1)) {
 201                         if (strstr(line+45, "continue"))
 202                                 return ACK_continue;
 203                         if (strstr(line+45, "common"))
 204                                 return ACK_common;
 205                         if (strstr(line+45, "ready"))
 206                                 return ACK_ready;
 207                         return ACK;
 208                 }
 209         }
 210         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 211 }
 212
 213 static void send_request(int fd, struct strbuf *buf)
 214 {
 215         if (args.stateless_rpc) {
 216                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 217                 packet_flush(fd);
 218         } else
 219                 safe_write(fd, buf->buf, buf->len);
 220 }
 221
 222 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 223 {
 224         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 225 }
 226
 227 static void insert_alternate_refs(void)
 228 {
 229         foreach_alt_odb(refs_from_alternate_cb, insert_one_alternate_ref);
 230 }
 231
 232 #define INITIAL_FLUSH 16
 233 #define LARGE_FLUSH 1024
 234
 235 static int next_flush(int count)
 236 {
 237         if (count < INITIAL_FLUSH * 2)
 238                 count += INITIAL_FLUSH;
 239         else if (count < LARGE_FLUSH)
 240                 count <<= 1;
 241         else
 242                 count += LARGE_FLUSH;
 243         return count;
 244 }
 245
 246 static int find_common(int fd[2], unsigned char *result_sha1,
 247                        struct ref *refs)
 248 {
 249         int fetching;
 250         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 251         const unsigned char *sha1;
 252         unsigned in_vain = 0;
 253         int got_continue = 0;
 254         int got_ready = 0;
 255         struct strbuf req_buf = STRBUF_INIT;
 256         size_t state_len = 0;
 257
 258         if (args.stateless_rpc && multi_ack == 1)
 259                 die("--stateless-rpc requires multi_ack_detailed");
 260         if (marked)
 261                 for_each_ref(clear_marks, NULL);
 262         marked = 1;
 263
 264         for_each_ref(rev_list_insert_ref, NULL);
 265         insert_alternate_refs();
 266
 267         fetching = 0;
 268         for ( ; refs ; refs = refs->next) {
 269                 unsigned char *remote = refs->old_sha1;
 270                 const char *remote_hex;
 271                 struct object *o;
 272
 273                 /*
 274                  * If that object is complete (i.e. it is an ancestor of a
 275                  * local ref), we tell them we have it but do not have to
 276                  * tell them about its ancestors, which they already know
 277                  * about.
 278                  *
 279                  * We use lookup_object here because we are only
 280                  * interested in the case we *know* the object is
 281                  * reachable and we have already scanned it.
 282                  */
 283                 if (((o = lookup_object(remote)) != NULL) &&
 284                                 (o->flags & COMPLETE)) {
 285                         continue;
 286                 }
 287
 288                 remote_hex = sha1_to_hex(remote);
 289                 if (!fetching) {
 290                         struct strbuf c = STRBUF_INIT;
 291                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 292                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 293                         if (no_done)            strbuf_addstr(&c, " no-done");
 294                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 295                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 296                         if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack");
 297                         if (args.no_progress)   strbuf_addstr(&c, " no-progress");
 298                         if (args.include_tag)   strbuf_addstr(&c, " include-tag");
 299                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 300                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 301                         strbuf_release(&c);
 302                 } else
 303                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 304                 fetching++;
 305         }
 306
 307         if (!fetching) {
 308                 strbuf_release(&req_buf);
 309                 packet_flush(fd[1]);
 310                 return 1;
 311         }
 312
 313         if (is_repository_shallow())
 314                 write_shallow_commits(&req_buf, 1);
 315         if (args.depth > 0)
 316                 packet_buf_write(&req_buf, "deepen %d", args.depth);
 317         packet_buf_flush(&req_buf);
 318         state_len = req_buf.len;
 319
 320         if (args.depth > 0) {
 321                 char line[1024];
 322                 unsigned char sha1[20];
 323
 324                 send_request(fd[1], &req_buf);
 325                 while (packet_read_line(fd[0], line, sizeof(line))) {
 326                         if (!prefixcmp(line, "shallow ")) {
 327                                 if (get_sha1_hex(line + 8, sha1))
 328                                         die("invalid shallow line: %s", line);
 329                                 register_shallow(sha1);
 330                                 continue;
 331                         }
 332                         if (!prefixcmp(line, "unshallow ")) {
 333                                 if (get_sha1_hex(line + 10, sha1))
 334                                         die("invalid unshallow line: %s", line);
 335                                 if (!lookup_object(sha1))
 336                                         die("object not found: %s", line);
 337                                 /* make sure that it is parsed as shallow */
 338                                 if (!parse_object(sha1))
 339                                         die("error in object: %s", line);
 340                                 if (unregister_shallow(sha1))
 341                                         die("no shallow found: %s", line);
 342                                 continue;
 343                         }
 344                         die("expected shallow/unshallow, got %s", line);
 345                 }
 346         } else if (!args.stateless_rpc)
 347                 send_request(fd[1], &req_buf);
 348
 349         if (!args.stateless_rpc) {
 350                 /* If we aren't using the stateless-rpc interface
 351                  * we don't need to retain the headers.
 352                  */
 353                 strbuf_setlen(&req_buf, 0);
 354                 state_len = 0;
 355         }
 356
 357         flushes = 0;
 358         retval = -1;
 359         while ((sha1 = get_rev())) {
 360                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 361                 if (args.verbose)
 362                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 363                 in_vain++;
 364                 if (flush_at <= ++count) {
 365                         int ack;
 366
 367                         packet_buf_flush(&req_buf);
 368                         send_request(fd[1], &req_buf);
 369                         strbuf_setlen(&req_buf, state_len);
 370                         flushes++;
 371                         flush_at = next_flush(count);
 372
 373                         /*
 374                          * We keep one window "ahead" of the other side, and
 375                          * will wait for an ACK only on the next one
 376                          */
 377                         if (!args.stateless_rpc && count == INITIAL_FLUSH)
 378                                 continue;
 379
 380                         consume_shallow_list(fd[0]);
 381                         do {
 382                                 ack = get_ack(fd[0], result_sha1);
 383                                 if (args.verbose && ack)
 384                                         fprintf(stderr, "got ack %d %s\n", ack,
 385                                                         sha1_to_hex(result_sha1));
 386                                 switch (ack) {
 387                                 case ACK:
 388                                         flushes = 0;
 389                                         multi_ack = 0;
 390                                         retval = 0;
 391                                         goto done;
 392                                 case ACK_common:
 393                                 case ACK_ready:
 394                                 case ACK_continue: {
 395                                         struct commit *commit =
 396                                                 lookup_commit(result_sha1);
 397                                         if (args.stateless_rpc
 398                                          && ack == ACK_common
 399                                          && !(commit->object.flags & COMMON)) {
 400                                                 /* We need to replay the have for this object
 401                                                  * on the next RPC request so the peer knows
 402                                                  * it is in common with us.
 403                                                  */
 404                                                 const char *hex = sha1_to_hex(result_sha1);
 405                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 406                                                 state_len = req_buf.len;
 407                                         }
 408                                         mark_common(commit, 0, 1);
 409                                         retval = 0;
 410                                         in_vain = 0;
 411                                         got_continue = 1;
 412                                         if (ack == ACK_ready) {
 413                                                 rev_list = NULL;
 414                                                 got_ready = 1;
 415                                         }
 416                                         break;
 417                                         }
 418                                 }
 419                         } while (ack);
 420                         flushes--;
 421                         if (got_continue && MAX_IN_VAIN < in_vain) {
 422                                 if (args.verbose)
 423                                         fprintf(stderr, "giving up\n");
 424                                 break; /* give up */
 425                         }
 426                 }
 427         }
 428 done:
 429         if (!got_ready || !no_done) {
 430                 packet_buf_write(&req_buf, "done\n");
 431                 send_request(fd[1], &req_buf);
 432         }
 433         if (args.verbose)
 434                 fprintf(stderr, "done\n");
 435         if (retval != 0) {
 436                 multi_ack = 0;
 437                 flushes++;
 438         }
 439         strbuf_release(&req_buf);
 440
 441         consume_shallow_list(fd[0]);
 442         while (flushes || multi_ack) {
 443                 int ack = get_ack(fd[0], result_sha1);
 444                 if (ack) {
 445                         if (args.verbose)
 446                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 447                                         sha1_to_hex(result_sha1));
 448                         if (ack == ACK)
 449                                 return 0;
 450                         multi_ack = 1;
 451                         continue;
 452                 }
 453                 flushes--;
 454         }
 455         /* it is no error to fetch into a completely empty repo */
 456         return count ? retval : 0;
 457 }
 458
 459 static struct commit_list *complete;
 460
 461 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 462 {
 463         struct object *o = parse_object(sha1);
 464
 465         while (o && o->type == OBJ_TAG) {
 466                 struct tag *t = (struct tag *) o;
 467                 if (!t->tagged)
 468                         break; /* broken repository */
 469                 o->flags |= COMPLETE;
 470                 o = parse_object(t->tagged->sha1);
 471         }
 472         if (o && o->type == OBJ_COMMIT) {
 473                 struct commit *commit = (struct commit *)o;
 474                 commit->object.flags |= COMPLETE;
 475                 commit_list_insert_by_date(commit, &complete);
 476         }
 477         return 0;
 478 }
 479
 480 static void mark_recent_complete_commits(unsigned long cutoff)
 481 {
 482         while (complete && cutoff <= complete->item->date) {
 483                 if (args.verbose)
 484                         fprintf(stderr, "Marking %s as complete\n",
 485                                 sha1_to_hex(complete->item->object.sha1));
 486                 pop_most_recent_commit(&complete, COMPLETE);
 487         }
 488 }
 489
 490 static void filter_refs(struct ref **refs, int nr_match, char **match)
 491 {
 492         struct ref **return_refs;
 493         struct ref *newlist = NULL;
 494         struct ref **newtail = &newlist;
 495         struct ref *ref, *next;
 496         struct ref *fastarray[32];
 497
 498         if (nr_match && !args.fetch_all) {
 499                 if (ARRAY_SIZE(fastarray) < nr_match)
 500                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 501                 else {
 502                         return_refs = fastarray;
 503                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 504                 }
 505         }
 506         else
 507                 return_refs = NULL;
 508
 509         for (ref = *refs; ref; ref = next) {
 510                 next = ref->next;
 511                 if (!memcmp(ref->name, "refs/", 5) &&
 512                     check_ref_format(ref->name + 5))
 513                         ; /* trash */
 514                 else if (args.fetch_all &&
 515                          (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
 516                         *newtail = ref;
 517                         ref->next = NULL;
 518                         newtail = &ref->next;
 519                         continue;
 520                 }
 521                 else {
 522                         int order = path_match(ref->name, nr_match, match);
 523                         if (order) {
 524                                 return_refs[order-1] = ref;
 525                                 continue; /* we will link it later */
 526                         }
 527                 }
 528                 free(ref);
 529         }
 530
 531         if (!args.fetch_all) {
 532                 int i;
 533                 for (i = 0; i < nr_match; i++) {
 534                         ref = return_refs[i];
 535                         if (ref) {
 536                                 *newtail = ref;
 537                                 ref->next = NULL;
 538                                 newtail = &ref->next;
 539                         }
 540                 }
 541                 if (return_refs != fastarray)
 542                         free(return_refs);
 543         }
 544         *refs = newlist;
 545 }
 546
 547 static int everything_local(struct ref **refs, int nr_match, char **match)
 548 {
 549         struct ref *ref;
 550         int retval;
 551         unsigned long cutoff = 0;
 552
 553         save_commit_buffer = 0;
 554
 555         for (ref = *refs; ref; ref = ref->next) {
 556                 struct object *o;
 557
 558                 o = parse_object(ref->old_sha1);
 559                 if (!o)
 560                         continue;
 561
 562                 /* We already have it -- which may mean that we were
 563                  * in sync with the other side at some time after
 564                  * that (it is OK if we guess wrong here).
 565                  */
 566                 if (o->type == OBJ_COMMIT) {
 567                         struct commit *commit = (struct commit *)o;
 568                         if (!cutoff || cutoff < commit->date)
 569                                 cutoff = commit->date;
 570                 }
 571         }
 572
 573         if (!args.depth) {
 574                 for_each_ref(mark_complete, NULL);
 575                 if (cutoff)
 576                         mark_recent_complete_commits(cutoff);
 577         }
 578
 579         /*
 580          * Mark all complete remote refs as common refs.
 581          * Don't mark them common yet; the server has to be told so first.
 582          */
 583         for (ref = *refs; ref; ref = ref->next) {
 584                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 585                                              NULL, 0);
 586
 587                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 588                         continue;
 589
 590                 if (!(o->flags & SEEN)) {
 591                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 592
 593                         mark_common((struct commit *)o, 1, 1);
 594                 }
 595         }
 596
 597         filter_refs(refs, nr_match, match);
 598
 599         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 600                 const unsigned char *remote = ref->old_sha1;
 601                 unsigned char local[20];
 602                 struct object *o;
 603
 604                 o = lookup_object(remote);
 605                 if (!o || !(o->flags & COMPLETE)) {
 606                         retval = 0;
 607                         if (!args.verbose)
 608                                 continue;
 609                         fprintf(stderr,
 610                                 "want %s (%s)\n", sha1_to_hex(remote),
 611                                 ref->name);
 612                         continue;
 613                 }
 614
 615                 hashcpy(ref->new_sha1, local);
 616                 if (!args.verbose)
 617                         continue;
 618                 fprintf(stderr,
 619                         "already have %s (%s)\n", sha1_to_hex(remote),
 620                         ref->name);
 621         }
 622         return retval;
 623 }
 624
 625 static int sideband_demux(int in, int out, void *data)
 626 {
 627         int *xd = data;
 628
 629         int ret = recv_sideband("fetch-pack", xd[0], out);
 630         close(out);
 631         return ret;
 632 }
 633
 634 static int get_pack(int xd[2], char **pack_lockfile)
 635 {
 636         struct async demux;
 637         const char *argv[20];
 638         char keep_arg[256];
 639         char hdr_arg[256];
 640         const char **av;
 641         int do_keep = args.keep_pack;
 642         struct child_process cmd;
 643
 644         memset(&demux, 0, sizeof(demux));
 645         if (use_sideband) {
 646                 /* xd[] is talking with upload-pack; subprocess reads from
 647                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 648                  * through demux->out.
 649                  */
 650                 demux.proc = sideband_demux;
 651                 demux.data = xd;
 652                 demux.out = -1;
 653                 if (start_async(&demux))
 654                         die("fetch-pack: unable to fork off sideband"
 655                             " demultiplexer");
 656         }
 657         else
 658                 demux.out = xd[0];
 659
 660         memset(&cmd, 0, sizeof(cmd));
 661         cmd.argv = argv;
 662         av = argv;
 663         *hdr_arg = 0;
 664         if (!args.keep_pack && unpack_limit) {
 665                 struct pack_header header;
 666
 667                 if (read_pack_header(demux.out, &header))
 668                         die("protocol error: bad pack header");
 669                 snprintf(hdr_arg, sizeof(hdr_arg),
 670                          "--pack_header=%"PRIu32",%"PRIu32,
 671                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 672                 if (ntohl(header.hdr_entries) < unpack_limit)
 673                         do_keep = 0;
 674                 else
 675                         do_keep = 1;
 676         }
 677
 678         if (do_keep) {
 679                 if (pack_lockfile)
 680                         cmd.out = -1;
 681                 *av++ = "index-pack";
 682                 *av++ = "--stdin";
 683                 if (!args.quiet && !args.no_progress)
 684                         *av++ = "-v";
 685                 if (args.use_thin_pack)
 686                         *av++ = "--fix-thin";
 687                 if (args.lock_pack || unpack_limit) {
 688                         int s = sprintf(keep_arg,
 689                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 690                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 691                                 strcpy(keep_arg + s, "localhost");
 692                         *av++ = keep_arg;
 693                 }
 694         }
 695         else {
 696                 *av++ = "unpack-objects";
 697                 if (args.quiet)
 698                         *av++ = "-q";
 699         }
 700         if (*hdr_arg)
 701                 *av++ = hdr_arg;
 702         *av++ = NULL;
 703
 704         cmd.in = demux.out;
 705         cmd.git_cmd = 1;
 706         if (start_command(&cmd))
 707                 die("fetch-pack: unable to fork off %s", argv[0]);
 708         if (do_keep && pack_lockfile) {
 709                 *pack_lockfile = index_pack_lockfile(cmd.out);
 710                 close(cmd.out);
 711         }
 712
 713         if (finish_command(&cmd))
 714                 die("%s failed", argv[0]);
 715         if (use_sideband && finish_async(&demux))
 716                 die("error in sideband demultiplexer");
 717         return 0;
 718 }
 719
 720 static struct ref *do_fetch_pack(int fd[2],
 721                 const struct ref *orig_ref,
 722                 int nr_match,
 723                 char **match,
 724                 char **pack_lockfile)
 725 {
 726         struct ref *ref = copy_ref_list(orig_ref);
 727         unsigned char sha1[20];
 728
 729         if (is_repository_shallow() && !server_supports("shallow"))
 730                 die("Server does not support shallow clients");
 731         if (server_supports("multi_ack_detailed")) {
 732                 if (args.verbose)
 733                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 734                 multi_ack = 2;
 735                 if (server_supports("no-done")) {
 736                         if (args.verbose)
 737                                 fprintf(stderr, "Server supports no-done\n");
 738                         no_done = 1;
 739                 }
 740         }
 741         else if (server_supports("multi_ack")) {
 742                 if (args.verbose)
 743                         fprintf(stderr, "Server supports multi_ack\n");
 744                 multi_ack = 1;
 745         }
 746         if (server_supports("side-band-64k")) {
 747                 if (args.verbose)
 748                         fprintf(stderr, "Server supports side-band-64k\n");
 749                 use_sideband = 2;
 750         }
 751         else if (server_supports("side-band")) {
 752                 if (args.verbose)
 753                         fprintf(stderr, "Server supports side-band\n");
 754                 use_sideband = 1;
 755         }
 756         if (server_supports("ofs-delta")) {
 757                 if (args.verbose)
 758                         fprintf(stderr, "Server supports ofs-delta\n");
 759         } else
 760                 prefer_ofs_delta = 0;
 761         if (everything_local(&ref, nr_match, match)) {
 762                 packet_flush(fd[1]);
 763                 goto all_done;
 764         }
 765         if (find_common(fd, sha1, ref) < 0)
 766                 if (!args.keep_pack)
 767                         /* When cloning, it is not unusual to have
 768                          * no common commit.
 769                          */
 770                         warning("no common commits");
 771
 772         if (args.stateless_rpc)
 773                 packet_flush(fd[1]);
 774         if (get_pack(fd, pack_lockfile))
 775                 die("git fetch-pack: fetch failed.");
 776
 777  all_done:
 778         return ref;
 779 }
 780
 781 static int remove_duplicates(int nr_heads, char **heads)
 782 {
 783         int src, dst;
 784
 785         for (src = dst = 0; src < nr_heads; src++) {
 786                 /* If heads[src] is different from any of
 787                  * heads[0..dst], push it in.
 788                  */
 789                 int i;
 790                 for (i = 0; i < dst; i++) {
 791                         if (!strcmp(heads[i], heads[src]))
 792                                 break;
 793                 }
 794                 if (i < dst)
 795                         continue;
 796                 if (src != dst)
 797                         heads[dst] = heads[src];
 798                 dst++;
 799         }
 800         return dst;
 801 }
 802
 803 static int fetch_pack_config(const char *var, const char *value, void *cb)
 804 {
 805         if (strcmp(var, "fetch.unpacklimit") == 0) {
 806                 fetch_unpack_limit = git_config_int(var, value);
 807                 return 0;
 808         }
 809
 810         if (strcmp(var, "transfer.unpacklimit") == 0) {
 811                 transfer_unpack_limit = git_config_int(var, value);
 812                 return 0;
 813         }
 814
 815         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 816                 prefer_ofs_delta = git_config_bool(var, value);
 817                 return 0;
 818         }
 819
 820         return git_default_config(var, value, cb);
 821 }
 822
 823 static struct lock_file lock;
 824
 825 static void fetch_pack_setup(void)
 826 {
 827         static int did_setup;
 828         if (did_setup)
 829                 return;
 830         git_config(fetch_pack_config, NULL);
 831         if (0 <= transfer_unpack_limit)
 832                 unpack_limit = transfer_unpack_limit;
 833         else if (0 <= fetch_unpack_limit)
 834                 unpack_limit = fetch_unpack_limit;
 835         did_setup = 1;
 836 }
 837
 838 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 839 {
 840         int i, ret, nr_heads;
 841         struct ref *ref = NULL;
 842         char *dest = NULL, **heads;
 843         int fd[2];
 844         char *pack_lockfile = NULL;
 845         char **pack_lockfile_ptr = NULL;
 846         struct child_process *conn;
 847
 848         packet_trace_identity("fetch-pack");
 849
 850         nr_heads = 0;
 851         heads = NULL;
 852         for (i = 1; i < argc; i++) {
 853                 const char *arg = argv[i];
 854
 855                 if (*arg == '-') {
 856                         if (!prefixcmp(arg, "--upload-pack=")) {
 857                                 args.uploadpack = arg + 14;
 858                                 continue;
 859                         }
 860                         if (!prefixcmp(arg, "--exec=")) {
 861                                 args.uploadpack = arg + 7;
 862                                 continue;
 863                         }
 864                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 865                                 args.quiet = 1;
 866                                 continue;
 867                         }
 868                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 869                                 args.lock_pack = args.keep_pack;
 870                                 args.keep_pack = 1;
 871                                 continue;
 872                         }
 873                         if (!strcmp("--thin", arg)) {
 874                                 args.use_thin_pack = 1;
 875                                 continue;
 876                         }
 877                         if (!strcmp("--include-tag", arg)) {
 878                                 args.include_tag = 1;
 879                                 continue;
 880                         }
 881                         if (!strcmp("--all", arg)) {
 882                                 args.fetch_all = 1;
 883                                 continue;
 884                         }
 885                         if (!strcmp("-v", arg)) {
 886                                 args.verbose = 1;
 887                                 continue;
 888                         }
 889                         if (!prefixcmp(arg, "--depth=")) {
 890                                 args.depth = strtol(arg + 8, NULL, 0);
 891                                 continue;
 892                         }
 893                         if (!strcmp("--no-progress", arg)) {
 894                                 args.no_progress = 1;
 895                                 continue;
 896                         }
 897                         if (!strcmp("--stateless-rpc", arg)) {
 898                                 args.stateless_rpc = 1;
 899                                 continue;
 900                         }
 901                         if (!strcmp("--lock-pack", arg)) {
 902                                 args.lock_pack = 1;
 903                                 pack_lockfile_ptr = &pack_lockfile;
 904                                 continue;
 905                         }
 906                         usage(fetch_pack_usage);
 907                 }
 908                 dest = (char *)arg;
 909                 heads = (char **)(argv + i + 1);
 910                 nr_heads = argc - i - 1;
 911                 break;
 912         }
 913         if (!dest)
 914                 usage(fetch_pack_usage);
 915
 916         if (args.stateless_rpc) {
 917                 conn = NULL;
 918                 fd[0] = 0;
 919                 fd[1] = 1;
 920         } else {
 921                 conn = git_connect(fd, (char *)dest, args.uploadpack,
 922                                    args.verbose ? CONNECT_VERBOSE : 0);
 923         }
 924
 925         get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL);
 926
 927         ref = fetch_pack(&args, fd, conn, ref, dest,
 928                 nr_heads, heads, pack_lockfile_ptr);
 929         if (pack_lockfile) {
 930                 printf("lock %s\n", pack_lockfile);
 931                 fflush(stdout);
 932         }
 933         close(fd[0]);
 934         close(fd[1]);
 935         if (finish_connect(conn))
 936                 ref = NULL;
 937         ret = !ref;
 938
 939         if (!ret && nr_heads) {
 940                 /* If the heads to pull were given, we should have
 941                  * consumed all of them by matching the remote.
 942                  * Otherwise, 'git fetch remote no-such-ref' would
 943                  * silently succeed without issuing an error.
 944                  */
 945                 for (i = 0; i < nr_heads; i++)
 946                         if (heads[i] && heads[i][0]) {
 947                                 error("no such remote ref %s", heads[i]);
 948                                 ret = 1;
 949                         }
 950         }
 951         while (ref) {
 952                 printf("%s %s\n",
 953                        sha1_to_hex(ref->old_sha1), ref->name);
 954                 ref = ref->next;
 955         }
 956
 957         return ret;
 958 }
 959
 960 struct ref *fetch_pack(struct fetch_pack_args *my_args,
 961                        int fd[], struct child_process *conn,
 962                        const struct ref *ref,
 963                 const char *dest,
 964                 int nr_heads,
 965                 char **heads,
 966                 char **pack_lockfile)
 967 {
 968         struct stat st;
 969         struct ref *ref_cpy;
 970
 971         fetch_pack_setup();
 972         if (&args != my_args)
 973                 memcpy(&args, my_args, sizeof(args));
 974         if (args.depth > 0) {
 975                 if (stat(git_path("shallow"), &st))
 976                         st.st_mtime = 0;
 977         }
 978
 979         if (heads && nr_heads)
 980                 nr_heads = remove_duplicates(nr_heads, heads);
 981         if (!ref) {
 982                 packet_flush(fd[1]);
 983                 die("no matching remote head");
 984         }
 985         ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
 986
 987         if (args.depth > 0) {
 988                 struct cache_time mtime;
 989                 struct strbuf sb = STRBUF_INIT;
 990                 char *shallow = git_path("shallow");
 991                 int fd;
 992
 993                 mtime.sec = st.st_mtime;
 994                 mtime.nsec = ST_MTIME_NSEC(st);
 995                 if (stat(shallow, &st)) {
 996                         if (mtime.sec)
 997                                 die("shallow file was removed during fetch");
 998                 } else if (st.st_mtime != mtime.sec
 999 #ifdef USE_NSEC
1000                                 || ST_MTIME_NSEC(st) != mtime.nsec
1001 #endif
1002                           )
1003                         die("shallow file was changed during fetch");
1004
1005                 fd = hold_lock_file_for_update(&lock, shallow,
1006                                                LOCK_DIE_ON_ERROR);
1007                 if (!write_shallow_commits(&sb, 0)
1008                  || write_in_full(fd, sb.buf, sb.len) != sb.len) {
1009                         unlink_or_warn(shallow);
1010                         rollback_lock_file(&lock);
1011                 } else {
1012                         commit_lock_file(&lock);
1013                 }
1014                 strbuf_release(&sb);
1015         }
1016
1017         reprepare_packed_git();
1018         return ref_cpy;
1019 }