builtin/fetch-pack.c

   1 #include "builtin.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "transport.h"
  13
  14 static int transfer_unpack_limit = -1;
  15 static int fetch_unpack_limit = -1;
  16 static int unpack_limit = 100;
  17 static int prefer_ofs_delta = 1;
  18 static int no_done = 0;
  19 static struct fetch_pack_args args = {
  20         /* .uploadpack = */ "git-upload-pack",
  21 };
  22
  23 static const char fetch_pack_usage[] =
  24 "git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
  25
  26 #define COMPLETE        (1U << 0)
  27 #define COMMON          (1U << 1)
  28 #define COMMON_REF      (1U << 2)
  29 #define SEEN            (1U << 3)
  30 #define POPPED          (1U << 4)
  31
  32 static int marked;
  33
  34 /*
  35  * After sending this many "have"s if we do not get any new ACK , we
  36  * give up traversing our history.
  37  */
  38 #define MAX_IN_VAIN 256
  39
  40 static struct commit_list *rev_list;
  41 static int non_common_revs, multi_ack, use_sideband;
  42
  43 static void rev_list_push(struct commit *commit, int mark)
  44 {
  45         if (!(commit->object.flags & mark)) {
  46                 commit->object.flags |= mark;
  47
  48                 if (!(commit->object.parsed))
  49                         if (parse_commit(commit))
  50                                 return;
  51
  52                 commit_list_insert_by_date(commit, &rev_list);
  53
  54                 if (!(commit->object.flags & COMMON))
  55                         non_common_revs++;
  56         }
  57 }
  58
  59 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  60 {
  61         struct object *o = deref_tag(parse_object(sha1), path, 0);
  62
  63         if (o && o->type == OBJ_COMMIT)
  64                 rev_list_push((struct commit *)o, SEEN);
  65
  66         return 0;
  67 }
  68
  69 static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  70 {
  71         struct object *o = deref_tag(parse_object(sha1), path, 0);
  72
  73         if (o && o->type == OBJ_COMMIT)
  74                 clear_commit_marks((struct commit *)o,
  75                                    COMMON | COMMON_REF | SEEN | POPPED);
  76         return 0;
  77 }
  78
  79 /*
  80    This function marks a rev and its ancestors as common.
  81    In some cases, it is desirable to mark only the ancestors (for example
  82    when only the server does not yet know that they are common).
  83 */
  84
  85 static void mark_common(struct commit *commit,
  86                 int ancestors_only, int dont_parse)
  87 {
  88         if (commit != NULL && !(commit->object.flags & COMMON)) {
  89                 struct object *o = (struct object *)commit;
  90
  91                 if (!ancestors_only)
  92                         o->flags |= COMMON;
  93
  94                 if (!(o->flags & SEEN))
  95                         rev_list_push(commit, SEEN);
  96                 else {
  97                         struct commit_list *parents;
  98
  99                         if (!ancestors_only && !(o->flags & POPPED))
 100                                 non_common_revs--;
 101                         if (!o->parsed && !dont_parse)
 102                                 if (parse_commit(commit))
 103                                         return;
 104
 105                         for (parents = commit->parents;
 106                                         parents;
 107                                         parents = parents->next)
 108                                 mark_common(parents->item, 0, dont_parse);
 109                 }
 110         }
 111 }
 112
 113 /*
 114   Get the next rev to send, ignoring the common.
 115 */
 116
 117 static const unsigned char *get_rev(void)
 118 {
 119         struct commit *commit = NULL;
 120
 121         while (commit == NULL) {
 122                 unsigned int mark;
 123                 struct commit_list *parents;
 124
 125                 if (rev_list == NULL || non_common_revs == 0)
 126                         return NULL;
 127
 128                 commit = rev_list->item;
 129                 if (!commit->object.parsed)
 130                         parse_commit(commit);
 131                 parents = commit->parents;
 132
 133                 commit->object.flags |= POPPED;
 134                 if (!(commit->object.flags & COMMON))
 135                         non_common_revs--;
 136
 137                 if (commit->object.flags & COMMON) {
 138                         /* do not send "have", and ignore ancestors */
 139                         commit = NULL;
 140                         mark = COMMON | SEEN;
 141                 } else if (commit->object.flags & COMMON_REF)
 142                         /* send "have", and ignore ancestors */
 143                         mark = COMMON | SEEN;
 144                 else
 145                         /* send "have", also for its ancestors */
 146                         mark = SEEN;
 147
 148                 while (parents) {
 149                         if (!(parents->item->object.flags & SEEN))
 150                                 rev_list_push(parents->item, mark);
 151                         if (mark & COMMON)
 152                                 mark_common(parents->item, 1, 0);
 153                         parents = parents->next;
 154                 }
 155
 156                 rev_list = rev_list->next;
 157         }
 158
 159         return commit->object.sha1;
 160 }
 161
 162 enum ack_type {
 163         NAK = 0,
 164         ACK,
 165         ACK_continue,
 166         ACK_common,
 167         ACK_ready
 168 };
 169
 170 static void consume_shallow_list(int fd)
 171 {
 172         if (args.stateless_rpc && args.depth > 0) {
 173                 /* If we sent a depth we will get back "duplicate"
 174                  * shallow and unshallow commands every time there
 175                  * is a block of have lines exchanged.
 176                  */
 177                 char line[1000];
 178                 while (packet_read_line(fd, line, sizeof(line))) {
 179                         if (!prefixcmp(line, "shallow "))
 180                                 continue;
 181                         if (!prefixcmp(line, "unshallow "))
 182                                 continue;
 183                         die("git fetch-pack: expected shallow list");
 184                 }
 185         }
 186 }
 187
 188 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 189 {
 190         static char line[1000];
 191         int len = packet_read_line(fd, line, sizeof(line));
 192
 193         if (!len)
 194                 die("git fetch-pack: expected ACK/NAK, got EOF");
 195         if (line[len-1] == '\n')
 196                 line[--len] = 0;
 197         if (!strcmp(line, "NAK"))
 198                 return NAK;
 199         if (!prefixcmp(line, "ACK ")) {
 200                 if (!get_sha1_hex(line+4, result_sha1)) {
 201                         if (strstr(line+45, "continue"))
 202                                 return ACK_continue;
 203                         if (strstr(line+45, "common"))
 204                                 return ACK_common;
 205                         if (strstr(line+45, "ready"))
 206                                 return ACK_ready;
 207                         return ACK;
 208                 }
 209         }
 210         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 211 }
 212
 213 static void send_request(int fd, struct strbuf *buf)
 214 {
 215         if (args.stateless_rpc) {
 216                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 217                 packet_flush(fd);
 218         } else
 219                 safe_write(fd, buf->buf, buf->len);
 220 }
 221
 222 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 223 {
 224         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 225 }
 226
 227 static void insert_alternate_refs(void)
 228 {
 229         foreach_alt_odb(refs_from_alternate_cb, insert_one_alternate_ref);
 230 }
 231
 232 #define INITIAL_FLUSH 16
 233 #define LARGE_FLUSH 1024
 234
 235 static int next_flush(int count)
 236 {
 237         if (count < INITIAL_FLUSH * 2)
 238                 count += INITIAL_FLUSH;
 239         else if (count < LARGE_FLUSH)
 240                 count <<= 1;
 241         else
 242                 count += LARGE_FLUSH;
 243         return count;
 244 }
 245
 246 static int find_common(int fd[2], unsigned char *result_sha1,
 247                        struct ref *refs)
 248 {
 249         int fetching;
 250         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 251         const unsigned char *sha1;
 252         unsigned in_vain = 0;
 253         int got_continue = 0;
 254         int got_ready = 0;
 255         struct strbuf req_buf = STRBUF_INIT;
 256         size_t state_len = 0;
 257
 258         if (args.stateless_rpc && multi_ack == 1)
 259                 die("--stateless-rpc requires multi_ack_detailed");
 260         if (marked)
 261                 for_each_ref(clear_marks, NULL);
 262         marked = 1;
 263
 264         for_each_ref(rev_list_insert_ref, NULL);
 265         insert_alternate_refs();
 266
 267         fetching = 0;
 268         for ( ; refs ; refs = refs->next) {
 269                 unsigned char *remote = refs->old_sha1;
 270                 const char *remote_hex;
 271                 struct object *o;
 272
 273                 /*
 274                  * If that object is complete (i.e. it is an ancestor of a
 275                  * local ref), we tell them we have it but do not have to
 276                  * tell them about its ancestors, which they already know
 277                  * about.
 278                  *
 279                  * We use lookup_object here because we are only
 280                  * interested in the case we *know* the object is
 281                  * reachable and we have already scanned it.
 282                  */
 283                 if (((o = lookup_object(remote)) != NULL) &&
 284                                 (o->flags & COMPLETE)) {
 285                         continue;
 286                 }
 287
 288                 remote_hex = sha1_to_hex(remote);
 289                 if (!fetching) {
 290                         struct strbuf c = STRBUF_INIT;
 291                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 292                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 293                         if (no_done)            strbuf_addstr(&c, " no-done");
 294                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 295                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 296                         if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack");
 297                         if (args.no_progress)   strbuf_addstr(&c, " no-progress");
 298                         if (args.include_tag)   strbuf_addstr(&c, " include-tag");
 299                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 300                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 301                         strbuf_release(&c);
 302                 } else
 303                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 304                 fetching++;
 305         }
 306
 307         if (!fetching) {
 308                 strbuf_release(&req_buf);
 309                 packet_flush(fd[1]);
 310                 return 1;
 311         }
 312
 313         if (is_repository_shallow())
 314                 write_shallow_commits(&req_buf, 1);
 315         if (args.depth > 0)
 316                 packet_buf_write(&req_buf, "deepen %d", args.depth);
 317         packet_buf_flush(&req_buf);
 318         state_len = req_buf.len;
 319
 320         if (args.depth > 0) {
 321                 char line[1024];
 322                 unsigned char sha1[20];
 323
 324                 send_request(fd[1], &req_buf);
 325                 while (packet_read_line(fd[0], line, sizeof(line))) {
 326                         if (!prefixcmp(line, "shallow ")) {
 327                                 if (get_sha1_hex(line + 8, sha1))
 328                                         die("invalid shallow line: %s", line);
 329                                 register_shallow(sha1);
 330                                 continue;
 331                         }
 332                         if (!prefixcmp(line, "unshallow ")) {
 333                                 if (get_sha1_hex(line + 10, sha1))
 334                                         die("invalid unshallow line: %s", line);
 335                                 if (!lookup_object(sha1))
 336                                         die("object not found: %s", line);
 337                                 /* make sure that it is parsed as shallow */
 338                                 if (!parse_object(sha1))
 339                                         die("error in object: %s", line);
 340                                 if (unregister_shallow(sha1))
 341                                         die("no shallow found: %s", line);
 342                                 continue;
 343                         }
 344                         die("expected shallow/unshallow, got %s", line);
 345                 }
 346         } else if (!args.stateless_rpc)
 347                 send_request(fd[1], &req_buf);
 348
 349         if (!args.stateless_rpc) {
 350                 /* If we aren't using the stateless-rpc interface
 351                  * we don't need to retain the headers.
 352                  */
 353                 strbuf_setlen(&req_buf, 0);
 354                 state_len = 0;
 355         }
 356
 357         flushes = 0;
 358         retval = -1;
 359         while ((sha1 = get_rev())) {
 360                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 361                 if (args.verbose)
 362                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 363                 in_vain++;
 364                 if (flush_at <= ++count) {
 365                         int ack;
 366
 367                         packet_buf_flush(&req_buf);
 368                         send_request(fd[1], &req_buf);
 369                         strbuf_setlen(&req_buf, state_len);
 370                         flushes++;
 371                         flush_at = next_flush(count);
 372
 373                         /*
 374                          * We keep one window "ahead" of the other side, and
 375                          * will wait for an ACK only on the next one
 376                          */
 377                         if (!args.stateless_rpc && count == INITIAL_FLUSH)
 378                                 continue;
 379
 380                         consume_shallow_list(fd[0]);
 381                         do {
 382                                 ack = get_ack(fd[0], result_sha1);
 383                                 if (args.verbose && ack)
 384                                         fprintf(stderr, "got ack %d %s\n", ack,
 385                                                         sha1_to_hex(result_sha1));
 386                                 switch (ack) {
 387                                 case ACK:
 388                                         flushes = 0;
 389                                         multi_ack = 0;
 390                                         retval = 0;
 391                                         goto done;
 392                                 case ACK_common:
 393                                 case ACK_ready:
 394                                 case ACK_continue: {
 395                                         struct commit *commit =
 396                                                 lookup_commit(result_sha1);
 397                                         if (args.stateless_rpc
 398                                          && ack == ACK_common
 399                                          && !(commit->object.flags & COMMON)) {
 400                                                 /* We need to replay the have for this object
 401                                                  * on the next RPC request so the peer knows
 402                                                  * it is in common with us.
 403                                                  */
 404                                                 const char *hex = sha1_to_hex(result_sha1);
 405                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 406                                                 state_len = req_buf.len;
 407                                         }
 408                                         mark_common(commit, 0, 1);
 409                                         retval = 0;
 410                                         in_vain = 0;
 411                                         got_continue = 1;
 412                                         if (ack == ACK_ready) {
 413                                                 rev_list = NULL;
 414                                                 got_ready = 1;
 415                                         }
 416                                         break;
 417                                         }
 418                                 }
 419                         } while (ack);
 420                         flushes--;
 421                         if (got_continue && MAX_IN_VAIN < in_vain) {
 422                                 if (args.verbose)
 423                                         fprintf(stderr, "giving up\n");
 424                                 break; /* give up */
 425                         }
 426                 }
 427         }
 428 done:
 429         if (!got_ready || !no_done) {
 430                 packet_buf_write(&req_buf, "done\n");
 431                 send_request(fd[1], &req_buf);
 432         }
 433         if (args.verbose)
 434                 fprintf(stderr, "done\n");
 435         if (retval != 0) {
 436                 multi_ack = 0;
 437                 flushes++;
 438         }
 439         strbuf_release(&req_buf);
 440
 441         consume_shallow_list(fd[0]);
 442         while (flushes || multi_ack) {
 443                 int ack = get_ack(fd[0], result_sha1);
 444                 if (ack) {
 445                         if (args.verbose)
 446                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 447                                         sha1_to_hex(result_sha1));
 448                         if (ack == ACK)
 449                                 return 0;
 450                         multi_ack = 1;
 451                         continue;
 452                 }
 453                 flushes--;
 454         }
 455         /* it is no error to fetch into a completely empty repo */
 456         return count ? retval : 0;
 457 }
 458
 459 static struct commit_list *complete;
 460
 461 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 462 {
 463         struct object *o = parse_object(sha1);
 464
 465         while (o && o->type == OBJ_TAG) {
 466                 struct tag *t = (struct tag *) o;
 467                 if (!t->tagged)
 468                         break; /* broken repository */
 469                 o->flags |= COMPLETE;
 470                 o = parse_object(t->tagged->sha1);
 471         }
 472         if (o && o->type == OBJ_COMMIT) {
 473                 struct commit *commit = (struct commit *)o;
 474                 commit->object.flags |= COMPLETE;
 475                 commit_list_insert_by_date(commit, &complete);
 476         }
 477         return 0;
 478 }
 479
 480 static void mark_recent_complete_commits(unsigned long cutoff)
 481 {
 482         while (complete && cutoff <= complete->item->date) {
 483                 if (args.verbose)
 484                         fprintf(stderr, "Marking %s as complete\n",
 485                                 sha1_to_hex(complete->item->object.sha1));
 486                 pop_most_recent_commit(&complete, COMPLETE);
 487         }
 488 }
 489
 490 static void filter_refs(struct ref **refs, int nr_match, char **match)
 491 {
 492         struct ref **return_refs;
 493         struct ref *newlist = NULL;
 494         struct ref **newtail = &newlist;
 495         struct ref *ref, *next;
 496         struct ref *fastarray[32];
 497
 498         if (nr_match && !args.fetch_all) {
 499                 if (ARRAY_SIZE(fastarray) < nr_match)
 500                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 501                 else {
 502                         return_refs = fastarray;
 503                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 504                 }
 505         }
 506         else
 507                 return_refs = NULL;
 508
 509         for (ref = *refs; ref; ref = next) {
 510                 next = ref->next;
 511                 if (!memcmp(ref->name, "refs/", 5) &&
 512                     check_ref_format(ref->name + 5))
 513                         ; /* trash */
 514                 else if (args.fetch_all &&
 515                          (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
 516                         *newtail = ref;
 517                         ref->next = NULL;
 518                         newtail = &ref->next;
 519                         continue;
 520                 }
 521                 else {
 522                         int order = path_match(ref->name, nr_match, match);
 523                         if (order) {
 524                                 return_refs[order-1] = ref;
 525                                 continue; /* we will link it later */
 526                         }
 527                 }
 528                 free(ref);
 529         }
 530
 531         if (!args.fetch_all) {
 532                 int i;
 533                 for (i = 0; i < nr_match; i++) {
 534                         ref = return_refs[i];
 535                         if (ref) {
 536                                 *newtail = ref;
 537                                 ref->next = NULL;
 538                                 newtail = &ref->next;
 539                         }
 540                 }
 541                 if (return_refs != fastarray)
 542                         free(return_refs);
 543         }
 544         *refs = newlist;
 545 }
 546
 547 static int everything_local(struct ref **refs, int nr_match, char **match)
 548 {
 549         struct ref *ref;
 550         int retval;
 551         unsigned long cutoff = 0;
 552
 553         save_commit_buffer = 0;
 554
 555         for (ref = *refs; ref; ref = ref->next) {
 556                 struct object *o;
 557
 558                 o = parse_object(ref->old_sha1);
 559                 if (!o)
 560                         continue;
 561
 562                 /* We already have it -- which may mean that we were
 563                  * in sync with the other side at some time after
 564                  * that (it is OK if we guess wrong here).
 565                  */
 566                 if (o->type == OBJ_COMMIT) {
 567                         struct commit *commit = (struct commit *)o;
 568                         if (!cutoff || cutoff < commit->date)
 569                                 cutoff = commit->date;
 570                 }
 571         }
 572
 573         if (!args.depth) {
 574                 for_each_ref(mark_complete, NULL);
 575                 if (cutoff)
 576                         mark_recent_complete_commits(cutoff);
 577         }
 578
 579         /*
 580          * Mark all complete remote refs as common refs.
 581          * Don't mark them common yet; the server has to be told so first.
 582          */
 583         for (ref = *refs; ref; ref = ref->next) {
 584                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 585                                              NULL, 0);
 586
 587                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 588                         continue;
 589
 590                 if (!(o->flags & SEEN)) {
 591                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 592
 593                         mark_common((struct commit *)o, 1, 1);
 594                 }
 595         }
 596
 597         filter_refs(refs, nr_match, match);
 598
 599         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 600                 const unsigned char *remote = ref->old_sha1;
 601                 unsigned char local[20];
 602                 struct object *o;
 603
 604                 o = lookup_object(remote);
 605                 if (!o || !(o->flags & COMPLETE)) {
 606                         retval = 0;
 607                         if (!args.verbose)
 608                                 continue;
 609                         fprintf(stderr,
 610                                 "want %s (%s)\n", sha1_to_hex(remote),
 611                                 ref->name);
 612                         continue;
 613                 }
 614
 615                 hashcpy(ref->new_sha1, local);
 616                 if (!args.verbose)
 617                         continue;
 618                 fprintf(stderr,
 619                         "already have %s (%s)\n", sha1_to_hex(remote),
 620                         ref->name);
 621         }
 622         return retval;
 623 }
 624
 625 static int sideband_demux(int in, int out, void *data)
 626 {
 627         int *xd = data;
 628
 629         int ret = recv_sideband("fetch-pack", xd[0], out);
 630         close(out);
 631         return ret;
 632 }
 633
 634 static int get_pack(int xd[2], char **pack_lockfile)
 635 {
 636         struct async demux;
 637         const char *argv[20];
 638         char keep_arg[256];
 639         char hdr_arg[256];
 640         const char **av;
 641         int do_keep = args.keep_pack;
 642         struct child_process cmd;
 643
 644         memset(&demux, 0, sizeof(demux));
 645         if (use_sideband) {
 646                 /* xd[] is talking with upload-pack; subprocess reads from
 647                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 648                  * through demux->out.
 649                  */
 650                 demux.proc = sideband_demux;
 651                 demux.data = xd;
 652                 demux.out = -1;
 653                 if (start_async(&demux))
 654                         die("fetch-pack: unable to fork off sideband"
 655                             " demultiplexer");
 656         }
 657         else
 658                 demux.out = xd[0];
 659
 660         memset(&cmd, 0, sizeof(cmd));
 661         cmd.argv = argv;
 662         av = argv;
 663         *hdr_arg = 0;
 664         if (!args.keep_pack && unpack_limit) {
 665                 struct pack_header header;
 666
 667                 if (read_pack_header(demux.out, &header))
 668                         die("protocol error: bad pack header");
 669                 snprintf(hdr_arg, sizeof(hdr_arg),
 670                          "--pack_header=%"PRIu32",%"PRIu32,
 671                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 672                 if (ntohl(header.hdr_entries) < unpack_limit)
 673                         do_keep = 0;
 674                 else
 675                         do_keep = 1;
 676         }
 677
 678         if (do_keep) {
 679                 if (pack_lockfile)
 680                         cmd.out = -1;
 681                 *av++ = "index-pack";
 682                 *av++ = "--stdin";
 683                 if (!args.quiet && !args.no_progress)
 684                         *av++ = "-v";
 685                 if (args.use_thin_pack)
 686                         *av++ = "--fix-thin";
 687                 if (args.lock_pack || unpack_limit) {
 688                         int s = sprintf(keep_arg,
 689                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 690                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 691                                 strcpy(keep_arg + s, "localhost");
 692                         *av++ = keep_arg;
 693                 }
 694         }
 695         else {
 696                 *av++ = "unpack-objects";
 697                 if (args.quiet)
 698                         *av++ = "-q";
 699         }
 700         if (*hdr_arg)
 701                 *av++ = hdr_arg;
 702         *av++ = NULL;
 703
 704         cmd.in = demux.out;
 705         cmd.git_cmd = 1;
 706         if (start_command(&cmd))
 707                 die("fetch-pack: unable to fork off %s", argv[0]);
 708         if (do_keep && pack_lockfile) {
 709                 *pack_lockfile = index_pack_lockfile(cmd.out);
 710                 close(cmd.out);
 711         }
 712
 713         if (finish_command(&cmd))
 714                 die("%s failed", argv[0]);
 715         if (use_sideband && finish_async(&demux))
 716                 die("error in sideband demultiplexer");
 717         return 0;
 718 }
 719
 720 static struct ref *do_fetch_pack(int fd[2],
 721                 const struct ref *orig_ref,
 722                 int nr_match,
 723                 char **match,
 724                 char **pack_lockfile)
 725 {
 726         struct ref *ref = copy_ref_list(orig_ref);
 727         unsigned char sha1[20];
 728
 729         if (is_repository_shallow() && !server_supports("shallow"))
 730                 die("Server does not support shallow clients");
 731         if (server_supports("multi_ack_detailed")) {
 732                 if (args.verbose)
 733                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 734                 multi_ack = 2;
 735                 if (server_supports("no-done")) {
 736                         if (args.verbose)
 737                                 fprintf(stderr, "Server supports no-done\n");
 738                         if (args.stateless_rpc)
 739                                 no_done = 1;
 740                 }
 741         }
 742         else if (server_supports("multi_ack")) {
 743                 if (args.verbose)
 744                         fprintf(stderr, "Server supports multi_ack\n");
 745                 multi_ack = 1;
 746         }
 747         if (server_supports("side-band-64k")) {
 748                 if (args.verbose)
 749                         fprintf(stderr, "Server supports side-band-64k\n");
 750                 use_sideband = 2;
 751         }
 752         else if (server_supports("side-band")) {
 753                 if (args.verbose)
 754                         fprintf(stderr, "Server supports side-band\n");
 755                 use_sideband = 1;
 756         }
 757         if (server_supports("ofs-delta")) {
 758                 if (args.verbose)
 759                         fprintf(stderr, "Server supports ofs-delta\n");
 760         } else
 761                 prefer_ofs_delta = 0;
 762         if (everything_local(&ref, nr_match, match)) {
 763                 packet_flush(fd[1]);
 764                 goto all_done;
 765         }
 766         if (find_common(fd, sha1, ref) < 0)
 767                 if (!args.keep_pack)
 768                         /* When cloning, it is not unusual to have
 769                          * no common commit.
 770                          */
 771                         warning("no common commits");
 772
 773         if (args.stateless_rpc)
 774                 packet_flush(fd[1]);
 775         if (get_pack(fd, pack_lockfile))
 776                 die("git fetch-pack: fetch failed.");
 777
 778  all_done:
 779         return ref;
 780 }
 781
 782 static int remove_duplicates(int nr_heads, char **heads)
 783 {
 784         int src, dst;
 785
 786         for (src = dst = 0; src < nr_heads; src++) {
 787                 /* If heads[src] is different from any of
 788                  * heads[0..dst], push it in.
 789                  */
 790                 int i;
 791                 for (i = 0; i < dst; i++) {
 792                         if (!strcmp(heads[i], heads[src]))
 793                                 break;
 794                 }
 795                 if (i < dst)
 796                         continue;
 797                 if (src != dst)
 798                         heads[dst] = heads[src];
 799                 dst++;
 800         }
 801         return dst;
 802 }
 803
 804 static int fetch_pack_config(const char *var, const char *value, void *cb)
 805 {
 806         if (strcmp(var, "fetch.unpacklimit") == 0) {
 807                 fetch_unpack_limit = git_config_int(var, value);
 808                 return 0;
 809         }
 810
 811         if (strcmp(var, "transfer.unpacklimit") == 0) {
 812                 transfer_unpack_limit = git_config_int(var, value);
 813                 return 0;
 814         }
 815
 816         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 817                 prefer_ofs_delta = git_config_bool(var, value);
 818                 return 0;
 819         }
 820
 821         return git_default_config(var, value, cb);
 822 }
 823
 824 static struct lock_file lock;
 825
 826 static void fetch_pack_setup(void)
 827 {
 828         static int did_setup;
 829         if (did_setup)
 830                 return;
 831         git_config(fetch_pack_config, NULL);
 832         if (0 <= transfer_unpack_limit)
 833                 unpack_limit = transfer_unpack_limit;
 834         else if (0 <= fetch_unpack_limit)
 835                 unpack_limit = fetch_unpack_limit;
 836         did_setup = 1;
 837 }
 838
 839 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 840 {
 841         int i, ret, nr_heads;
 842         struct ref *ref = NULL;
 843         char *dest = NULL, **heads;
 844         int fd[2];
 845         char *pack_lockfile = NULL;
 846         char **pack_lockfile_ptr = NULL;
 847         struct child_process *conn;
 848
 849         packet_trace_identity("fetch-pack");
 850
 851         nr_heads = 0;
 852         heads = NULL;
 853         for (i = 1; i < argc; i++) {
 854                 const char *arg = argv[i];
 855
 856                 if (*arg == '-') {
 857                         if (!prefixcmp(arg, "--upload-pack=")) {
 858                                 args.uploadpack = arg + 14;
 859                                 continue;
 860                         }
 861                         if (!prefixcmp(arg, "--exec=")) {
 862                                 args.uploadpack = arg + 7;
 863                                 continue;
 864                         }
 865                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 866                                 args.quiet = 1;
 867                                 continue;
 868                         }
 869                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 870                                 args.lock_pack = args.keep_pack;
 871                                 args.keep_pack = 1;
 872                                 continue;
 873                         }
 874                         if (!strcmp("--thin", arg)) {
 875                                 args.use_thin_pack = 1;
 876                                 continue;
 877                         }
 878                         if (!strcmp("--include-tag", arg)) {
 879                                 args.include_tag = 1;
 880                                 continue;
 881                         }
 882                         if (!strcmp("--all", arg)) {
 883                                 args.fetch_all = 1;
 884                                 continue;
 885                         }
 886                         if (!strcmp("-v", arg)) {
 887                                 args.verbose = 1;
 888                                 continue;
 889                         }
 890                         if (!prefixcmp(arg, "--depth=")) {
 891                                 args.depth = strtol(arg + 8, NULL, 0);
 892                                 continue;
 893                         }
 894                         if (!strcmp("--no-progress", arg)) {
 895                                 args.no_progress = 1;
 896                                 continue;
 897                         }
 898                         if (!strcmp("--stateless-rpc", arg)) {
 899                                 args.stateless_rpc = 1;
 900                                 continue;
 901                         }
 902                         if (!strcmp("--lock-pack", arg)) {
 903                                 args.lock_pack = 1;
 904                                 pack_lockfile_ptr = &pack_lockfile;
 905                                 continue;
 906                         }
 907                         usage(fetch_pack_usage);
 908                 }
 909                 dest = (char *)arg;
 910                 heads = (char **)(argv + i + 1);
 911                 nr_heads = argc - i - 1;
 912                 break;
 913         }
 914         if (!dest)
 915                 usage(fetch_pack_usage);
 916
 917         if (args.stateless_rpc) {
 918                 conn = NULL;
 919                 fd[0] = 0;
 920                 fd[1] = 1;
 921         } else {
 922                 conn = git_connect(fd, (char *)dest, args.uploadpack,
 923                                    args.verbose ? CONNECT_VERBOSE : 0);
 924         }
 925
 926         get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL);
 927
 928         ref = fetch_pack(&args, fd, conn, ref, dest,
 929                 nr_heads, heads, pack_lockfile_ptr);
 930         if (pack_lockfile) {
 931                 printf("lock %s\n", pack_lockfile);
 932                 fflush(stdout);
 933         }
 934         close(fd[0]);
 935         close(fd[1]);
 936         if (finish_connect(conn))
 937                 ref = NULL;
 938         ret = !ref;
 939
 940         if (!ret && nr_heads) {
 941                 /* If the heads to pull were given, we should have
 942                  * consumed all of them by matching the remote.
 943                  * Otherwise, 'git fetch remote no-such-ref' would
 944                  * silently succeed without issuing an error.
 945                  */
 946                 for (i = 0; i < nr_heads; i++)
 947                         if (heads[i] && heads[i][0]) {
 948                                 error("no such remote ref %s", heads[i]);
 949                                 ret = 1;
 950                         }
 951         }
 952         while (ref) {
 953                 printf("%s %s\n",
 954                        sha1_to_hex(ref->old_sha1), ref->name);
 955                 ref = ref->next;
 956         }
 957
 958         return ret;
 959 }
 960
 961 struct ref *fetch_pack(struct fetch_pack_args *my_args,
 962                        int fd[], struct child_process *conn,
 963                        const struct ref *ref,
 964                 const char *dest,
 965                 int nr_heads,
 966                 char **heads,
 967                 char **pack_lockfile)
 968 {
 969         struct stat st;
 970         struct ref *ref_cpy;
 971
 972         fetch_pack_setup();
 973         if (&args != my_args)
 974                 memcpy(&args, my_args, sizeof(args));
 975         if (args.depth > 0) {
 976                 if (stat(git_path("shallow"), &st))
 977                         st.st_mtime = 0;
 978         }
 979
 980         if (heads && nr_heads)
 981                 nr_heads = remove_duplicates(nr_heads, heads);
 982         if (!ref) {
 983                 packet_flush(fd[1]);
 984                 die("no matching remote head");
 985         }
 986         ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
 987
 988         if (args.depth > 0) {
 989                 struct cache_time mtime;
 990                 struct strbuf sb = STRBUF_INIT;
 991                 char *shallow = git_path("shallow");
 992                 int fd;
 993
 994                 mtime.sec = st.st_mtime;
 995                 mtime.nsec = ST_MTIME_NSEC(st);
 996                 if (stat(shallow, &st)) {
 997                         if (mtime.sec)
 998                                 die("shallow file was removed during fetch");
 999                 } else if (st.st_mtime != mtime.sec
1000 #ifdef USE_NSEC
1001                                 || ST_MTIME_NSEC(st) != mtime.nsec
1002 #endif
1003                           )
1004                         die("shallow file was changed during fetch");
1005
1006                 fd = hold_lock_file_for_update(&lock, shallow,
1007                                                LOCK_DIE_ON_ERROR);
1008                 if (!write_shallow_commits(&sb, 0)
1009                  || write_in_full(fd, sb.buf, sb.len) != sb.len) {
1010                         unlink_or_warn(shallow);
1011                         rollback_lock_file(&lock);
1012                 } else {
1013                         commit_lock_file(&lock);
1014                 }
1015                 strbuf_release(&sb);
1016         }
1017
1018         reprepare_packed_git();
1019         return ref_cpy;
1020 }