builtin/fetch-pack.c

   1 #include "builtin.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "transport.h"
  13
  14 static int transfer_unpack_limit = -1;
  15 static int fetch_unpack_limit = -1;
  16 static int unpack_limit = 100;
  17 static int prefer_ofs_delta = 1;
  18 static int no_done = 0;
  19 static struct fetch_pack_args args = {
  20         /* .uploadpack = */ "git-upload-pack",
  21 };
  22
  23 static const char fetch_pack_usage[] =
  24 "git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
  25
  26 #define COMPLETE        (1U << 0)
  27 #define COMMON          (1U << 1)
  28 #define COMMON_REF      (1U << 2)
  29 #define SEEN            (1U << 3)
  30 #define POPPED          (1U << 4)
  31
  32 static int marked;
  33
  34 /*
  35  * After sending this many "have"s if we do not get any new ACK , we
  36  * give up traversing our history.
  37  */
  38 #define MAX_IN_VAIN 256
  39
  40 static struct commit_list *rev_list;
  41 static int non_common_revs, multi_ack, use_sideband;
  42
  43 static void rev_list_push(struct commit *commit, int mark)
  44 {
  45         if (!(commit->object.flags & mark)) {
  46                 commit->object.flags |= mark;
  47
  48                 if (!(commit->object.parsed))
  49                         if (parse_commit(commit))
  50                                 return;
  51
  52                 commit_list_insert_by_date(commit, &rev_list);
  53
  54                 if (!(commit->object.flags & COMMON))
  55                         non_common_revs++;
  56         }
  57 }
  58
  59 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  60 {
  61         struct object *o = deref_tag(parse_object(sha1), path, 0);
  62
  63         if (o && o->type == OBJ_COMMIT)
  64                 rev_list_push((struct commit *)o, SEEN);
  65
  66         return 0;
  67 }
  68
  69 static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  70 {
  71         struct object *o = deref_tag(parse_object(sha1), path, 0);
  72
  73         if (o && o->type == OBJ_COMMIT)
  74                 clear_commit_marks((struct commit *)o,
  75                                    COMMON | COMMON_REF | SEEN | POPPED);
  76         return 0;
  77 }
  78
  79 /*
  80    This function marks a rev and its ancestors as common.
  81    In some cases, it is desirable to mark only the ancestors (for example
  82    when only the server does not yet know that they are common).
  83 */
  84
  85 static void mark_common(struct commit *commit,
  86                 int ancestors_only, int dont_parse)
  87 {
  88         if (commit != NULL && !(commit->object.flags & COMMON)) {
  89                 struct object *o = (struct object *)commit;
  90
  91                 if (!ancestors_only)
  92                         o->flags |= COMMON;
  93
  94                 if (!(o->flags & SEEN))
  95                         rev_list_push(commit, SEEN);
  96                 else {
  97                         struct commit_list *parents;
  98
  99                         if (!ancestors_only && !(o->flags & POPPED))
 100                                 non_common_revs--;
 101                         if (!o->parsed && !dont_parse)
 102                                 if (parse_commit(commit))
 103                                         return;
 104
 105                         for (parents = commit->parents;
 106                                         parents;
 107                                         parents = parents->next)
 108                                 mark_common(parents->item, 0, dont_parse);
 109                 }
 110         }
 111 }
 112
 113 /*
 114   Get the next rev to send, ignoring the common.
 115 */
 116
 117 static const unsigned char *get_rev(void)
 118 {
 119         struct commit *commit = NULL;
 120
 121         while (commit == NULL) {
 122                 unsigned int mark;
 123                 struct commit_list *parents;
 124
 125                 if (rev_list == NULL || non_common_revs == 0)
 126                         return NULL;
 127
 128                 commit = rev_list->item;
 129                 if (!commit->object.parsed)
 130                         parse_commit(commit);
 131                 parents = commit->parents;
 132
 133                 commit->object.flags |= POPPED;
 134                 if (!(commit->object.flags & COMMON))
 135                         non_common_revs--;
 136
 137                 if (commit->object.flags & COMMON) {
 138                         /* do not send "have", and ignore ancestors */
 139                         commit = NULL;
 140                         mark = COMMON | SEEN;
 141                 } else if (commit->object.flags & COMMON_REF)
 142                         /* send "have", and ignore ancestors */
 143                         mark = COMMON | SEEN;
 144                 else
 145                         /* send "have", also for its ancestors */
 146                         mark = SEEN;
 147
 148                 while (parents) {
 149                         if (!(parents->item->object.flags & SEEN))
 150                                 rev_list_push(parents->item, mark);
 151                         if (mark & COMMON)
 152                                 mark_common(parents->item, 1, 0);
 153                         parents = parents->next;
 154                 }
 155
 156                 rev_list = rev_list->next;
 157         }
 158
 159         return commit->object.sha1;
 160 }
 161
 162 enum ack_type {
 163         NAK = 0,
 164         ACK,
 165         ACK_continue,
 166         ACK_common,
 167         ACK_ready
 168 };
 169
 170 static void consume_shallow_list(int fd)
 171 {
 172         if (args.stateless_rpc && args.depth > 0) {
 173                 /* If we sent a depth we will get back "duplicate"
 174                  * shallow and unshallow commands every time there
 175                  * is a block of have lines exchanged.
 176                  */
 177                 char line[1000];
 178                 while (packet_read_line(fd, line, sizeof(line))) {
 179                         if (!prefixcmp(line, "shallow "))
 180                                 continue;
 181                         if (!prefixcmp(line, "unshallow "))
 182                                 continue;
 183                         die("git fetch-pack: expected shallow list");
 184                 }
 185         }
 186 }
 187
 188 struct write_shallow_data {
 189         struct strbuf *out;
 190         int use_pack_protocol;
 191         int count;
 192 };
 193
 194 static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
 195 {
 196         struct write_shallow_data *data = cb_data;
 197         const char *hex = sha1_to_hex(graft->sha1);
 198         data->count++;
 199         if (data->use_pack_protocol)
 200                 packet_buf_write(data->out, "shallow %s", hex);
 201         else {
 202                 strbuf_addstr(data->out, hex);
 203                 strbuf_addch(data->out, '\n');
 204         }
 205         return 0;
 206 }
 207
 208 static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
 209 {
 210         struct write_shallow_data data;
 211         data.out = out;
 212         data.use_pack_protocol = use_pack_protocol;
 213         data.count = 0;
 214         for_each_commit_graft(write_one_shallow, &data);
 215         return data.count;
 216 }
 217
 218 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 219 {
 220         static char line[1000];
 221         int len = packet_read_line(fd, line, sizeof(line));
 222
 223         if (!len)
 224                 die("git fetch-pack: expected ACK/NAK, got EOF");
 225         if (line[len-1] == '\n')
 226                 line[--len] = 0;
 227         if (!strcmp(line, "NAK"))
 228                 return NAK;
 229         if (!prefixcmp(line, "ACK ")) {
 230                 if (!get_sha1_hex(line+4, result_sha1)) {
 231                         if (strstr(line+45, "continue"))
 232                                 return ACK_continue;
 233                         if (strstr(line+45, "common"))
 234                                 return ACK_common;
 235                         if (strstr(line+45, "ready"))
 236                                 return ACK_ready;
 237                         return ACK;
 238                 }
 239         }
 240         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 241 }
 242
 243 static void send_request(int fd, struct strbuf *buf)
 244 {
 245         if (args.stateless_rpc) {
 246                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 247                 packet_flush(fd);
 248         } else
 249                 safe_write(fd, buf->buf, buf->len);
 250 }
 251
 252 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 253 {
 254         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 255 }
 256
 257 static void insert_alternate_refs(void)
 258 {
 259         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 260 }
 261
 262 #define INITIAL_FLUSH 16
 263 #define PIPESAFE_FLUSH 32
 264 #define LARGE_FLUSH 1024
 265
 266 static int next_flush(int count)
 267 {
 268         int flush_limit = args.stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 269
 270         if (count < flush_limit)
 271                 count <<= 1;
 272         else
 273                 count += flush_limit;
 274         return count;
 275 }
 276
 277 static int find_common(int fd[2], unsigned char *result_sha1,
 278                        struct ref *refs)
 279 {
 280         int fetching;
 281         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 282         const unsigned char *sha1;
 283         unsigned in_vain = 0;
 284         int got_continue = 0;
 285         int got_ready = 0;
 286         struct strbuf req_buf = STRBUF_INIT;
 287         size_t state_len = 0;
 288
 289         if (args.stateless_rpc && multi_ack == 1)
 290                 die("--stateless-rpc requires multi_ack_detailed");
 291         if (marked)
 292                 for_each_ref(clear_marks, NULL);
 293         marked = 1;
 294
 295         for_each_ref(rev_list_insert_ref, NULL);
 296         insert_alternate_refs();
 297
 298         fetching = 0;
 299         for ( ; refs ; refs = refs->next) {
 300                 unsigned char *remote = refs->old_sha1;
 301                 const char *remote_hex;
 302                 struct object *o;
 303
 304                 /*
 305                  * If that object is complete (i.e. it is an ancestor of a
 306                  * local ref), we tell them we have it but do not have to
 307                  * tell them about its ancestors, which they already know
 308                  * about.
 309                  *
 310                  * We use lookup_object here because we are only
 311                  * interested in the case we *know* the object is
 312                  * reachable and we have already scanned it.
 313                  */
 314                 if (((o = lookup_object(remote)) != NULL) &&
 315                                 (o->flags & COMPLETE)) {
 316                         continue;
 317                 }
 318
 319                 remote_hex = sha1_to_hex(remote);
 320                 if (!fetching) {
 321                         struct strbuf c = STRBUF_INIT;
 322                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 323                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 324                         if (no_done)            strbuf_addstr(&c, " no-done");
 325                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 326                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 327                         if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack");
 328                         if (args.no_progress)   strbuf_addstr(&c, " no-progress");
 329                         if (args.include_tag)   strbuf_addstr(&c, " include-tag");
 330                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 331                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 332                         strbuf_release(&c);
 333                 } else
 334                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 335                 fetching++;
 336         }
 337
 338         if (!fetching) {
 339                 strbuf_release(&req_buf);
 340                 packet_flush(fd[1]);
 341                 return 1;
 342         }
 343
 344         if (is_repository_shallow())
 345                 write_shallow_commits(&req_buf, 1);
 346         if (args.depth > 0)
 347                 packet_buf_write(&req_buf, "deepen %d", args.depth);
 348         packet_buf_flush(&req_buf);
 349         state_len = req_buf.len;
 350
 351         if (args.depth > 0) {
 352                 char line[1024];
 353                 unsigned char sha1[20];
 354
 355                 send_request(fd[1], &req_buf);
 356                 while (packet_read_line(fd[0], line, sizeof(line))) {
 357                         if (!prefixcmp(line, "shallow ")) {
 358                                 if (get_sha1_hex(line + 8, sha1))
 359                                         die("invalid shallow line: %s", line);
 360                                 register_shallow(sha1);
 361                                 continue;
 362                         }
 363                         if (!prefixcmp(line, "unshallow ")) {
 364                                 if (get_sha1_hex(line + 10, sha1))
 365                                         die("invalid unshallow line: %s", line);
 366                                 if (!lookup_object(sha1))
 367                                         die("object not found: %s", line);
 368                                 /* make sure that it is parsed as shallow */
 369                                 if (!parse_object(sha1))
 370                                         die("error in object: %s", line);
 371                                 if (unregister_shallow(sha1))
 372                                         die("no shallow found: %s", line);
 373                                 continue;
 374                         }
 375                         die("expected shallow/unshallow, got %s", line);
 376                 }
 377         } else if (!args.stateless_rpc)
 378                 send_request(fd[1], &req_buf);
 379
 380         if (!args.stateless_rpc) {
 381                 /* If we aren't using the stateless-rpc interface
 382                  * we don't need to retain the headers.
 383                  */
 384                 strbuf_setlen(&req_buf, 0);
 385                 state_len = 0;
 386         }
 387
 388         flushes = 0;
 389         retval = -1;
 390         while ((sha1 = get_rev())) {
 391                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 392                 if (args.verbose)
 393                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 394                 in_vain++;
 395                 if (flush_at <= ++count) {
 396                         int ack;
 397
 398                         packet_buf_flush(&req_buf);
 399                         send_request(fd[1], &req_buf);
 400                         strbuf_setlen(&req_buf, state_len);
 401                         flushes++;
 402                         flush_at = next_flush(count);
 403
 404                         /*
 405                          * We keep one window "ahead" of the other side, and
 406                          * will wait for an ACK only on the next one
 407                          */
 408                         if (!args.stateless_rpc && count == INITIAL_FLUSH)
 409                                 continue;
 410
 411                         consume_shallow_list(fd[0]);
 412                         do {
 413                                 ack = get_ack(fd[0], result_sha1);
 414                                 if (args.verbose && ack)
 415                                         fprintf(stderr, "got ack %d %s\n", ack,
 416                                                         sha1_to_hex(result_sha1));
 417                                 switch (ack) {
 418                                 case ACK:
 419                                         flushes = 0;
 420                                         multi_ack = 0;
 421                                         retval = 0;
 422                                         goto done;
 423                                 case ACK_common:
 424                                 case ACK_ready:
 425                                 case ACK_continue: {
 426                                         struct commit *commit =
 427                                                 lookup_commit(result_sha1);
 428                                         if (args.stateless_rpc
 429                                          && ack == ACK_common
 430                                          && !(commit->object.flags & COMMON)) {
 431                                                 /* We need to replay the have for this object
 432                                                  * on the next RPC request so the peer knows
 433                                                  * it is in common with us.
 434                                                  */
 435                                                 const char *hex = sha1_to_hex(result_sha1);
 436                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 437                                                 state_len = req_buf.len;
 438                                         }
 439                                         mark_common(commit, 0, 1);
 440                                         retval = 0;
 441                                         in_vain = 0;
 442                                         got_continue = 1;
 443                                         if (ack == ACK_ready) {
 444                                                 rev_list = NULL;
 445                                                 got_ready = 1;
 446                                         }
 447                                         break;
 448                                         }
 449                                 }
 450                         } while (ack);
 451                         flushes--;
 452                         if (got_continue && MAX_IN_VAIN < in_vain) {
 453                                 if (args.verbose)
 454                                         fprintf(stderr, "giving up\n");
 455                                 break; /* give up */
 456                         }
 457                 }
 458         }
 459 done:
 460         if (!got_ready || !no_done) {
 461                 packet_buf_write(&req_buf, "done\n");
 462                 send_request(fd[1], &req_buf);
 463         }
 464         if (args.verbose)
 465                 fprintf(stderr, "done\n");
 466         if (retval != 0) {
 467                 multi_ack = 0;
 468                 flushes++;
 469         }
 470         strbuf_release(&req_buf);
 471
 472         consume_shallow_list(fd[0]);
 473         while (flushes || multi_ack) {
 474                 int ack = get_ack(fd[0], result_sha1);
 475                 if (ack) {
 476                         if (args.verbose)
 477                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 478                                         sha1_to_hex(result_sha1));
 479                         if (ack == ACK)
 480                                 return 0;
 481                         multi_ack = 1;
 482                         continue;
 483                 }
 484                 flushes--;
 485         }
 486         /* it is no error to fetch into a completely empty repo */
 487         return count ? retval : 0;
 488 }
 489
 490 static struct commit_list *complete;
 491
 492 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 493 {
 494         struct object *o = parse_object(sha1);
 495
 496         while (o && o->type == OBJ_TAG) {
 497                 struct tag *t = (struct tag *) o;
 498                 if (!t->tagged)
 499                         break; /* broken repository */
 500                 o->flags |= COMPLETE;
 501                 o = parse_object(t->tagged->sha1);
 502         }
 503         if (o && o->type == OBJ_COMMIT) {
 504                 struct commit *commit = (struct commit *)o;
 505                 if (!(commit->object.flags & COMPLETE)) {
 506                         commit->object.flags |= COMPLETE;
 507                         commit_list_insert_by_date(commit, &complete);
 508                 }
 509         }
 510         return 0;
 511 }
 512
 513 static void mark_recent_complete_commits(unsigned long cutoff)
 514 {
 515         while (complete && cutoff <= complete->item->date) {
 516                 if (args.verbose)
 517                         fprintf(stderr, "Marking %s as complete\n",
 518                                 sha1_to_hex(complete->item->object.sha1));
 519                 pop_most_recent_commit(&complete, COMPLETE);
 520         }
 521 }
 522
 523 static void filter_refs(struct ref **refs, int nr_match, char **match)
 524 {
 525         struct ref **return_refs;
 526         struct ref *newlist = NULL;
 527         struct ref **newtail = &newlist;
 528         struct ref *ref, *next;
 529         struct ref *fastarray[32];
 530
 531         if (nr_match && !args.fetch_all) {
 532                 if (ARRAY_SIZE(fastarray) < nr_match)
 533                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 534                 else {
 535                         return_refs = fastarray;
 536                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 537                 }
 538         }
 539         else
 540                 return_refs = NULL;
 541
 542         for (ref = *refs; ref; ref = next) {
 543                 next = ref->next;
 544                 if (!memcmp(ref->name, "refs/", 5) &&
 545                     check_ref_format(ref->name + 5))
 546                         ; /* trash */
 547                 else if (args.fetch_all &&
 548                          (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
 549                         *newtail = ref;
 550                         ref->next = NULL;
 551                         newtail = &ref->next;
 552                         continue;
 553                 }
 554                 else {
 555                         int order = path_match(ref->name, nr_match, match);
 556                         if (order) {
 557                                 return_refs[order-1] = ref;
 558                                 continue; /* we will link it later */
 559                         }
 560                 }
 561                 free(ref);
 562         }
 563
 564         if (!args.fetch_all) {
 565                 int i;
 566                 for (i = 0; i < nr_match; i++) {
 567                         ref = return_refs[i];
 568                         if (ref) {
 569                                 *newtail = ref;
 570                                 ref->next = NULL;
 571                                 newtail = &ref->next;
 572                         }
 573                 }
 574                 if (return_refs != fastarray)
 575                         free(return_refs);
 576         }
 577         *refs = newlist;
 578 }
 579
 580 static int everything_local(struct ref **refs, int nr_match, char **match)
 581 {
 582         struct ref *ref;
 583         int retval;
 584         unsigned long cutoff = 0;
 585
 586         save_commit_buffer = 0;
 587
 588         for (ref = *refs; ref; ref = ref->next) {
 589                 struct object *o;
 590
 591                 o = parse_object(ref->old_sha1);
 592                 if (!o)
 593                         continue;
 594
 595                 /* We already have it -- which may mean that we were
 596                  * in sync with the other side at some time after
 597                  * that (it is OK if we guess wrong here).
 598                  */
 599                 if (o->type == OBJ_COMMIT) {
 600                         struct commit *commit = (struct commit *)o;
 601                         if (!cutoff || cutoff < commit->date)
 602                                 cutoff = commit->date;
 603                 }
 604         }
 605
 606         if (!args.depth) {
 607                 for_each_ref(mark_complete, NULL);
 608                 if (cutoff)
 609                         mark_recent_complete_commits(cutoff);
 610         }
 611
 612         /*
 613          * Mark all complete remote refs as common refs.
 614          * Don't mark them common yet; the server has to be told so first.
 615          */
 616         for (ref = *refs; ref; ref = ref->next) {
 617                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 618                                              NULL, 0);
 619
 620                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 621                         continue;
 622
 623                 if (!(o->flags & SEEN)) {
 624                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 625
 626                         mark_common((struct commit *)o, 1, 1);
 627                 }
 628         }
 629
 630         filter_refs(refs, nr_match, match);
 631
 632         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 633                 const unsigned char *remote = ref->old_sha1;
 634                 unsigned char local[20];
 635                 struct object *o;
 636
 637                 o = lookup_object(remote);
 638                 if (!o || !(o->flags & COMPLETE)) {
 639                         retval = 0;
 640                         if (!args.verbose)
 641                                 continue;
 642                         fprintf(stderr,
 643                                 "want %s (%s)\n", sha1_to_hex(remote),
 644                                 ref->name);
 645                         continue;
 646                 }
 647
 648                 hashcpy(ref->new_sha1, local);
 649                 if (!args.verbose)
 650                         continue;
 651                 fprintf(stderr,
 652                         "already have %s (%s)\n", sha1_to_hex(remote),
 653                         ref->name);
 654         }
 655         return retval;
 656 }
 657
 658 static int sideband_demux(int in, int out, void *data)
 659 {
 660         int *xd = data;
 661
 662         int ret = recv_sideband("fetch-pack", xd[0], out);
 663         close(out);
 664         return ret;
 665 }
 666
 667 static int get_pack(int xd[2], char **pack_lockfile)
 668 {
 669         struct async demux;
 670         const char *argv[20];
 671         char keep_arg[256];
 672         char hdr_arg[256];
 673         const char **av;
 674         int do_keep = args.keep_pack;
 675         struct child_process cmd;
 676
 677         memset(&demux, 0, sizeof(demux));
 678         if (use_sideband) {
 679                 /* xd[] is talking with upload-pack; subprocess reads from
 680                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 681                  * through demux->out.
 682                  */
 683                 demux.proc = sideband_demux;
 684                 demux.data = xd;
 685                 demux.out = -1;
 686                 if (start_async(&demux))
 687                         die("fetch-pack: unable to fork off sideband"
 688                             " demultiplexer");
 689         }
 690         else
 691                 demux.out = xd[0];
 692
 693         memset(&cmd, 0, sizeof(cmd));
 694         cmd.argv = argv;
 695         av = argv;
 696         *hdr_arg = 0;
 697         if (!args.keep_pack && unpack_limit) {
 698                 struct pack_header header;
 699
 700                 if (read_pack_header(demux.out, &header))
 701                         die("protocol error: bad pack header");
 702                 snprintf(hdr_arg, sizeof(hdr_arg),
 703                          "--pack_header=%"PRIu32",%"PRIu32,
 704                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 705                 if (ntohl(header.hdr_entries) < unpack_limit)
 706                         do_keep = 0;
 707                 else
 708                         do_keep = 1;
 709         }
 710
 711         if (do_keep) {
 712                 if (pack_lockfile)
 713                         cmd.out = -1;
 714                 *av++ = "index-pack";
 715                 *av++ = "--stdin";
 716                 if (!args.quiet && !args.no_progress)
 717                         *av++ = "-v";
 718                 if (args.use_thin_pack)
 719                         *av++ = "--fix-thin";
 720                 if (args.lock_pack || unpack_limit) {
 721                         int s = sprintf(keep_arg,
 722                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 723                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 724                                 strcpy(keep_arg + s, "localhost");
 725                         *av++ = keep_arg;
 726                 }
 727         }
 728         else {
 729                 *av++ = "unpack-objects";
 730                 if (args.quiet)
 731                         *av++ = "-q";
 732         }
 733         if (*hdr_arg)
 734                 *av++ = hdr_arg;
 735         *av++ = NULL;
 736
 737         cmd.in = demux.out;
 738         cmd.git_cmd = 1;
 739         if (start_command(&cmd))
 740                 die("fetch-pack: unable to fork off %s", argv[0]);
 741         if (do_keep && pack_lockfile) {
 742                 *pack_lockfile = index_pack_lockfile(cmd.out);
 743                 close(cmd.out);
 744         }
 745
 746         if (finish_command(&cmd))
 747                 die("%s failed", argv[0]);
 748         if (use_sideband && finish_async(&demux))
 749                 die("error in sideband demultiplexer");
 750         return 0;
 751 }
 752
 753 static struct ref *do_fetch_pack(int fd[2],
 754                 const struct ref *orig_ref,
 755                 int nr_match,
 756                 char **match,
 757                 char **pack_lockfile)
 758 {
 759         struct ref *ref = copy_ref_list(orig_ref);
 760         unsigned char sha1[20];
 761
 762         if (is_repository_shallow() && !server_supports("shallow"))
 763                 die("Server does not support shallow clients");
 764         if (server_supports("multi_ack_detailed")) {
 765                 if (args.verbose)
 766                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 767                 multi_ack = 2;
 768                 if (server_supports("no-done")) {
 769                         if (args.verbose)
 770                                 fprintf(stderr, "Server supports no-done\n");
 771                         if (args.stateless_rpc)
 772                                 no_done = 1;
 773                 }
 774         }
 775         else if (server_supports("multi_ack")) {
 776                 if (args.verbose)
 777                         fprintf(stderr, "Server supports multi_ack\n");
 778                 multi_ack = 1;
 779         }
 780         if (server_supports("side-band-64k")) {
 781                 if (args.verbose)
 782                         fprintf(stderr, "Server supports side-band-64k\n");
 783                 use_sideband = 2;
 784         }
 785         else if (server_supports("side-band")) {
 786                 if (args.verbose)
 787                         fprintf(stderr, "Server supports side-band\n");
 788                 use_sideband = 1;
 789         }
 790         if (server_supports("ofs-delta")) {
 791                 if (args.verbose)
 792                         fprintf(stderr, "Server supports ofs-delta\n");
 793         } else
 794                 prefer_ofs_delta = 0;
 795         if (everything_local(&ref, nr_match, match)) {
 796                 packet_flush(fd[1]);
 797                 goto all_done;
 798         }
 799         if (find_common(fd, sha1, ref) < 0)
 800                 if (!args.keep_pack)
 801                         /* When cloning, it is not unusual to have
 802                          * no common commit.
 803                          */
 804                         warning("no common commits");
 805
 806         if (args.stateless_rpc)
 807                 packet_flush(fd[1]);
 808         if (get_pack(fd, pack_lockfile))
 809                 die("git fetch-pack: fetch failed.");
 810
 811  all_done:
 812         return ref;
 813 }
 814
 815 static int remove_duplicates(int nr_heads, char **heads)
 816 {
 817         int src, dst;
 818
 819         for (src = dst = 0; src < nr_heads; src++) {
 820                 /* If heads[src] is different from any of
 821                  * heads[0..dst], push it in.
 822                  */
 823                 int i;
 824                 for (i = 0; i < dst; i++) {
 825                         if (!strcmp(heads[i], heads[src]))
 826                                 break;
 827                 }
 828                 if (i < dst)
 829                         continue;
 830                 if (src != dst)
 831                         heads[dst] = heads[src];
 832                 dst++;
 833         }
 834         return dst;
 835 }
 836
 837 static int fetch_pack_config(const char *var, const char *value, void *cb)
 838 {
 839         if (strcmp(var, "fetch.unpacklimit") == 0) {
 840                 fetch_unpack_limit = git_config_int(var, value);
 841                 return 0;
 842         }
 843
 844         if (strcmp(var, "transfer.unpacklimit") == 0) {
 845                 transfer_unpack_limit = git_config_int(var, value);
 846                 return 0;
 847         }
 848
 849         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 850                 prefer_ofs_delta = git_config_bool(var, value);
 851                 return 0;
 852         }
 853
 854         return git_default_config(var, value, cb);
 855 }
 856
 857 static struct lock_file lock;
 858
 859 static void fetch_pack_setup(void)
 860 {
 861         static int did_setup;
 862         if (did_setup)
 863                 return;
 864         git_config(fetch_pack_config, NULL);
 865         if (0 <= transfer_unpack_limit)
 866                 unpack_limit = transfer_unpack_limit;
 867         else if (0 <= fetch_unpack_limit)
 868                 unpack_limit = fetch_unpack_limit;
 869         did_setup = 1;
 870 }
 871
 872 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 873 {
 874         int i, ret, nr_heads;
 875         struct ref *ref = NULL;
 876         char *dest = NULL, **heads;
 877         int fd[2];
 878         char *pack_lockfile = NULL;
 879         char **pack_lockfile_ptr = NULL;
 880         struct child_process *conn;
 881
 882         packet_trace_identity("fetch-pack");
 883
 884         nr_heads = 0;
 885         heads = NULL;
 886         for (i = 1; i < argc; i++) {
 887                 const char *arg = argv[i];
 888
 889                 if (*arg == '-') {
 890                         if (!prefixcmp(arg, "--upload-pack=")) {
 891                                 args.uploadpack = arg + 14;
 892                                 continue;
 893                         }
 894                         if (!prefixcmp(arg, "--exec=")) {
 895                                 args.uploadpack = arg + 7;
 896                                 continue;
 897                         }
 898                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 899                                 args.quiet = 1;
 900                                 continue;
 901                         }
 902                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 903                                 args.lock_pack = args.keep_pack;
 904                                 args.keep_pack = 1;
 905                                 continue;
 906                         }
 907                         if (!strcmp("--thin", arg)) {
 908                                 args.use_thin_pack = 1;
 909                                 continue;
 910                         }
 911                         if (!strcmp("--include-tag", arg)) {
 912                                 args.include_tag = 1;
 913                                 continue;
 914                         }
 915                         if (!strcmp("--all", arg)) {
 916                                 args.fetch_all = 1;
 917                                 continue;
 918                         }
 919                         if (!strcmp("-v", arg)) {
 920                                 args.verbose = 1;
 921                                 continue;
 922                         }
 923                         if (!prefixcmp(arg, "--depth=")) {
 924                                 args.depth = strtol(arg + 8, NULL, 0);
 925                                 continue;
 926                         }
 927                         if (!strcmp("--no-progress", arg)) {
 928                                 args.no_progress = 1;
 929                                 continue;
 930                         }
 931                         if (!strcmp("--stateless-rpc", arg)) {
 932                                 args.stateless_rpc = 1;
 933                                 continue;
 934                         }
 935                         if (!strcmp("--lock-pack", arg)) {
 936                                 args.lock_pack = 1;
 937                                 pack_lockfile_ptr = &pack_lockfile;
 938                                 continue;
 939                         }
 940                         usage(fetch_pack_usage);
 941                 }
 942                 dest = (char *)arg;
 943                 heads = (char **)(argv + i + 1);
 944                 nr_heads = argc - i - 1;
 945                 break;
 946         }
 947         if (!dest)
 948                 usage(fetch_pack_usage);
 949
 950         if (args.stateless_rpc) {
 951                 conn = NULL;
 952                 fd[0] = 0;
 953                 fd[1] = 1;
 954         } else {
 955                 conn = git_connect(fd, (char *)dest, args.uploadpack,
 956                                    args.verbose ? CONNECT_VERBOSE : 0);
 957         }
 958
 959         get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL);
 960
 961         ref = fetch_pack(&args, fd, conn, ref, dest,
 962                 nr_heads, heads, pack_lockfile_ptr);
 963         if (pack_lockfile) {
 964                 printf("lock %s\n", pack_lockfile);
 965                 fflush(stdout);
 966         }
 967         close(fd[0]);
 968         close(fd[1]);
 969         if (finish_connect(conn))
 970                 ref = NULL;
 971         ret = !ref;
 972
 973         if (!ret && nr_heads) {
 974                 /* If the heads to pull were given, we should have
 975                  * consumed all of them by matching the remote.
 976                  * Otherwise, 'git fetch remote no-such-ref' would
 977                  * silently succeed without issuing an error.
 978                  */
 979                 for (i = 0; i < nr_heads; i++)
 980                         if (heads[i] && heads[i][0]) {
 981                                 error("no such remote ref %s", heads[i]);
 982                                 ret = 1;
 983                         }
 984         }
 985         while (ref) {
 986                 printf("%s %s\n",
 987                        sha1_to_hex(ref->old_sha1), ref->name);
 988                 ref = ref->next;
 989         }
 990
 991         return ret;
 992 }
 993
 994 struct ref *fetch_pack(struct fetch_pack_args *my_args,
 995                        int fd[], struct child_process *conn,
 996                        const struct ref *ref,
 997                 const char *dest,
 998                 int nr_heads,
 999                 char **heads,
1000                 char **pack_lockfile)
1001 {
1002         struct stat st;
1003         struct ref *ref_cpy;
1004
1005         fetch_pack_setup();
1006         if (&args != my_args)
1007                 memcpy(&args, my_args, sizeof(args));
1008         if (args.depth > 0) {
1009                 if (stat(git_path("shallow"), &st))
1010                         st.st_mtime = 0;
1011         }
1012
1013         if (heads && nr_heads)
1014                 nr_heads = remove_duplicates(nr_heads, heads);
1015         if (!ref) {
1016                 packet_flush(fd[1]);
1017                 die("no matching remote head");
1018         }
1019         ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
1020
1021         if (args.depth > 0) {
1022                 struct cache_time mtime;
1023                 struct strbuf sb = STRBUF_INIT;
1024                 char *shallow = git_path("shallow");
1025                 int fd;
1026
1027                 mtime.sec = st.st_mtime;
1028                 mtime.nsec = ST_MTIME_NSEC(st);
1029                 if (stat(shallow, &st)) {
1030                         if (mtime.sec)
1031                                 die("shallow file was removed during fetch");
1032                 } else if (st.st_mtime != mtime.sec
1033 #ifdef USE_NSEC
1034                                 || ST_MTIME_NSEC(st) != mtime.nsec
1035 #endif
1036                           )
1037                         die("shallow file was changed during fetch");
1038
1039                 fd = hold_lock_file_for_update(&lock, shallow,
1040                                                LOCK_DIE_ON_ERROR);
1041                 if (!write_shallow_commits(&sb, 0)
1042                  || write_in_full(fd, sb.buf, sb.len) != sb.len) {
1043                         unlink_or_warn(shallow);
1044                         rollback_lock_file(&lock);
1045                 } else {
1046                         commit_lock_file(&lock);
1047                 }
1048                 strbuf_release(&sb);
1049         }
1050
1051         reprepare_packed_git();
1052         return ref_cpy;
1053 }