builtin/fetch-pack.c

   1 #include "builtin.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "transport.h"
  13
  14 static int transfer_unpack_limit = -1;
  15 static int fetch_unpack_limit = -1;
  16 static int unpack_limit = 100;
  17 static int prefer_ofs_delta = 1;
  18 static int no_done;
  19 static int fetch_fsck_objects = -1;
  20 static int transfer_fsck_objects = -1;
  21 static struct fetch_pack_args args = {
  22         /* .uploadpack = */ "git-upload-pack",
  23 };
  24
  25 static const char fetch_pack_usage[] =
  26 "git fetch-pack [--all] [--stdin] [--quiet|-q] [--keep|-k] [--thin] "
  27 "[--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] "
  28 "[--no-progress] [-v] [<host>:]<directory> [<refs>...]";
  29
  30 #define COMPLETE        (1U << 0)
  31 #define COMMON          (1U << 1)
  32 #define COMMON_REF      (1U << 2)
  33 #define SEEN            (1U << 3)
  34 #define POPPED          (1U << 4)
  35
  36 static int marked;
  37
  38 /*
  39  * After sending this many "have"s if we do not get any new ACK , we
  40  * give up traversing our history.
  41  */
  42 #define MAX_IN_VAIN 256
  43
  44 static struct commit_list *rev_list;
  45 static int non_common_revs, multi_ack, use_sideband;
  46
  47 static void rev_list_push(struct commit *commit, int mark)
  48 {
  49         if (!(commit->object.flags & mark)) {
  50                 commit->object.flags |= mark;
  51
  52                 if (!(commit->object.parsed))
  53                         if (parse_commit(commit))
  54                                 return;
  55
  56                 commit_list_insert_by_date(commit, &rev_list);
  57
  58                 if (!(commit->object.flags & COMMON))
  59                         non_common_revs++;
  60         }
  61 }
  62
  63 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  64 {
  65         struct object *o = deref_tag(parse_object(sha1), path, 0);
  66
  67         if (o && o->type == OBJ_COMMIT)
  68                 rev_list_push((struct commit *)o, SEEN);
  69
  70         return 0;
  71 }
  72
  73 static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  74 {
  75         struct object *o = deref_tag(parse_object(sha1), path, 0);
  76
  77         if (o && o->type == OBJ_COMMIT)
  78                 clear_commit_marks((struct commit *)o,
  79                                    COMMON | COMMON_REF | SEEN | POPPED);
  80         return 0;
  81 }
  82
  83 /*
  84    This function marks a rev and its ancestors as common.
  85    In some cases, it is desirable to mark only the ancestors (for example
  86    when only the server does not yet know that they are common).
  87 */
  88
  89 static void mark_common(struct commit *commit,
  90                 int ancestors_only, int dont_parse)
  91 {
  92         if (commit != NULL && !(commit->object.flags & COMMON)) {
  93                 struct object *o = (struct object *)commit;
  94
  95                 if (!ancestors_only)
  96                         o->flags |= COMMON;
  97
  98                 if (!(o->flags & SEEN))
  99                         rev_list_push(commit, SEEN);
 100                 else {
 101                         struct commit_list *parents;
 102
 103                         if (!ancestors_only && !(o->flags & POPPED))
 104                                 non_common_revs--;
 105                         if (!o->parsed && !dont_parse)
 106                                 if (parse_commit(commit))
 107                                         return;
 108
 109                         for (parents = commit->parents;
 110                                         parents;
 111                                         parents = parents->next)
 112                                 mark_common(parents->item, 0, dont_parse);
 113                 }
 114         }
 115 }
 116
 117 /*
 118   Get the next rev to send, ignoring the common.
 119 */
 120
 121 static const unsigned char *get_rev(void)
 122 {
 123         struct commit *commit = NULL;
 124
 125         while (commit == NULL) {
 126                 unsigned int mark;
 127                 struct commit_list *parents;
 128
 129                 if (rev_list == NULL || non_common_revs == 0)
 130                         return NULL;
 131
 132                 commit = rev_list->item;
 133                 if (!commit->object.parsed)
 134                         parse_commit(commit);
 135                 parents = commit->parents;
 136
 137                 commit->object.flags |= POPPED;
 138                 if (!(commit->object.flags & COMMON))
 139                         non_common_revs--;
 140
 141                 if (commit->object.flags & COMMON) {
 142                         /* do not send "have", and ignore ancestors */
 143                         commit = NULL;
 144                         mark = COMMON | SEEN;
 145                 } else if (commit->object.flags & COMMON_REF)
 146                         /* send "have", and ignore ancestors */
 147                         mark = COMMON | SEEN;
 148                 else
 149                         /* send "have", also for its ancestors */
 150                         mark = SEEN;
 151
 152                 while (parents) {
 153                         if (!(parents->item->object.flags & SEEN))
 154                                 rev_list_push(parents->item, mark);
 155                         if (mark & COMMON)
 156                                 mark_common(parents->item, 1, 0);
 157                         parents = parents->next;
 158                 }
 159
 160                 rev_list = rev_list->next;
 161         }
 162
 163         return commit->object.sha1;
 164 }
 165
 166 enum ack_type {
 167         NAK = 0,
 168         ACK,
 169         ACK_continue,
 170         ACK_common,
 171         ACK_ready
 172 };
 173
 174 static void consume_shallow_list(int fd)
 175 {
 176         if (args.stateless_rpc && args.depth > 0) {
 177                 /* If we sent a depth we will get back "duplicate"
 178                  * shallow and unshallow commands every time there
 179                  * is a block of have lines exchanged.
 180                  */
 181                 char line[1000];
 182                 while (packet_read_line(fd, line, sizeof(line))) {
 183                         if (!prefixcmp(line, "shallow "))
 184                                 continue;
 185                         if (!prefixcmp(line, "unshallow "))
 186                                 continue;
 187                         die("git fetch-pack: expected shallow list");
 188                 }
 189         }
 190 }
 191
 192 struct write_shallow_data {
 193         struct strbuf *out;
 194         int use_pack_protocol;
 195         int count;
 196 };
 197
 198 static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
 199 {
 200         struct write_shallow_data *data = cb_data;
 201         const char *hex = sha1_to_hex(graft->sha1);
 202         data->count++;
 203         if (data->use_pack_protocol)
 204                 packet_buf_write(data->out, "shallow %s", hex);
 205         else {
 206                 strbuf_addstr(data->out, hex);
 207                 strbuf_addch(data->out, '\n');
 208         }
 209         return 0;
 210 }
 211
 212 static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
 213 {
 214         struct write_shallow_data data;
 215         data.out = out;
 216         data.use_pack_protocol = use_pack_protocol;
 217         data.count = 0;
 218         for_each_commit_graft(write_one_shallow, &data);
 219         return data.count;
 220 }
 221
 222 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 223 {
 224         static char line[1000];
 225         int len = packet_read_line(fd, line, sizeof(line));
 226
 227         if (!len)
 228                 die("git fetch-pack: expected ACK/NAK, got EOF");
 229         if (line[len-1] == '\n')
 230                 line[--len] = 0;
 231         if (!strcmp(line, "NAK"))
 232                 return NAK;
 233         if (!prefixcmp(line, "ACK ")) {
 234                 if (!get_sha1_hex(line+4, result_sha1)) {
 235                         if (strstr(line+45, "continue"))
 236                                 return ACK_continue;
 237                         if (strstr(line+45, "common"))
 238                                 return ACK_common;
 239                         if (strstr(line+45, "ready"))
 240                                 return ACK_ready;
 241                         return ACK;
 242                 }
 243         }
 244         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 245 }
 246
 247 static void send_request(int fd, struct strbuf *buf)
 248 {
 249         if (args.stateless_rpc) {
 250                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 251                 packet_flush(fd);
 252         } else
 253                 safe_write(fd, buf->buf, buf->len);
 254 }
 255
 256 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 257 {
 258         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 259 }
 260
 261 static void insert_alternate_refs(void)
 262 {
 263         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 264 }
 265
 266 #define INITIAL_FLUSH 16
 267 #define PIPESAFE_FLUSH 32
 268 #define LARGE_FLUSH 1024
 269
 270 static int next_flush(int count)
 271 {
 272         int flush_limit = args.stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 273
 274         if (count < flush_limit)
 275                 count <<= 1;
 276         else
 277                 count += flush_limit;
 278         return count;
 279 }
 280
 281 static int find_common(int fd[2], unsigned char *result_sha1,
 282                        struct ref *refs)
 283 {
 284         int fetching;
 285         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 286         const unsigned char *sha1;
 287         unsigned in_vain = 0;
 288         int got_continue = 0;
 289         int got_ready = 0;
 290         struct strbuf req_buf = STRBUF_INIT;
 291         size_t state_len = 0;
 292
 293         if (args.stateless_rpc && multi_ack == 1)
 294                 die("--stateless-rpc requires multi_ack_detailed");
 295         if (marked)
 296                 for_each_ref(clear_marks, NULL);
 297         marked = 1;
 298
 299         for_each_ref(rev_list_insert_ref, NULL);
 300         insert_alternate_refs();
 301
 302         fetching = 0;
 303         for ( ; refs ; refs = refs->next) {
 304                 unsigned char *remote = refs->old_sha1;
 305                 const char *remote_hex;
 306                 struct object *o;
 307
 308                 /*
 309                  * If that object is complete (i.e. it is an ancestor of a
 310                  * local ref), we tell them we have it but do not have to
 311                  * tell them about its ancestors, which they already know
 312                  * about.
 313                  *
 314                  * We use lookup_object here because we are only
 315                  * interested in the case we *know* the object is
 316                  * reachable and we have already scanned it.
 317                  */
 318                 if (((o = lookup_object(remote)) != NULL) &&
 319                                 (o->flags & COMPLETE)) {
 320                         continue;
 321                 }
 322
 323                 remote_hex = sha1_to_hex(remote);
 324                 if (!fetching) {
 325                         struct strbuf c = STRBUF_INIT;
 326                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 327                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 328                         if (no_done)            strbuf_addstr(&c, " no-done");
 329                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 330                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 331                         if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack");
 332                         if (args.no_progress)   strbuf_addstr(&c, " no-progress");
 333                         if (args.include_tag)   strbuf_addstr(&c, " include-tag");
 334                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 335                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 336                         strbuf_release(&c);
 337                 } else
 338                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 339                 fetching++;
 340         }
 341
 342         if (!fetching) {
 343                 strbuf_release(&req_buf);
 344                 packet_flush(fd[1]);
 345                 return 1;
 346         }
 347
 348         if (is_repository_shallow())
 349                 write_shallow_commits(&req_buf, 1);
 350         if (args.depth > 0)
 351                 packet_buf_write(&req_buf, "deepen %d", args.depth);
 352         packet_buf_flush(&req_buf);
 353         state_len = req_buf.len;
 354
 355         if (args.depth > 0) {
 356                 char line[1024];
 357                 unsigned char sha1[20];
 358
 359                 send_request(fd[1], &req_buf);
 360                 while (packet_read_line(fd[0], line, sizeof(line))) {
 361                         if (!prefixcmp(line, "shallow ")) {
 362                                 if (get_sha1_hex(line + 8, sha1))
 363                                         die("invalid shallow line: %s", line);
 364                                 register_shallow(sha1);
 365                                 continue;
 366                         }
 367                         if (!prefixcmp(line, "unshallow ")) {
 368                                 if (get_sha1_hex(line + 10, sha1))
 369                                         die("invalid unshallow line: %s", line);
 370                                 if (!lookup_object(sha1))
 371                                         die("object not found: %s", line);
 372                                 /* make sure that it is parsed as shallow */
 373                                 if (!parse_object(sha1))
 374                                         die("error in object: %s", line);
 375                                 if (unregister_shallow(sha1))
 376                                         die("no shallow found: %s", line);
 377                                 continue;
 378                         }
 379                         die("expected shallow/unshallow, got %s", line);
 380                 }
 381         } else if (!args.stateless_rpc)
 382                 send_request(fd[1], &req_buf);
 383
 384         if (!args.stateless_rpc) {
 385                 /* If we aren't using the stateless-rpc interface
 386                  * we don't need to retain the headers.
 387                  */
 388                 strbuf_setlen(&req_buf, 0);
 389                 state_len = 0;
 390         }
 391
 392         flushes = 0;
 393         retval = -1;
 394         while ((sha1 = get_rev())) {
 395                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 396                 if (args.verbose)
 397                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 398                 in_vain++;
 399                 if (flush_at <= ++count) {
 400                         int ack;
 401
 402                         packet_buf_flush(&req_buf);
 403                         send_request(fd[1], &req_buf);
 404                         strbuf_setlen(&req_buf, state_len);
 405                         flushes++;
 406                         flush_at = next_flush(count);
 407
 408                         /*
 409                          * We keep one window "ahead" of the other side, and
 410                          * will wait for an ACK only on the next one
 411                          */
 412                         if (!args.stateless_rpc && count == INITIAL_FLUSH)
 413                                 continue;
 414
 415                         consume_shallow_list(fd[0]);
 416                         do {
 417                                 ack = get_ack(fd[0], result_sha1);
 418                                 if (args.verbose && ack)
 419                                         fprintf(stderr, "got ack %d %s\n", ack,
 420                                                         sha1_to_hex(result_sha1));
 421                                 switch (ack) {
 422                                 case ACK:
 423                                         flushes = 0;
 424                                         multi_ack = 0;
 425                                         retval = 0;
 426                                         goto done;
 427                                 case ACK_common:
 428                                 case ACK_ready:
 429                                 case ACK_continue: {
 430                                         struct commit *commit =
 431                                                 lookup_commit(result_sha1);
 432                                         if (!commit)
 433                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 434                                         if (args.stateless_rpc
 435                                          && ack == ACK_common
 436                                          && !(commit->object.flags & COMMON)) {
 437                                                 /* We need to replay the have for this object
 438                                                  * on the next RPC request so the peer knows
 439                                                  * it is in common with us.
 440                                                  */
 441                                                 const char *hex = sha1_to_hex(result_sha1);
 442                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 443                                                 state_len = req_buf.len;
 444                                         }
 445                                         mark_common(commit, 0, 1);
 446                                         retval = 0;
 447                                         in_vain = 0;
 448                                         got_continue = 1;
 449                                         if (ack == ACK_ready) {
 450                                                 rev_list = NULL;
 451                                                 got_ready = 1;
 452                                         }
 453                                         break;
 454                                         }
 455                                 }
 456                         } while (ack);
 457                         flushes--;
 458                         if (got_continue && MAX_IN_VAIN < in_vain) {
 459                                 if (args.verbose)
 460                                         fprintf(stderr, "giving up\n");
 461                                 break; /* give up */
 462                         }
 463                 }
 464         }
 465 done:
 466         if (!got_ready || !no_done) {
 467                 packet_buf_write(&req_buf, "done\n");
 468                 send_request(fd[1], &req_buf);
 469         }
 470         if (args.verbose)
 471                 fprintf(stderr, "done\n");
 472         if (retval != 0) {
 473                 multi_ack = 0;
 474                 flushes++;
 475         }
 476         strbuf_release(&req_buf);
 477
 478         consume_shallow_list(fd[0]);
 479         while (flushes || multi_ack) {
 480                 int ack = get_ack(fd[0], result_sha1);
 481                 if (ack) {
 482                         if (args.verbose)
 483                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 484                                         sha1_to_hex(result_sha1));
 485                         if (ack == ACK)
 486                                 return 0;
 487                         multi_ack = 1;
 488                         continue;
 489                 }
 490                 flushes--;
 491         }
 492         /* it is no error to fetch into a completely empty repo */
 493         return count ? retval : 0;
 494 }
 495
 496 static struct commit_list *complete;
 497
 498 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 499 {
 500         struct object *o = parse_object(sha1);
 501
 502         while (o && o->type == OBJ_TAG) {
 503                 struct tag *t = (struct tag *) o;
 504                 if (!t->tagged)
 505                         break; /* broken repository */
 506                 o->flags |= COMPLETE;
 507                 o = parse_object(t->tagged->sha1);
 508         }
 509         if (o && o->type == OBJ_COMMIT) {
 510                 struct commit *commit = (struct commit *)o;
 511                 if (!(commit->object.flags & COMPLETE)) {
 512                         commit->object.flags |= COMPLETE;
 513                         commit_list_insert_by_date(commit, &complete);
 514                 }
 515         }
 516         return 0;
 517 }
 518
 519 static void mark_recent_complete_commits(unsigned long cutoff)
 520 {
 521         while (complete && cutoff <= complete->item->date) {
 522                 if (args.verbose)
 523                         fprintf(stderr, "Marking %s as complete\n",
 524                                 sha1_to_hex(complete->item->object.sha1));
 525                 pop_most_recent_commit(&complete, COMPLETE);
 526         }
 527 }
 528
 529 static void filter_refs(struct ref **refs, int nr_match, char **match)
 530 {
 531         struct ref **return_refs;
 532         struct ref *newlist = NULL;
 533         struct ref **newtail = &newlist;
 534         struct ref *ref, *next;
 535         struct ref *fastarray[32];
 536
 537         if (nr_match && !args.fetch_all) {
 538                 if (ARRAY_SIZE(fastarray) < nr_match)
 539                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 540                 else {
 541                         return_refs = fastarray;
 542                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 543                 }
 544         }
 545         else
 546                 return_refs = NULL;
 547
 548         for (ref = *refs; ref; ref = next) {
 549                 next = ref->next;
 550                 if (!memcmp(ref->name, "refs/", 5) &&
 551                     check_refname_format(ref->name + 5, 0))
 552                         ; /* trash */
 553                 else if (args.fetch_all &&
 554                          (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
 555                         *newtail = ref;
 556                         ref->next = NULL;
 557                         newtail = &ref->next;
 558                         continue;
 559                 }
 560                 else {
 561                         int i;
 562                         for (i = 0; i < nr_match; i++) {
 563                                 if (!strcmp(ref->name, match[i])) {
 564                                         match[i][0] = '\0';
 565                                         return_refs[i] = ref;
 566                                         break;
 567                                 }
 568                         }
 569                         if (i < nr_match)
 570                                 continue; /* we will link it later */
 571                 }
 572                 free(ref);
 573         }
 574
 575         if (!args.fetch_all) {
 576                 int i;
 577                 for (i = 0; i < nr_match; i++) {
 578                         ref = return_refs[i];
 579                         if (ref) {
 580                                 *newtail = ref;
 581                                 ref->next = NULL;
 582                                 newtail = &ref->next;
 583                         }
 584                 }
 585                 if (return_refs != fastarray)
 586                         free(return_refs);
 587         }
 588         *refs = newlist;
 589 }
 590
 591 static int everything_local(struct ref **refs, int nr_match, char **match)
 592 {
 593         struct ref *ref;
 594         int retval;
 595         unsigned long cutoff = 0;
 596
 597         save_commit_buffer = 0;
 598
 599         for (ref = *refs; ref; ref = ref->next) {
 600                 struct object *o;
 601
 602                 o = parse_object(ref->old_sha1);
 603                 if (!o)
 604                         continue;
 605
 606                 /* We already have it -- which may mean that we were
 607                  * in sync with the other side at some time after
 608                  * that (it is OK if we guess wrong here).
 609                  */
 610                 if (o->type == OBJ_COMMIT) {
 611                         struct commit *commit = (struct commit *)o;
 612                         if (!cutoff || cutoff < commit->date)
 613                                 cutoff = commit->date;
 614                 }
 615         }
 616
 617         if (!args.depth) {
 618                 for_each_ref(mark_complete, NULL);
 619                 if (cutoff)
 620                         mark_recent_complete_commits(cutoff);
 621         }
 622
 623         /*
 624          * Mark all complete remote refs as common refs.
 625          * Don't mark them common yet; the server has to be told so first.
 626          */
 627         for (ref = *refs; ref; ref = ref->next) {
 628                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 629                                              NULL, 0);
 630
 631                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 632                         continue;
 633
 634                 if (!(o->flags & SEEN)) {
 635                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 636
 637                         mark_common((struct commit *)o, 1, 1);
 638                 }
 639         }
 640
 641         filter_refs(refs, nr_match, match);
 642
 643         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 644                 const unsigned char *remote = ref->old_sha1;
 645                 unsigned char local[20];
 646                 struct object *o;
 647
 648                 o = lookup_object(remote);
 649                 if (!o || !(o->flags & COMPLETE)) {
 650                         retval = 0;
 651                         if (!args.verbose)
 652                                 continue;
 653                         fprintf(stderr,
 654                                 "want %s (%s)\n", sha1_to_hex(remote),
 655                                 ref->name);
 656                         continue;
 657                 }
 658
 659                 hashcpy(ref->new_sha1, local);
 660                 if (!args.verbose)
 661                         continue;
 662                 fprintf(stderr,
 663                         "already have %s (%s)\n", sha1_to_hex(remote),
 664                         ref->name);
 665         }
 666         return retval;
 667 }
 668
 669 static int sideband_demux(int in, int out, void *data)
 670 {
 671         int *xd = data;
 672
 673         int ret = recv_sideband("fetch-pack", xd[0], out);
 674         close(out);
 675         return ret;
 676 }
 677
 678 static int get_pack(int xd[2], char **pack_lockfile)
 679 {
 680         struct async demux;
 681         const char *argv[20];
 682         char keep_arg[256];
 683         char hdr_arg[256];
 684         const char **av;
 685         int do_keep = args.keep_pack;
 686         struct child_process cmd;
 687
 688         memset(&demux, 0, sizeof(demux));
 689         if (use_sideband) {
 690                 /* xd[] is talking with upload-pack; subprocess reads from
 691                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 692                  * through demux->out.
 693                  */
 694                 demux.proc = sideband_demux;
 695                 demux.data = xd;
 696                 demux.out = -1;
 697                 if (start_async(&demux))
 698                         die("fetch-pack: unable to fork off sideband"
 699                             " demultiplexer");
 700         }
 701         else
 702                 demux.out = xd[0];
 703
 704         memset(&cmd, 0, sizeof(cmd));
 705         cmd.argv = argv;
 706         av = argv;
 707         *hdr_arg = 0;
 708         if (!args.keep_pack && unpack_limit) {
 709                 struct pack_header header;
 710
 711                 if (read_pack_header(demux.out, &header))
 712                         die("protocol error: bad pack header");
 713                 snprintf(hdr_arg, sizeof(hdr_arg),
 714                          "--pack_header=%"PRIu32",%"PRIu32,
 715                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 716                 if (ntohl(header.hdr_entries) < unpack_limit)
 717                         do_keep = 0;
 718                 else
 719                         do_keep = 1;
 720         }
 721
 722         if (do_keep) {
 723                 if (pack_lockfile)
 724                         cmd.out = -1;
 725                 *av++ = "index-pack";
 726                 *av++ = "--stdin";
 727                 if (!args.quiet && !args.no_progress)
 728                         *av++ = "-v";
 729                 if (args.use_thin_pack)
 730                         *av++ = "--fix-thin";
 731                 if (args.lock_pack || unpack_limit) {
 732                         int s = sprintf(keep_arg,
 733                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 734                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 735                                 strcpy(keep_arg + s, "localhost");
 736                         *av++ = keep_arg;
 737                 }
 738         }
 739         else {
 740                 *av++ = "unpack-objects";
 741                 if (args.quiet)
 742                         *av++ = "-q";
 743         }
 744         if (*hdr_arg)
 745                 *av++ = hdr_arg;
 746         if (fetch_fsck_objects >= 0
 747             ? fetch_fsck_objects
 748             : transfer_fsck_objects >= 0
 749             ? transfer_fsck_objects
 750             : 0)
 751                 *av++ = "--strict";
 752         *av++ = NULL;
 753
 754         cmd.in = demux.out;
 755         cmd.git_cmd = 1;
 756         if (start_command(&cmd))
 757                 die("fetch-pack: unable to fork off %s", argv[0]);
 758         if (do_keep && pack_lockfile) {
 759                 *pack_lockfile = index_pack_lockfile(cmd.out);
 760                 close(cmd.out);
 761         }
 762
 763         if (finish_command(&cmd))
 764                 die("%s failed", argv[0]);
 765         if (use_sideband && finish_async(&demux))
 766                 die("error in sideband demultiplexer");
 767         return 0;
 768 }
 769
 770 static struct ref *do_fetch_pack(int fd[2],
 771                 const struct ref *orig_ref,
 772                 int nr_match,
 773                 char **match,
 774                 char **pack_lockfile)
 775 {
 776         struct ref *ref = copy_ref_list(orig_ref);
 777         unsigned char sha1[20];
 778
 779         if (is_repository_shallow() && !server_supports("shallow"))
 780                 die("Server does not support shallow clients");
 781         if (server_supports("multi_ack_detailed")) {
 782                 if (args.verbose)
 783                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 784                 multi_ack = 2;
 785                 if (server_supports("no-done")) {
 786                         if (args.verbose)
 787                                 fprintf(stderr, "Server supports no-done\n");
 788                         if (args.stateless_rpc)
 789                                 no_done = 1;
 790                 }
 791         }
 792         else if (server_supports("multi_ack")) {
 793                 if (args.verbose)
 794                         fprintf(stderr, "Server supports multi_ack\n");
 795                 multi_ack = 1;
 796         }
 797         if (server_supports("side-band-64k")) {
 798                 if (args.verbose)
 799                         fprintf(stderr, "Server supports side-band-64k\n");
 800                 use_sideband = 2;
 801         }
 802         else if (server_supports("side-band")) {
 803                 if (args.verbose)
 804                         fprintf(stderr, "Server supports side-band\n");
 805                 use_sideband = 1;
 806         }
 807         if (server_supports("ofs-delta")) {
 808                 if (args.verbose)
 809                         fprintf(stderr, "Server supports ofs-delta\n");
 810         } else
 811                 prefer_ofs_delta = 0;
 812         if (everything_local(&ref, nr_match, match)) {
 813                 packet_flush(fd[1]);
 814                 goto all_done;
 815         }
 816         if (find_common(fd, sha1, ref) < 0)
 817                 if (!args.keep_pack)
 818                         /* When cloning, it is not unusual to have
 819                          * no common commit.
 820                          */
 821                         warning("no common commits");
 822
 823         if (args.stateless_rpc)
 824                 packet_flush(fd[1]);
 825         if (get_pack(fd, pack_lockfile))
 826                 die("git fetch-pack: fetch failed.");
 827
 828  all_done:
 829         return ref;
 830 }
 831
 832 static int remove_duplicates(int nr_heads, char **heads)
 833 {
 834         int src, dst;
 835
 836         for (src = dst = 0; src < nr_heads; src++) {
 837                 /* If heads[src] is different from any of
 838                  * heads[0..dst], push it in.
 839                  */
 840                 int i;
 841                 for (i = 0; i < dst; i++) {
 842                         if (!strcmp(heads[i], heads[src]))
 843                                 break;
 844                 }
 845                 if (i < dst)
 846                         continue;
 847                 if (src != dst)
 848                         heads[dst] = heads[src];
 849                 dst++;
 850         }
 851         return dst;
 852 }
 853
 854 static int fetch_pack_config(const char *var, const char *value, void *cb)
 855 {
 856         if (strcmp(var, "fetch.unpacklimit") == 0) {
 857                 fetch_unpack_limit = git_config_int(var, value);
 858                 return 0;
 859         }
 860
 861         if (strcmp(var, "transfer.unpacklimit") == 0) {
 862                 transfer_unpack_limit = git_config_int(var, value);
 863                 return 0;
 864         }
 865
 866         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 867                 prefer_ofs_delta = git_config_bool(var, value);
 868                 return 0;
 869         }
 870
 871         if (!strcmp(var, "fetch.fsckobjects")) {
 872                 fetch_fsck_objects = git_config_bool(var, value);
 873                 return 0;
 874         }
 875
 876         if (!strcmp(var, "transfer.fsckobjects")) {
 877                 transfer_fsck_objects = git_config_bool(var, value);
 878                 return 0;
 879         }
 880
 881         return git_default_config(var, value, cb);
 882 }
 883
 884 static struct lock_file lock;
 885
 886 static void fetch_pack_setup(void)
 887 {
 888         static int did_setup;
 889         if (did_setup)
 890                 return;
 891         git_config(fetch_pack_config, NULL);
 892         if (0 <= transfer_unpack_limit)
 893                 unpack_limit = transfer_unpack_limit;
 894         else if (0 <= fetch_unpack_limit)
 895                 unpack_limit = fetch_unpack_limit;
 896         did_setup = 1;
 897 }
 898
 899 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 900 {
 901         int i, ret;
 902         struct ref *ref = NULL;
 903         const char *dest = NULL;
 904         int alloc_heads = 0, nr_heads = 0;
 905         char **heads = NULL;
 906         int fd[2];
 907         char *pack_lockfile = NULL;
 908         char **pack_lockfile_ptr = NULL;
 909         struct child_process *conn;
 910
 911         packet_trace_identity("fetch-pack");
 912
 913         for (i = 1; i < argc && *argv[i] == '-'; i++) {
 914                 const char *arg = argv[i];
 915
 916                 if (!prefixcmp(arg, "--upload-pack=")) {
 917                         args.uploadpack = arg + 14;
 918                         continue;
 919                 }
 920                 if (!prefixcmp(arg, "--exec=")) {
 921                         args.uploadpack = arg + 7;
 922                         continue;
 923                 }
 924                 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 925                         args.quiet = 1;
 926                         continue;
 927                 }
 928                 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 929                         args.lock_pack = args.keep_pack;
 930                         args.keep_pack = 1;
 931                         continue;
 932                 }
 933                 if (!strcmp("--thin", arg)) {
 934                         args.use_thin_pack = 1;
 935                         continue;
 936                 }
 937                 if (!strcmp("--include-tag", arg)) {
 938                         args.include_tag = 1;
 939                         continue;
 940                 }
 941                 if (!strcmp("--all", arg)) {
 942                         args.fetch_all = 1;
 943                         continue;
 944                 }
 945                 if (!strcmp("--stdin", arg)) {
 946                         args.stdin_refs = 1;
 947                         continue;
 948                 }
 949                 if (!strcmp("-v", arg)) {
 950                         args.verbose = 1;
 951                         continue;
 952                 }
 953                 if (!prefixcmp(arg, "--depth=")) {
 954                         args.depth = strtol(arg + 8, NULL, 0);
 955                         continue;
 956                 }
 957                 if (!strcmp("--no-progress", arg)) {
 958                         args.no_progress = 1;
 959                         continue;
 960                 }
 961                 if (!strcmp("--stateless-rpc", arg)) {
 962                         args.stateless_rpc = 1;
 963                         continue;
 964                 }
 965                 if (!strcmp("--lock-pack", arg)) {
 966                         args.lock_pack = 1;
 967                         pack_lockfile_ptr = &pack_lockfile;
 968                         continue;
 969                 }
 970                 usage(fetch_pack_usage);
 971         }
 972
 973         if (i < argc)
 974                 dest = argv[i++];
 975         else
 976                 usage(fetch_pack_usage);
 977
 978         /*
 979          * Copy refs from cmdline to growable list, then append any
 980          * refs from the standard input:
 981          */
 982         ALLOC_GROW(heads, argc - i, alloc_heads);
 983         for (; i < argc; i++)
 984                 heads[nr_heads++] = xstrdup(argv[i]);
 985         if (args.stdin_refs) {
 986                 if (args.stateless_rpc) {
 987                         /* in stateless RPC mode we use pkt-line to read
 988                          * from stdin, until we get a flush packet
 989                          */
 990                         static char line[1000];
 991                         for (;;) {
 992                                 int n = packet_read_line(0, line, sizeof(line));
 993                                 if (!n)
 994                                         break;
 995                                 if (line[n-1] == '\n')
 996                                         n--;
 997                                 ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
 998                                 heads[nr_heads++] = xmemdupz(line, n);
 999                         }
1000                 }
1001                 else {
1002                         /* read from stdin one ref per line, until EOF */
1003                         struct strbuf line = STRBUF_INIT;
1004                         while (strbuf_getline(&line, stdin, '\n') != EOF) {
1005                                 ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
1006                                 heads[nr_heads++] = strbuf_detach(&line, NULL);
1007                         }
1008                         strbuf_release(&line);
1009                 }
1010         }
1011
1012         if (args.stateless_rpc) {
1013                 conn = NULL;
1014                 fd[0] = 0;
1015                 fd[1] = 1;
1016         } else {
1017                 conn = git_connect(fd, dest, args.uploadpack,
1018                                    args.verbose ? CONNECT_VERBOSE : 0);
1019         }
1020
1021         get_remote_heads(fd[0], &ref, 0, NULL);
1022
1023         ref = fetch_pack(&args, fd, conn, ref, dest,
1024                 nr_heads, heads, pack_lockfile_ptr);
1025         if (pack_lockfile) {
1026                 printf("lock %s\n", pack_lockfile);
1027                 fflush(stdout);
1028         }
1029         close(fd[0]);
1030         close(fd[1]);
1031         if (finish_connect(conn))
1032                 ref = NULL;
1033         ret = !ref;
1034
1035         if (!ret && nr_heads) {
1036                 /* If the heads to pull were given, we should have
1037                  * consumed all of them by matching the remote.
1038                  * Otherwise, 'git fetch remote no-such-ref' would
1039                  * silently succeed without issuing an error.
1040                  */
1041                 for (i = 0; i < nr_heads; i++)
1042                         if (heads[i] && heads[i][0]) {
1043                                 error("no such remote ref %s", heads[i]);
1044                                 ret = 1;
1045                         }
1046         }
1047         while (ref) {
1048                 printf("%s %s\n",
1049                        sha1_to_hex(ref->old_sha1), ref->name);
1050                 ref = ref->next;
1051         }
1052
1053         return ret;
1054 }
1055
1056 struct ref *fetch_pack(struct fetch_pack_args *my_args,
1057                        int fd[], struct child_process *conn,
1058                        const struct ref *ref,
1059                 const char *dest,
1060                 int nr_heads,
1061                 char **heads,
1062                 char **pack_lockfile)
1063 {
1064         struct stat st;
1065         struct ref *ref_cpy;
1066
1067         fetch_pack_setup();
1068         if (&args != my_args)
1069                 memcpy(&args, my_args, sizeof(args));
1070         if (args.depth > 0) {
1071                 if (stat(git_path("shallow"), &st))
1072                         st.st_mtime = 0;
1073         }
1074
1075         if (heads && nr_heads)
1076                 nr_heads = remove_duplicates(nr_heads, heads);
1077         if (!ref) {
1078                 packet_flush(fd[1]);
1079                 die("no matching remote head");
1080         }
1081         ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
1082
1083         if (args.depth > 0) {
1084                 struct cache_time mtime;
1085                 struct strbuf sb = STRBUF_INIT;
1086                 char *shallow = git_path("shallow");
1087                 int fd;
1088
1089                 mtime.sec = st.st_mtime;
1090                 mtime.nsec = ST_MTIME_NSEC(st);
1091                 if (stat(shallow, &st)) {
1092                         if (mtime.sec)
1093                                 die("shallow file was removed during fetch");
1094                 } else if (st.st_mtime != mtime.sec
1095 #ifdef USE_NSEC
1096                                 || ST_MTIME_NSEC(st) != mtime.nsec
1097 #endif
1098                           )
1099                         die("shallow file was changed during fetch");
1100
1101                 fd = hold_lock_file_for_update(&lock, shallow,
1102                                                LOCK_DIE_ON_ERROR);
1103                 if (!write_shallow_commits(&sb, 0)
1104                  || write_in_full(fd, sb.buf, sb.len) != sb.len) {
1105                         unlink_or_warn(shallow);
1106                         rollback_lock_file(&lock);
1107                 } else {
1108                         commit_lock_file(&lock);
1109                 }
1110                 strbuf_release(&sb);
1111         }
1112
1113         reprepare_packed_git();
1114         return ref_cpy;
1115 }