builtin/fetch-pack.c

   1 #include "builtin.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "transport.h"
  13 #include "version.h"
  14
  15 static int transfer_unpack_limit = -1;
  16 static int fetch_unpack_limit = -1;
  17 static int unpack_limit = 100;
  18 static int prefer_ofs_delta = 1;
  19 static int no_done;
  20 static int fetch_fsck_objects = -1;
  21 static int transfer_fsck_objects = -1;
  22 static int agent_supported;
  23
  24 static const char fetch_pack_usage[] =
  25 "git fetch-pack [--all] [--stdin] [--quiet|-q] [--keep|-k] [--thin] "
  26 "[--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] "
  27 "[--no-progress] [-v] [<host>:]<directory> [<refs>...]";
  28
  29 #define COMPLETE        (1U << 0)
  30 #define COMMON          (1U << 1)
  31 #define COMMON_REF      (1U << 2)
  32 #define SEEN            (1U << 3)
  33 #define POPPED          (1U << 4)
  34
  35 static int marked;
  36
  37 /*
  38  * After sending this many "have"s if we do not get any new ACK , we
  39  * give up traversing our history.
  40  */
  41 #define MAX_IN_VAIN 256
  42
  43 static struct commit_list *rev_list;
  44 static int non_common_revs, multi_ack, use_sideband;
  45
  46 static void rev_list_push(struct commit *commit, int mark)
  47 {
  48         if (!(commit->object.flags & mark)) {
  49                 commit->object.flags |= mark;
  50
  51                 if (!(commit->object.parsed))
  52                         if (parse_commit(commit))
  53                                 return;
  54
  55                 commit_list_insert_by_date(commit, &rev_list);
  56
  57                 if (!(commit->object.flags & COMMON))
  58                         non_common_revs++;
  59         }
  60 }
  61
  62 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  63 {
  64         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  65
  66         if (o && o->type == OBJ_COMMIT)
  67                 rev_list_push((struct commit *)o, SEEN);
  68
  69         return 0;
  70 }
  71
  72 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  73 {
  74         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  75
  76         if (o && o->type == OBJ_COMMIT)
  77                 clear_commit_marks((struct commit *)o,
  78                                    COMMON | COMMON_REF | SEEN | POPPED);
  79         return 0;
  80 }
  81
  82 /*
  83    This function marks a rev and its ancestors as common.
  84    In some cases, it is desirable to mark only the ancestors (for example
  85    when only the server does not yet know that they are common).
  86 */
  87
  88 static void mark_common(struct commit *commit,
  89                 int ancestors_only, int dont_parse)
  90 {
  91         if (commit != NULL && !(commit->object.flags & COMMON)) {
  92                 struct object *o = (struct object *)commit;
  93
  94                 if (!ancestors_only)
  95                         o->flags |= COMMON;
  96
  97                 if (!(o->flags & SEEN))
  98                         rev_list_push(commit, SEEN);
  99                 else {
 100                         struct commit_list *parents;
 101
 102                         if (!ancestors_only && !(o->flags & POPPED))
 103                                 non_common_revs--;
 104                         if (!o->parsed && !dont_parse)
 105                                 if (parse_commit(commit))
 106                                         return;
 107
 108                         for (parents = commit->parents;
 109                                         parents;
 110                                         parents = parents->next)
 111                                 mark_common(parents->item, 0, dont_parse);
 112                 }
 113         }
 114 }
 115
 116 /*
 117   Get the next rev to send, ignoring the common.
 118 */
 119
 120 static const unsigned char *get_rev(void)
 121 {
 122         struct commit *commit = NULL;
 123
 124         while (commit == NULL) {
 125                 unsigned int mark;
 126                 struct commit_list *parents;
 127
 128                 if (rev_list == NULL || non_common_revs == 0)
 129                         return NULL;
 130
 131                 commit = rev_list->item;
 132                 if (!commit->object.parsed)
 133                         parse_commit(commit);
 134                 parents = commit->parents;
 135
 136                 commit->object.flags |= POPPED;
 137                 if (!(commit->object.flags & COMMON))
 138                         non_common_revs--;
 139
 140                 if (commit->object.flags & COMMON) {
 141                         /* do not send "have", and ignore ancestors */
 142                         commit = NULL;
 143                         mark = COMMON | SEEN;
 144                 } else if (commit->object.flags & COMMON_REF)
 145                         /* send "have", and ignore ancestors */
 146                         mark = COMMON | SEEN;
 147                 else
 148                         /* send "have", also for its ancestors */
 149                         mark = SEEN;
 150
 151                 while (parents) {
 152                         if (!(parents->item->object.flags & SEEN))
 153                                 rev_list_push(parents->item, mark);
 154                         if (mark & COMMON)
 155                                 mark_common(parents->item, 1, 0);
 156                         parents = parents->next;
 157                 }
 158
 159                 rev_list = rev_list->next;
 160         }
 161
 162         return commit->object.sha1;
 163 }
 164
 165 enum ack_type {
 166         NAK = 0,
 167         ACK,
 168         ACK_continue,
 169         ACK_common,
 170         ACK_ready
 171 };
 172
 173 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 174 {
 175         if (args->stateless_rpc && args->depth > 0) {
 176                 /* If we sent a depth we will get back "duplicate"
 177                  * shallow and unshallow commands every time there
 178                  * is a block of have lines exchanged.
 179                  */
 180                 char line[1000];
 181                 while (packet_read_line(fd, line, sizeof(line))) {
 182                         if (!prefixcmp(line, "shallow "))
 183                                 continue;
 184                         if (!prefixcmp(line, "unshallow "))
 185                                 continue;
 186                         die("git fetch-pack: expected shallow list");
 187                 }
 188         }
 189 }
 190
 191 struct write_shallow_data {
 192         struct strbuf *out;
 193         int use_pack_protocol;
 194         int count;
 195 };
 196
 197 static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
 198 {
 199         struct write_shallow_data *data = cb_data;
 200         const char *hex = sha1_to_hex(graft->sha1);
 201         data->count++;
 202         if (data->use_pack_protocol)
 203                 packet_buf_write(data->out, "shallow %s", hex);
 204         else {
 205                 strbuf_addstr(data->out, hex);
 206                 strbuf_addch(data->out, '\n');
 207         }
 208         return 0;
 209 }
 210
 211 static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
 212 {
 213         struct write_shallow_data data;
 214         data.out = out;
 215         data.use_pack_protocol = use_pack_protocol;
 216         data.count = 0;
 217         for_each_commit_graft(write_one_shallow, &data);
 218         return data.count;
 219 }
 220
 221 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 222 {
 223         static char line[1000];
 224         int len = packet_read_line(fd, line, sizeof(line));
 225
 226         if (!len)
 227                 die("git fetch-pack: expected ACK/NAK, got EOF");
 228         if (line[len-1] == '\n')
 229                 line[--len] = 0;
 230         if (!strcmp(line, "NAK"))
 231                 return NAK;
 232         if (!prefixcmp(line, "ACK ")) {
 233                 if (!get_sha1_hex(line+4, result_sha1)) {
 234                         if (strstr(line+45, "continue"))
 235                                 return ACK_continue;
 236                         if (strstr(line+45, "common"))
 237                                 return ACK_common;
 238                         if (strstr(line+45, "ready"))
 239                                 return ACK_ready;
 240                         return ACK;
 241                 }
 242         }
 243         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 244 }
 245
 246 static void send_request(struct fetch_pack_args *args,
 247                          int fd, struct strbuf *buf)
 248 {
 249         if (args->stateless_rpc) {
 250                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 251                 packet_flush(fd);
 252         } else
 253                 safe_write(fd, buf->buf, buf->len);
 254 }
 255
 256 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 257 {
 258         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 259 }
 260
 261 #define INITIAL_FLUSH 16
 262 #define PIPESAFE_FLUSH 32
 263 #define LARGE_FLUSH 1024
 264
 265 static int next_flush(struct fetch_pack_args *args, int count)
 266 {
 267         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 268
 269         if (count < flush_limit)
 270                 count <<= 1;
 271         else
 272                 count += flush_limit;
 273         return count;
 274 }
 275
 276 static int find_common(struct fetch_pack_args *args,
 277                        int fd[2], unsigned char *result_sha1,
 278                        struct ref *refs)
 279 {
 280         int fetching;
 281         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 282         const unsigned char *sha1;
 283         unsigned in_vain = 0;
 284         int got_continue = 0;
 285         int got_ready = 0;
 286         struct strbuf req_buf = STRBUF_INIT;
 287         size_t state_len = 0;
 288
 289         if (args->stateless_rpc && multi_ack == 1)
 290                 die("--stateless-rpc requires multi_ack_detailed");
 291         if (marked)
 292                 for_each_ref(clear_marks, NULL);
 293         marked = 1;
 294
 295         for_each_ref(rev_list_insert_ref, NULL);
 296         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 297
 298         fetching = 0;
 299         for ( ; refs ; refs = refs->next) {
 300                 unsigned char *remote = refs->old_sha1;
 301                 const char *remote_hex;
 302                 struct object *o;
 303
 304                 /*
 305                  * If that object is complete (i.e. it is an ancestor of a
 306                  * local ref), we tell them we have it but do not have to
 307                  * tell them about its ancestors, which they already know
 308                  * about.
 309                  *
 310                  * We use lookup_object here because we are only
 311                  * interested in the case we *know* the object is
 312                  * reachable and we have already scanned it.
 313                  */
 314                 if (((o = lookup_object(remote)) != NULL) &&
 315                                 (o->flags & COMPLETE)) {
 316                         continue;
 317                 }
 318
 319                 remote_hex = sha1_to_hex(remote);
 320                 if (!fetching) {
 321                         struct strbuf c = STRBUF_INIT;
 322                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 323                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 324                         if (no_done)            strbuf_addstr(&c, " no-done");
 325                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 326                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 327                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 328                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 329                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 330                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 331                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 332                                                             git_user_agent_sanitized());
 333                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 334                         strbuf_release(&c);
 335                 } else
 336                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 337                 fetching++;
 338         }
 339
 340         if (!fetching) {
 341                 strbuf_release(&req_buf);
 342                 packet_flush(fd[1]);
 343                 return 1;
 344         }
 345
 346         if (is_repository_shallow())
 347                 write_shallow_commits(&req_buf, 1);
 348         if (args->depth > 0)
 349                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 350         packet_buf_flush(&req_buf);
 351         state_len = req_buf.len;
 352
 353         if (args->depth > 0) {
 354                 char line[1024];
 355                 unsigned char sha1[20];
 356
 357                 send_request(args, fd[1], &req_buf);
 358                 while (packet_read_line(fd[0], line, sizeof(line))) {
 359                         if (!prefixcmp(line, "shallow ")) {
 360                                 if (get_sha1_hex(line + 8, sha1))
 361                                         die("invalid shallow line: %s", line);
 362                                 register_shallow(sha1);
 363                                 continue;
 364                         }
 365                         if (!prefixcmp(line, "unshallow ")) {
 366                                 if (get_sha1_hex(line + 10, sha1))
 367                                         die("invalid unshallow line: %s", line);
 368                                 if (!lookup_object(sha1))
 369                                         die("object not found: %s", line);
 370                                 /* make sure that it is parsed as shallow */
 371                                 if (!parse_object(sha1))
 372                                         die("error in object: %s", line);
 373                                 if (unregister_shallow(sha1))
 374                                         die("no shallow found: %s", line);
 375                                 continue;
 376                         }
 377                         die("expected shallow/unshallow, got %s", line);
 378                 }
 379         } else if (!args->stateless_rpc)
 380                 send_request(args, fd[1], &req_buf);
 381
 382         if (!args->stateless_rpc) {
 383                 /* If we aren't using the stateless-rpc interface
 384                  * we don't need to retain the headers.
 385                  */
 386                 strbuf_setlen(&req_buf, 0);
 387                 state_len = 0;
 388         }
 389
 390         flushes = 0;
 391         retval = -1;
 392         while ((sha1 = get_rev())) {
 393                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 394                 if (args->verbose)
 395                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 396                 in_vain++;
 397                 if (flush_at <= ++count) {
 398                         int ack;
 399
 400                         packet_buf_flush(&req_buf);
 401                         send_request(args, fd[1], &req_buf);
 402                         strbuf_setlen(&req_buf, state_len);
 403                         flushes++;
 404                         flush_at = next_flush(args, count);
 405
 406                         /*
 407                          * We keep one window "ahead" of the other side, and
 408                          * will wait for an ACK only on the next one
 409                          */
 410                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 411                                 continue;
 412
 413                         consume_shallow_list(args, fd[0]);
 414                         do {
 415                                 ack = get_ack(fd[0], result_sha1);
 416                                 if (args->verbose && ack)
 417                                         fprintf(stderr, "got ack %d %s\n", ack,
 418                                                         sha1_to_hex(result_sha1));
 419                                 switch (ack) {
 420                                 case ACK:
 421                                         flushes = 0;
 422                                         multi_ack = 0;
 423                                         retval = 0;
 424                                         goto done;
 425                                 case ACK_common:
 426                                 case ACK_ready:
 427                                 case ACK_continue: {
 428                                         struct commit *commit =
 429                                                 lookup_commit(result_sha1);
 430                                         if (!commit)
 431                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 432                                         if (args->stateless_rpc
 433                                          && ack == ACK_common
 434                                          && !(commit->object.flags & COMMON)) {
 435                                                 /* We need to replay the have for this object
 436                                                  * on the next RPC request so the peer knows
 437                                                  * it is in common with us.
 438                                                  */
 439                                                 const char *hex = sha1_to_hex(result_sha1);
 440                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 441                                                 state_len = req_buf.len;
 442                                         }
 443                                         mark_common(commit, 0, 1);
 444                                         retval = 0;
 445                                         in_vain = 0;
 446                                         got_continue = 1;
 447                                         if (ack == ACK_ready) {
 448                                                 rev_list = NULL;
 449                                                 got_ready = 1;
 450                                         }
 451                                         break;
 452                                         }
 453                                 }
 454                         } while (ack);
 455                         flushes--;
 456                         if (got_continue && MAX_IN_VAIN < in_vain) {
 457                                 if (args->verbose)
 458                                         fprintf(stderr, "giving up\n");
 459                                 break; /* give up */
 460                         }
 461                 }
 462         }
 463 done:
 464         if (!got_ready || !no_done) {
 465                 packet_buf_write(&req_buf, "done\n");
 466                 send_request(args, fd[1], &req_buf);
 467         }
 468         if (args->verbose)
 469                 fprintf(stderr, "done\n");
 470         if (retval != 0) {
 471                 multi_ack = 0;
 472                 flushes++;
 473         }
 474         strbuf_release(&req_buf);
 475
 476         consume_shallow_list(args, fd[0]);
 477         while (flushes || multi_ack) {
 478                 int ack = get_ack(fd[0], result_sha1);
 479                 if (ack) {
 480                         if (args->verbose)
 481                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 482                                         sha1_to_hex(result_sha1));
 483                         if (ack == ACK)
 484                                 return 0;
 485                         multi_ack = 1;
 486                         continue;
 487                 }
 488                 flushes--;
 489         }
 490         /* it is no error to fetch into a completely empty repo */
 491         return count ? retval : 0;
 492 }
 493
 494 static struct commit_list *complete;
 495
 496 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 497 {
 498         struct object *o = parse_object(sha1);
 499
 500         while (o && o->type == OBJ_TAG) {
 501                 struct tag *t = (struct tag *) o;
 502                 if (!t->tagged)
 503                         break; /* broken repository */
 504                 o->flags |= COMPLETE;
 505                 o = parse_object(t->tagged->sha1);
 506         }
 507         if (o && o->type == OBJ_COMMIT) {
 508                 struct commit *commit = (struct commit *)o;
 509                 if (!(commit->object.flags & COMPLETE)) {
 510                         commit->object.flags |= COMPLETE;
 511                         commit_list_insert_by_date(commit, &complete);
 512                 }
 513         }
 514         return 0;
 515 }
 516
 517 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 518                                          unsigned long cutoff)
 519 {
 520         while (complete && cutoff <= complete->item->date) {
 521                 if (args->verbose)
 522                         fprintf(stderr, "Marking %s as complete\n",
 523                                 sha1_to_hex(complete->item->object.sha1));
 524                 pop_most_recent_commit(&complete, COMPLETE);
 525         }
 526 }
 527
 528 static int non_matching_ref(struct string_list_item *item, void *unused)
 529 {
 530         if (item->util) {
 531                 item->util = NULL;
 532                 return 0;
 533         }
 534         else
 535                 return 1;
 536 }
 537
 538 static void filter_refs(struct fetch_pack_args *args,
 539                         struct ref **refs, struct string_list *sought)
 540 {
 541         struct ref *newlist = NULL;
 542         struct ref **newtail = &newlist;
 543         struct ref *ref, *next;
 544         int sought_pos;
 545
 546         sought_pos = 0;
 547         for (ref = *refs; ref; ref = next) {
 548                 int keep = 0;
 549                 next = ref->next;
 550                 if (!memcmp(ref->name, "refs/", 5) &&
 551                     check_refname_format(ref->name + 5, 0))
 552                         ; /* trash */
 553                 else {
 554                         while (sought_pos < sought->nr) {
 555                                 int cmp = strcmp(ref->name, sought->items[sought_pos].string);
 556                                 if (cmp < 0)
 557                                         break; /* definitely do not have it */
 558                                 else if (cmp == 0) {
 559                                         keep = 1; /* definitely have it */
 560                                         sought->items[sought_pos++].util = "matched";
 561                                         break;
 562                                 }
 563                                 else
 564                                         sought_pos++; /* might have it; keep looking */
 565                         }
 566                 }
 567
 568                 if (! keep && args->fetch_all &&
 569                     (!args->depth || prefixcmp(ref->name, "refs/tags/")))
 570                         keep = 1;
 571
 572                 if (keep) {
 573                         *newtail = ref;
 574                         ref->next = NULL;
 575                         newtail = &ref->next;
 576                 } else {
 577                         free(ref);
 578                 }
 579         }
 580
 581         filter_string_list(sought, 0, non_matching_ref, NULL);
 582         *refs = newlist;
 583 }
 584
 585 static void mark_alternate_complete(const struct ref *ref, void *unused)
 586 {
 587         mark_complete(NULL, ref->old_sha1, 0, NULL);
 588 }
 589
 590 static int everything_local(struct fetch_pack_args *args,
 591                             struct ref **refs, struct string_list *sought)
 592 {
 593         struct ref *ref;
 594         int retval;
 595         unsigned long cutoff = 0;
 596
 597         save_commit_buffer = 0;
 598
 599         for (ref = *refs; ref; ref = ref->next) {
 600                 struct object *o;
 601
 602                 o = parse_object(ref->old_sha1);
 603                 if (!o)
 604                         continue;
 605
 606                 /* We already have it -- which may mean that we were
 607                  * in sync with the other side at some time after
 608                  * that (it is OK if we guess wrong here).
 609                  */
 610                 if (o->type == OBJ_COMMIT) {
 611                         struct commit *commit = (struct commit *)o;
 612                         if (!cutoff || cutoff < commit->date)
 613                                 cutoff = commit->date;
 614                 }
 615         }
 616
 617         if (!args->depth) {
 618                 for_each_ref(mark_complete, NULL);
 619                 for_each_alternate_ref(mark_alternate_complete, NULL);
 620                 if (cutoff)
 621                         mark_recent_complete_commits(args, cutoff);
 622         }
 623
 624         /*
 625          * Mark all complete remote refs as common refs.
 626          * Don't mark them common yet; the server has to be told so first.
 627          */
 628         for (ref = *refs; ref; ref = ref->next) {
 629                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 630                                              NULL, 0);
 631
 632                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 633                         continue;
 634
 635                 if (!(o->flags & SEEN)) {
 636                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 637
 638                         mark_common((struct commit *)o, 1, 1);
 639                 }
 640         }
 641
 642         filter_refs(args, refs, sought);
 643
 644         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 645                 const unsigned char *remote = ref->old_sha1;
 646                 unsigned char local[20];
 647                 struct object *o;
 648
 649                 o = lookup_object(remote);
 650                 if (!o || !(o->flags & COMPLETE)) {
 651                         retval = 0;
 652                         if (!args->verbose)
 653                                 continue;
 654                         fprintf(stderr,
 655                                 "want %s (%s)\n", sha1_to_hex(remote),
 656                                 ref->name);
 657                         continue;
 658                 }
 659
 660                 hashcpy(ref->new_sha1, local);
 661                 if (!args->verbose)
 662                         continue;
 663                 fprintf(stderr,
 664                         "already have %s (%s)\n", sha1_to_hex(remote),
 665                         ref->name);
 666         }
 667         return retval;
 668 }
 669
 670 static int sideband_demux(int in, int out, void *data)
 671 {
 672         int *xd = data;
 673
 674         int ret = recv_sideband("fetch-pack", xd[0], out);
 675         close(out);
 676         return ret;
 677 }
 678
 679 static int get_pack(struct fetch_pack_args *args,
 680                     int xd[2], char **pack_lockfile)
 681 {
 682         struct async demux;
 683         const char *argv[20];
 684         char keep_arg[256];
 685         char hdr_arg[256];
 686         const char **av;
 687         int do_keep = args->keep_pack;
 688         struct child_process cmd;
 689
 690         memset(&demux, 0, sizeof(demux));
 691         if (use_sideband) {
 692                 /* xd[] is talking with upload-pack; subprocess reads from
 693                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 694                  * through demux->out.
 695                  */
 696                 demux.proc = sideband_demux;
 697                 demux.data = xd;
 698                 demux.out = -1;
 699                 if (start_async(&demux))
 700                         die("fetch-pack: unable to fork off sideband"
 701                             " demultiplexer");
 702         }
 703         else
 704                 demux.out = xd[0];
 705
 706         memset(&cmd, 0, sizeof(cmd));
 707         cmd.argv = argv;
 708         av = argv;
 709         *hdr_arg = 0;
 710         if (!args->keep_pack && unpack_limit) {
 711                 struct pack_header header;
 712
 713                 if (read_pack_header(demux.out, &header))
 714                         die("protocol error: bad pack header");
 715                 snprintf(hdr_arg, sizeof(hdr_arg),
 716                          "--pack_header=%"PRIu32",%"PRIu32,
 717                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 718                 if (ntohl(header.hdr_entries) < unpack_limit)
 719                         do_keep = 0;
 720                 else
 721                         do_keep = 1;
 722         }
 723
 724         if (do_keep) {
 725                 if (pack_lockfile)
 726                         cmd.out = -1;
 727                 *av++ = "index-pack";
 728                 *av++ = "--stdin";
 729                 if (!args->quiet && !args->no_progress)
 730                         *av++ = "-v";
 731                 if (args->use_thin_pack)
 732                         *av++ = "--fix-thin";
 733                 if (args->lock_pack || unpack_limit) {
 734                         int s = sprintf(keep_arg,
 735                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 736                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 737                                 strcpy(keep_arg + s, "localhost");
 738                         *av++ = keep_arg;
 739                 }
 740         }
 741         else {
 742                 *av++ = "unpack-objects";
 743                 if (args->quiet || args->no_progress)
 744                         *av++ = "-q";
 745         }
 746         if (*hdr_arg)
 747                 *av++ = hdr_arg;
 748         if (fetch_fsck_objects >= 0
 749             ? fetch_fsck_objects
 750             : transfer_fsck_objects >= 0
 751             ? transfer_fsck_objects
 752             : 0)
 753                 *av++ = "--strict";
 754         *av++ = NULL;
 755
 756         cmd.in = demux.out;
 757         cmd.git_cmd = 1;
 758         if (start_command(&cmd))
 759                 die("fetch-pack: unable to fork off %s", argv[0]);
 760         if (do_keep && pack_lockfile) {
 761                 *pack_lockfile = index_pack_lockfile(cmd.out);
 762                 close(cmd.out);
 763         }
 764
 765         if (finish_command(&cmd))
 766                 die("%s failed", argv[0]);
 767         if (use_sideband && finish_async(&demux))
 768                 die("error in sideband demultiplexer");
 769         return 0;
 770 }
 771
 772 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 773                                  int fd[2],
 774                                  const struct ref *orig_ref,
 775                                  struct string_list *sought,
 776                                  char **pack_lockfile)
 777 {
 778         struct ref *ref = copy_ref_list(orig_ref);
 779         unsigned char sha1[20];
 780         const char *agent_feature;
 781         int agent_len;
 782
 783         sort_ref_list(&ref, ref_compare_name);
 784
 785         if (is_repository_shallow() && !server_supports("shallow"))
 786                 die("Server does not support shallow clients");
 787         if (server_supports("multi_ack_detailed")) {
 788                 if (args->verbose)
 789                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 790                 multi_ack = 2;
 791                 if (server_supports("no-done")) {
 792                         if (args->verbose)
 793                                 fprintf(stderr, "Server supports no-done\n");
 794                         if (args->stateless_rpc)
 795                                 no_done = 1;
 796                 }
 797         }
 798         else if (server_supports("multi_ack")) {
 799                 if (args->verbose)
 800                         fprintf(stderr, "Server supports multi_ack\n");
 801                 multi_ack = 1;
 802         }
 803         if (server_supports("side-band-64k")) {
 804                 if (args->verbose)
 805                         fprintf(stderr, "Server supports side-band-64k\n");
 806                 use_sideband = 2;
 807         }
 808         else if (server_supports("side-band")) {
 809                 if (args->verbose)
 810                         fprintf(stderr, "Server supports side-band\n");
 811                 use_sideband = 1;
 812         }
 813         if (!server_supports("thin-pack"))
 814                 args->use_thin_pack = 0;
 815         if (!server_supports("no-progress"))
 816                 args->no_progress = 0;
 817         if (!server_supports("include-tag"))
 818                 args->include_tag = 0;
 819         if (server_supports("ofs-delta")) {
 820                 if (args->verbose)
 821                         fprintf(stderr, "Server supports ofs-delta\n");
 822         } else
 823                 prefer_ofs_delta = 0;
 824
 825         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 826                 agent_supported = 1;
 827                 if (args->verbose && agent_len)
 828                         fprintf(stderr, "Server version is %.*s\n",
 829                                 agent_len, agent_feature);
 830         }
 831
 832         if (everything_local(args, &ref, sought)) {
 833                 packet_flush(fd[1]);
 834                 goto all_done;
 835         }
 836         if (find_common(args, fd, sha1, ref) < 0)
 837                 if (!args->keep_pack)
 838                         /* When cloning, it is not unusual to have
 839                          * no common commit.
 840                          */
 841                         warning("no common commits");
 842
 843         if (args->stateless_rpc)
 844                 packet_flush(fd[1]);
 845         if (get_pack(args, fd, pack_lockfile))
 846                 die("git fetch-pack: fetch failed.");
 847
 848  all_done:
 849         return ref;
 850 }
 851
 852 static int fetch_pack_config(const char *var, const char *value, void *cb)
 853 {
 854         if (strcmp(var, "fetch.unpacklimit") == 0) {
 855                 fetch_unpack_limit = git_config_int(var, value);
 856                 return 0;
 857         }
 858
 859         if (strcmp(var, "transfer.unpacklimit") == 0) {
 860                 transfer_unpack_limit = git_config_int(var, value);
 861                 return 0;
 862         }
 863
 864         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 865                 prefer_ofs_delta = git_config_bool(var, value);
 866                 return 0;
 867         }
 868
 869         if (!strcmp(var, "fetch.fsckobjects")) {
 870                 fetch_fsck_objects = git_config_bool(var, value);
 871                 return 0;
 872         }
 873
 874         if (!strcmp(var, "transfer.fsckobjects")) {
 875                 transfer_fsck_objects = git_config_bool(var, value);
 876                 return 0;
 877         }
 878
 879         return git_default_config(var, value, cb);
 880 }
 881
 882 static struct lock_file lock;
 883
 884 static void fetch_pack_setup(void)
 885 {
 886         static int did_setup;
 887         if (did_setup)
 888                 return;
 889         git_config(fetch_pack_config, NULL);
 890         if (0 <= transfer_unpack_limit)
 891                 unpack_limit = transfer_unpack_limit;
 892         else if (0 <= fetch_unpack_limit)
 893                 unpack_limit = fetch_unpack_limit;
 894         did_setup = 1;
 895 }
 896
 897 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 898 {
 899         int i, ret;
 900         struct ref *ref = NULL;
 901         const char *dest = NULL;
 902         struct string_list sought = STRING_LIST_INIT_DUP;
 903         int fd[2];
 904         char *pack_lockfile = NULL;
 905         char **pack_lockfile_ptr = NULL;
 906         struct child_process *conn;
 907         struct fetch_pack_args args;
 908
 909         packet_trace_identity("fetch-pack");
 910
 911         memset(&args, 0, sizeof(args));
 912         args.uploadpack = "git-upload-pack";
 913
 914         for (i = 1; i < argc && *argv[i] == '-'; i++) {
 915                 const char *arg = argv[i];
 916
 917                 if (!prefixcmp(arg, "--upload-pack=")) {
 918                         args.uploadpack = arg + 14;
 919                         continue;
 920                 }
 921                 if (!prefixcmp(arg, "--exec=")) {
 922                         args.uploadpack = arg + 7;
 923                         continue;
 924                 }
 925                 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 926                         args.quiet = 1;
 927                         continue;
 928                 }
 929                 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 930                         args.lock_pack = args.keep_pack;
 931                         args.keep_pack = 1;
 932                         continue;
 933                 }
 934                 if (!strcmp("--thin", arg)) {
 935                         args.use_thin_pack = 1;
 936                         continue;
 937                 }
 938                 if (!strcmp("--include-tag", arg)) {
 939                         args.include_tag = 1;
 940                         continue;
 941                 }
 942                 if (!strcmp("--all", arg)) {
 943                         args.fetch_all = 1;
 944                         continue;
 945                 }
 946                 if (!strcmp("--stdin", arg)) {
 947                         args.stdin_refs = 1;
 948                         continue;
 949                 }
 950                 if (!strcmp("-v", arg)) {
 951                         args.verbose = 1;
 952                         continue;
 953                 }
 954                 if (!prefixcmp(arg, "--depth=")) {
 955                         args.depth = strtol(arg + 8, NULL, 0);
 956                         continue;
 957                 }
 958                 if (!strcmp("--no-progress", arg)) {
 959                         args.no_progress = 1;
 960                         continue;
 961                 }
 962                 if (!strcmp("--stateless-rpc", arg)) {
 963                         args.stateless_rpc = 1;
 964                         continue;
 965                 }
 966                 if (!strcmp("--lock-pack", arg)) {
 967                         args.lock_pack = 1;
 968                         pack_lockfile_ptr = &pack_lockfile;
 969                         continue;
 970                 }
 971                 usage(fetch_pack_usage);
 972         }
 973
 974         if (i < argc)
 975                 dest = argv[i++];
 976         else
 977                 usage(fetch_pack_usage);
 978
 979         /*
 980          * Copy refs from cmdline to growable list, then append any
 981          * refs from the standard input:
 982          */
 983         for (; i < argc; i++)
 984                 string_list_append(&sought, xstrdup(argv[i]));
 985         if (args.stdin_refs) {
 986                 if (args.stateless_rpc) {
 987                         /* in stateless RPC mode we use pkt-line to read
 988                          * from stdin, until we get a flush packet
 989                          */
 990                         static char line[1000];
 991                         for (;;) {
 992                                 int n = packet_read_line(0, line, sizeof(line));
 993                                 if (!n)
 994                                         break;
 995                                 if (line[n-1] == '\n')
 996                                         n--;
 997                                 string_list_append(&sought, xmemdupz(line, n));
 998                         }
 999                 }
1000                 else {
1001                         /* read from stdin one ref per line, until EOF */
1002                         struct strbuf line = STRBUF_INIT;
1003                         while (strbuf_getline(&line, stdin, '\n') != EOF)
1004                                 string_list_append(&sought, strbuf_detach(&line, NULL));
1005                         strbuf_release(&line);
1006                 }
1007         }
1008
1009         if (args.stateless_rpc) {
1010                 conn = NULL;
1011                 fd[0] = 0;
1012                 fd[1] = 1;
1013         } else {
1014                 conn = git_connect(fd, dest, args.uploadpack,
1015                                    args.verbose ? CONNECT_VERBOSE : 0);
1016         }
1017
1018         get_remote_heads(fd[0], &ref, 0, NULL);
1019
1020         ref = fetch_pack(&args, fd, conn, ref, dest,
1021                          &sought, pack_lockfile_ptr);
1022         if (pack_lockfile) {
1023                 printf("lock %s\n", pack_lockfile);
1024                 fflush(stdout);
1025         }
1026         close(fd[0]);
1027         close(fd[1]);
1028         if (finish_connect(conn))
1029                 return 1;
1030
1031         ret = !ref || sought.nr;
1032
1033         /*
1034          * If the heads to pull were given, we should have consumed
1035          * all of them by matching the remote.  Otherwise, 'git fetch
1036          * remote no-such-ref' would silently succeed without issuing
1037          * an error.
1038          */
1039         for (i = 0; i < sought.nr; i++)
1040                 error("no such remote ref %s", sought.items[i].string);
1041         while (ref) {
1042                 printf("%s %s\n",
1043                        sha1_to_hex(ref->old_sha1), ref->name);
1044                 ref = ref->next;
1045         }
1046
1047         return ret;
1048 }
1049
1050 struct ref *fetch_pack(struct fetch_pack_args *args,
1051                        int fd[], struct child_process *conn,
1052                        const struct ref *ref,
1053                        const char *dest,
1054                        struct string_list *sought,
1055                        char **pack_lockfile)
1056 {
1057         struct stat st;
1058         struct ref *ref_cpy;
1059
1060         fetch_pack_setup();
1061         if (args->depth > 0) {
1062                 if (stat(git_path("shallow"), &st))
1063                         st.st_mtime = 0;
1064         }
1065
1066         if (sought->nr) {
1067                 sort_string_list(sought);
1068                 string_list_remove_duplicates(sought, 0);
1069         }
1070
1071         if (!ref) {
1072                 packet_flush(fd[1]);
1073                 die("no matching remote head");
1074         }
1075         ref_cpy = do_fetch_pack(args, fd, ref, sought, pack_lockfile);
1076
1077         if (args->depth > 0) {
1078                 struct cache_time mtime;
1079                 struct strbuf sb = STRBUF_INIT;
1080                 char *shallow = git_path("shallow");
1081                 int fd;
1082
1083                 mtime.sec = st.st_mtime;
1084                 mtime.nsec = ST_MTIME_NSEC(st);
1085                 if (stat(shallow, &st)) {
1086                         if (mtime.sec)
1087                                 die("shallow file was removed during fetch");
1088                 } else if (st.st_mtime != mtime.sec
1089 #ifdef USE_NSEC
1090                                 || ST_MTIME_NSEC(st) != mtime.nsec
1091 #endif
1092                           )
1093                         die("shallow file was changed during fetch");
1094
1095                 fd = hold_lock_file_for_update(&lock, shallow,
1096                                                LOCK_DIE_ON_ERROR);
1097                 if (!write_shallow_commits(&sb, 0)
1098                  || write_in_full(fd, sb.buf, sb.len) != sb.len) {
1099                         unlink_or_warn(shallow);
1100                         rollback_lock_file(&lock);
1101                 } else {
1102                         commit_lock_file(&lock);
1103                 }
1104                 strbuf_release(&sb);
1105         }
1106
1107         reprepare_packed_git();
1108         return ref_cpy;
1109 }