builtin/fetch-pack.c

   1 #include "builtin.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "transport.h"
  13 #include "version.h"
  14
  15 static int transfer_unpack_limit = -1;
  16 static int fetch_unpack_limit = -1;
  17 static int unpack_limit = 100;
  18 static int prefer_ofs_delta = 1;
  19 static int no_done;
  20 static int fetch_fsck_objects = -1;
  21 static int transfer_fsck_objects = -1;
  22 static int agent_supported;
  23 static struct fetch_pack_args args = {
  24         /* .uploadpack = */ "git-upload-pack",
  25 };
  26
  27 static const char fetch_pack_usage[] =
  28 "git fetch-pack [--all] [--stdin] [--quiet|-q] [--keep|-k] [--thin] "
  29 "[--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] "
  30 "[--no-progress] [-v] [<host>:]<directory> [<refs>...]";
  31
  32 #define COMPLETE        (1U << 0)
  33 #define COMMON          (1U << 1)
  34 #define COMMON_REF      (1U << 2)
  35 #define SEEN            (1U << 3)
  36 #define POPPED          (1U << 4)
  37
  38 static int marked;
  39
  40 /*
  41  * After sending this many "have"s if we do not get any new ACK , we
  42  * give up traversing our history.
  43  */
  44 #define MAX_IN_VAIN 256
  45
  46 static struct commit_list *rev_list;
  47 static int non_common_revs, multi_ack, use_sideband;
  48
  49 static void rev_list_push(struct commit *commit, int mark)
  50 {
  51         if (!(commit->object.flags & mark)) {
  52                 commit->object.flags |= mark;
  53
  54                 if (!(commit->object.parsed))
  55                         if (parse_commit(commit))
  56                                 return;
  57
  58                 commit_list_insert_by_date(commit, &rev_list);
  59
  60                 if (!(commit->object.flags & COMMON))
  61                         non_common_revs++;
  62         }
  63 }
  64
  65 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  66 {
  67         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  68
  69         if (o && o->type == OBJ_COMMIT)
  70                 rev_list_push((struct commit *)o, SEEN);
  71
  72         return 0;
  73 }
  74
  75 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  76 {
  77         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  78
  79         if (o && o->type == OBJ_COMMIT)
  80                 clear_commit_marks((struct commit *)o,
  81                                    COMMON | COMMON_REF | SEEN | POPPED);
  82         return 0;
  83 }
  84
  85 /*
  86    This function marks a rev and its ancestors as common.
  87    In some cases, it is desirable to mark only the ancestors (for example
  88    when only the server does not yet know that they are common).
  89 */
  90
  91 static void mark_common(struct commit *commit,
  92                 int ancestors_only, int dont_parse)
  93 {
  94         if (commit != NULL && !(commit->object.flags & COMMON)) {
  95                 struct object *o = (struct object *)commit;
  96
  97                 if (!ancestors_only)
  98                         o->flags |= COMMON;
  99
 100                 if (!(o->flags & SEEN))
 101                         rev_list_push(commit, SEEN);
 102                 else {
 103                         struct commit_list *parents;
 104
 105                         if (!ancestors_only && !(o->flags & POPPED))
 106                                 non_common_revs--;
 107                         if (!o->parsed && !dont_parse)
 108                                 if (parse_commit(commit))
 109                                         return;
 110
 111                         for (parents = commit->parents;
 112                                         parents;
 113                                         parents = parents->next)
 114                                 mark_common(parents->item, 0, dont_parse);
 115                 }
 116         }
 117 }
 118
 119 /*
 120   Get the next rev to send, ignoring the common.
 121 */
 122
 123 static const unsigned char *get_rev(void)
 124 {
 125         struct commit *commit = NULL;
 126
 127         while (commit == NULL) {
 128                 unsigned int mark;
 129                 struct commit_list *parents;
 130
 131                 if (rev_list == NULL || non_common_revs == 0)
 132                         return NULL;
 133
 134                 commit = rev_list->item;
 135                 if (!commit->object.parsed)
 136                         parse_commit(commit);
 137                 parents = commit->parents;
 138
 139                 commit->object.flags |= POPPED;
 140                 if (!(commit->object.flags & COMMON))
 141                         non_common_revs--;
 142
 143                 if (commit->object.flags & COMMON) {
 144                         /* do not send "have", and ignore ancestors */
 145                         commit = NULL;
 146                         mark = COMMON | SEEN;
 147                 } else if (commit->object.flags & COMMON_REF)
 148                         /* send "have", and ignore ancestors */
 149                         mark = COMMON | SEEN;
 150                 else
 151                         /* send "have", also for its ancestors */
 152                         mark = SEEN;
 153
 154                 while (parents) {
 155                         if (!(parents->item->object.flags & SEEN))
 156                                 rev_list_push(parents->item, mark);
 157                         if (mark & COMMON)
 158                                 mark_common(parents->item, 1, 0);
 159                         parents = parents->next;
 160                 }
 161
 162                 rev_list = rev_list->next;
 163         }
 164
 165         return commit->object.sha1;
 166 }
 167
 168 enum ack_type {
 169         NAK = 0,
 170         ACK,
 171         ACK_continue,
 172         ACK_common,
 173         ACK_ready
 174 };
 175
 176 static void consume_shallow_list(int fd)
 177 {
 178         if (args.stateless_rpc && args.depth > 0) {
 179                 /* If we sent a depth we will get back "duplicate"
 180                  * shallow and unshallow commands every time there
 181                  * is a block of have lines exchanged.
 182                  */
 183                 char line[1000];
 184                 while (packet_read_line(fd, line, sizeof(line))) {
 185                         if (!prefixcmp(line, "shallow "))
 186                                 continue;
 187                         if (!prefixcmp(line, "unshallow "))
 188                                 continue;
 189                         die("git fetch-pack: expected shallow list");
 190                 }
 191         }
 192 }
 193
 194 struct write_shallow_data {
 195         struct strbuf *out;
 196         int use_pack_protocol;
 197         int count;
 198 };
 199
 200 static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
 201 {
 202         struct write_shallow_data *data = cb_data;
 203         const char *hex = sha1_to_hex(graft->sha1);
 204         data->count++;
 205         if (data->use_pack_protocol)
 206                 packet_buf_write(data->out, "shallow %s", hex);
 207         else {
 208                 strbuf_addstr(data->out, hex);
 209                 strbuf_addch(data->out, '\n');
 210         }
 211         return 0;
 212 }
 213
 214 static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
 215 {
 216         struct write_shallow_data data;
 217         data.out = out;
 218         data.use_pack_protocol = use_pack_protocol;
 219         data.count = 0;
 220         for_each_commit_graft(write_one_shallow, &data);
 221         return data.count;
 222 }
 223
 224 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 225 {
 226         static char line[1000];
 227         int len = packet_read_line(fd, line, sizeof(line));
 228
 229         if (!len)
 230                 die("git fetch-pack: expected ACK/NAK, got EOF");
 231         if (line[len-1] == '\n')
 232                 line[--len] = 0;
 233         if (!strcmp(line, "NAK"))
 234                 return NAK;
 235         if (!prefixcmp(line, "ACK ")) {
 236                 if (!get_sha1_hex(line+4, result_sha1)) {
 237                         if (strstr(line+45, "continue"))
 238                                 return ACK_continue;
 239                         if (strstr(line+45, "common"))
 240                                 return ACK_common;
 241                         if (strstr(line+45, "ready"))
 242                                 return ACK_ready;
 243                         return ACK;
 244                 }
 245         }
 246         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 247 }
 248
 249 static void send_request(int fd, struct strbuf *buf)
 250 {
 251         if (args.stateless_rpc) {
 252                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 253                 packet_flush(fd);
 254         } else
 255                 safe_write(fd, buf->buf, buf->len);
 256 }
 257
 258 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 259 {
 260         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 261 }
 262
 263 #define INITIAL_FLUSH 16
 264 #define PIPESAFE_FLUSH 32
 265 #define LARGE_FLUSH 1024
 266
 267 static int next_flush(int count)
 268 {
 269         int flush_limit = args.stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 270
 271         if (count < flush_limit)
 272                 count <<= 1;
 273         else
 274                 count += flush_limit;
 275         return count;
 276 }
 277
 278 static int find_common(int fd[2], unsigned char *result_sha1,
 279                        struct ref *refs)
 280 {
 281         int fetching;
 282         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 283         const unsigned char *sha1;
 284         unsigned in_vain = 0;
 285         int got_continue = 0;
 286         int got_ready = 0;
 287         struct strbuf req_buf = STRBUF_INIT;
 288         size_t state_len = 0;
 289
 290         if (args.stateless_rpc && multi_ack == 1)
 291                 die("--stateless-rpc requires multi_ack_detailed");
 292         if (marked)
 293                 for_each_ref(clear_marks, NULL);
 294         marked = 1;
 295
 296         for_each_ref(rev_list_insert_ref, NULL);
 297         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 298
 299         fetching = 0;
 300         for ( ; refs ; refs = refs->next) {
 301                 unsigned char *remote = refs->old_sha1;
 302                 const char *remote_hex;
 303                 struct object *o;
 304
 305                 /*
 306                  * If that object is complete (i.e. it is an ancestor of a
 307                  * local ref), we tell them we have it but do not have to
 308                  * tell them about its ancestors, which they already know
 309                  * about.
 310                  *
 311                  * We use lookup_object here because we are only
 312                  * interested in the case we *know* the object is
 313                  * reachable and we have already scanned it.
 314                  */
 315                 if (((o = lookup_object(remote)) != NULL) &&
 316                                 (o->flags & COMPLETE)) {
 317                         continue;
 318                 }
 319
 320                 remote_hex = sha1_to_hex(remote);
 321                 if (!fetching) {
 322                         struct strbuf c = STRBUF_INIT;
 323                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 324                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 325                         if (no_done)            strbuf_addstr(&c, " no-done");
 326                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 327                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 328                         if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack");
 329                         if (args.no_progress)   strbuf_addstr(&c, " no-progress");
 330                         if (args.include_tag)   strbuf_addstr(&c, " include-tag");
 331                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 332                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 333                                                             git_user_agent_sanitized());
 334                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 335                         strbuf_release(&c);
 336                 } else
 337                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 338                 fetching++;
 339         }
 340
 341         if (!fetching) {
 342                 strbuf_release(&req_buf);
 343                 packet_flush(fd[1]);
 344                 return 1;
 345         }
 346
 347         if (is_repository_shallow())
 348                 write_shallow_commits(&req_buf, 1);
 349         if (args.depth > 0)
 350                 packet_buf_write(&req_buf, "deepen %d", args.depth);
 351         packet_buf_flush(&req_buf);
 352         state_len = req_buf.len;
 353
 354         if (args.depth > 0) {
 355                 char line[1024];
 356                 unsigned char sha1[20];
 357
 358                 send_request(fd[1], &req_buf);
 359                 while (packet_read_line(fd[0], line, sizeof(line))) {
 360                         if (!prefixcmp(line, "shallow ")) {
 361                                 if (get_sha1_hex(line + 8, sha1))
 362                                         die("invalid shallow line: %s", line);
 363                                 register_shallow(sha1);
 364                                 continue;
 365                         }
 366                         if (!prefixcmp(line, "unshallow ")) {
 367                                 if (get_sha1_hex(line + 10, sha1))
 368                                         die("invalid unshallow line: %s", line);
 369                                 if (!lookup_object(sha1))
 370                                         die("object not found: %s", line);
 371                                 /* make sure that it is parsed as shallow */
 372                                 if (!parse_object(sha1))
 373                                         die("error in object: %s", line);
 374                                 if (unregister_shallow(sha1))
 375                                         die("no shallow found: %s", line);
 376                                 continue;
 377                         }
 378                         die("expected shallow/unshallow, got %s", line);
 379                 }
 380         } else if (!args.stateless_rpc)
 381                 send_request(fd[1], &req_buf);
 382
 383         if (!args.stateless_rpc) {
 384                 /* If we aren't using the stateless-rpc interface
 385                  * we don't need to retain the headers.
 386                  */
 387                 strbuf_setlen(&req_buf, 0);
 388                 state_len = 0;
 389         }
 390
 391         flushes = 0;
 392         retval = -1;
 393         while ((sha1 = get_rev())) {
 394                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 395                 if (args.verbose)
 396                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 397                 in_vain++;
 398                 if (flush_at <= ++count) {
 399                         int ack;
 400
 401                         packet_buf_flush(&req_buf);
 402                         send_request(fd[1], &req_buf);
 403                         strbuf_setlen(&req_buf, state_len);
 404                         flushes++;
 405                         flush_at = next_flush(count);
 406
 407                         /*
 408                          * We keep one window "ahead" of the other side, and
 409                          * will wait for an ACK only on the next one
 410                          */
 411                         if (!args.stateless_rpc && count == INITIAL_FLUSH)
 412                                 continue;
 413
 414                         consume_shallow_list(fd[0]);
 415                         do {
 416                                 ack = get_ack(fd[0], result_sha1);
 417                                 if (args.verbose && ack)
 418                                         fprintf(stderr, "got ack %d %s\n", ack,
 419                                                         sha1_to_hex(result_sha1));
 420                                 switch (ack) {
 421                                 case ACK:
 422                                         flushes = 0;
 423                                         multi_ack = 0;
 424                                         retval = 0;
 425                                         goto done;
 426                                 case ACK_common:
 427                                 case ACK_ready:
 428                                 case ACK_continue: {
 429                                         struct commit *commit =
 430                                                 lookup_commit(result_sha1);
 431                                         if (!commit)
 432                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 433                                         if (args.stateless_rpc
 434                                          && ack == ACK_common
 435                                          && !(commit->object.flags & COMMON)) {
 436                                                 /* We need to replay the have for this object
 437                                                  * on the next RPC request so the peer knows
 438                                                  * it is in common with us.
 439                                                  */
 440                                                 const char *hex = sha1_to_hex(result_sha1);
 441                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 442                                                 state_len = req_buf.len;
 443                                         }
 444                                         mark_common(commit, 0, 1);
 445                                         retval = 0;
 446                                         in_vain = 0;
 447                                         got_continue = 1;
 448                                         if (ack == ACK_ready) {
 449                                                 rev_list = NULL;
 450                                                 got_ready = 1;
 451                                         }
 452                                         break;
 453                                         }
 454                                 }
 455                         } while (ack);
 456                         flushes--;
 457                         if (got_continue && MAX_IN_VAIN < in_vain) {
 458                                 if (args.verbose)
 459                                         fprintf(stderr, "giving up\n");
 460                                 break; /* give up */
 461                         }
 462                 }
 463         }
 464 done:
 465         if (!got_ready || !no_done) {
 466                 packet_buf_write(&req_buf, "done\n");
 467                 send_request(fd[1], &req_buf);
 468         }
 469         if (args.verbose)
 470                 fprintf(stderr, "done\n");
 471         if (retval != 0) {
 472                 multi_ack = 0;
 473                 flushes++;
 474         }
 475         strbuf_release(&req_buf);
 476
 477         consume_shallow_list(fd[0]);
 478         while (flushes || multi_ack) {
 479                 int ack = get_ack(fd[0], result_sha1);
 480                 if (ack) {
 481                         if (args.verbose)
 482                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 483                                         sha1_to_hex(result_sha1));
 484                         if (ack == ACK)
 485                                 return 0;
 486                         multi_ack = 1;
 487                         continue;
 488                 }
 489                 flushes--;
 490         }
 491         /* it is no error to fetch into a completely empty repo */
 492         return count ? retval : 0;
 493 }
 494
 495 static struct commit_list *complete;
 496
 497 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 498 {
 499         struct object *o = parse_object(sha1);
 500
 501         while (o && o->type == OBJ_TAG) {
 502                 struct tag *t = (struct tag *) o;
 503                 if (!t->tagged)
 504                         break; /* broken repository */
 505                 o->flags |= COMPLETE;
 506                 o = parse_object(t->tagged->sha1);
 507         }
 508         if (o && o->type == OBJ_COMMIT) {
 509                 struct commit *commit = (struct commit *)o;
 510                 if (!(commit->object.flags & COMPLETE)) {
 511                         commit->object.flags |= COMPLETE;
 512                         commit_list_insert_by_date(commit, &complete);
 513                 }
 514         }
 515         return 0;
 516 }
 517
 518 static void mark_recent_complete_commits(unsigned long cutoff)
 519 {
 520         while (complete && cutoff <= complete->item->date) {
 521                 if (args.verbose)
 522                         fprintf(stderr, "Marking %s as complete\n",
 523                                 sha1_to_hex(complete->item->object.sha1));
 524                 pop_most_recent_commit(&complete, COMPLETE);
 525         }
 526 }
 527
 528 static void filter_refs(struct ref **refs, int nr_match, char **match)
 529 {
 530         struct ref **return_refs;
 531         struct ref *newlist = NULL;
 532         struct ref **newtail = &newlist;
 533         struct ref *ref, *next;
 534         struct ref *fastarray[32];
 535         int match_pos;
 536
 537         if (nr_match && !args.fetch_all) {
 538                 if (ARRAY_SIZE(fastarray) < nr_match)
 539                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 540                 else {
 541                         return_refs = fastarray;
 542                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 543                 }
 544         }
 545         else
 546                 return_refs = NULL;
 547
 548         match_pos = 0;
 549         for (ref = *refs; ref; ref = next) {
 550                 next = ref->next;
 551                 if (!memcmp(ref->name, "refs/", 5) &&
 552                     check_refname_format(ref->name + 5, 0))
 553                         ; /* trash */
 554                 else if (args.fetch_all &&
 555                          (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
 556                         *newtail = ref;
 557                         ref->next = NULL;
 558                         newtail = &ref->next;
 559                         continue;
 560                 }
 561                 else {
 562                         int cmp = -1;
 563                         while (match_pos < nr_match) {
 564                                 cmp = strcmp(ref->name, match[match_pos]);
 565                                 if (cmp < 0) /* definitely do not have it */
 566                                         break;
 567                                 else if (cmp == 0) { /* definitely have it */
 568                                         match[match_pos][0] = '\0';
 569                                         return_refs[match_pos] = ref;
 570                                         break;
 571                                 }
 572                                 else /* might have it; keep looking */
 573                                         match_pos++;
 574                         }
 575                         if (!cmp)
 576                                 continue; /* we will link it later */
 577                 }
 578                 free(ref);
 579         }
 580
 581         if (!args.fetch_all) {
 582                 int i;
 583                 for (i = 0; i < nr_match; i++) {
 584                         ref = return_refs[i];
 585                         if (ref) {
 586                                 *newtail = ref;
 587                                 ref->next = NULL;
 588                                 newtail = &ref->next;
 589                         }
 590                 }
 591                 if (return_refs != fastarray)
 592                         free(return_refs);
 593         }
 594         *refs = newlist;
 595 }
 596
 597 static void mark_alternate_complete(const struct ref *ref, void *unused)
 598 {
 599         mark_complete(NULL, ref->old_sha1, 0, NULL);
 600 }
 601
 602 static int everything_local(struct ref **refs, int nr_match, char **match)
 603 {
 604         struct ref *ref;
 605         int retval;
 606         unsigned long cutoff = 0;
 607
 608         save_commit_buffer = 0;
 609
 610         for (ref = *refs; ref; ref = ref->next) {
 611                 struct object *o;
 612
 613                 if (!has_sha1_file(ref->old_sha1))
 614                         continue;
 615
 616                 o = parse_object(ref->old_sha1);
 617                 if (!o)
 618                         continue;
 619
 620                 /* We already have it -- which may mean that we were
 621                  * in sync with the other side at some time after
 622                  * that (it is OK if we guess wrong here).
 623                  */
 624                 if (o->type == OBJ_COMMIT) {
 625                         struct commit *commit = (struct commit *)o;
 626                         if (!cutoff || cutoff < commit->date)
 627                                 cutoff = commit->date;
 628                 }
 629         }
 630
 631         if (!args.depth) {
 632                 for_each_ref(mark_complete, NULL);
 633                 for_each_alternate_ref(mark_alternate_complete, NULL);
 634                 if (cutoff)
 635                         mark_recent_complete_commits(cutoff);
 636         }
 637
 638         /*
 639          * Mark all complete remote refs as common refs.
 640          * Don't mark them common yet; the server has to be told so first.
 641          */
 642         for (ref = *refs; ref; ref = ref->next) {
 643                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 644                                              NULL, 0);
 645
 646                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 647                         continue;
 648
 649                 if (!(o->flags & SEEN)) {
 650                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 651
 652                         mark_common((struct commit *)o, 1, 1);
 653                 }
 654         }
 655
 656         filter_refs(refs, nr_match, match);
 657
 658         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 659                 const unsigned char *remote = ref->old_sha1;
 660                 unsigned char local[20];
 661                 struct object *o;
 662
 663                 o = lookup_object(remote);
 664                 if (!o || !(o->flags & COMPLETE)) {
 665                         retval = 0;
 666                         if (!args.verbose)
 667                                 continue;
 668                         fprintf(stderr,
 669                                 "want %s (%s)\n", sha1_to_hex(remote),
 670                                 ref->name);
 671                         continue;
 672                 }
 673
 674                 hashcpy(ref->new_sha1, local);
 675                 if (!args.verbose)
 676                         continue;
 677                 fprintf(stderr,
 678                         "already have %s (%s)\n", sha1_to_hex(remote),
 679                         ref->name);
 680         }
 681         return retval;
 682 }
 683
 684 static int sideband_demux(int in, int out, void *data)
 685 {
 686         int *xd = data;
 687
 688         int ret = recv_sideband("fetch-pack", xd[0], out);
 689         close(out);
 690         return ret;
 691 }
 692
 693 static int get_pack(int xd[2], char **pack_lockfile)
 694 {
 695         struct async demux;
 696         const char *argv[20];
 697         char keep_arg[256];
 698         char hdr_arg[256];
 699         const char **av;
 700         int do_keep = args.keep_pack;
 701         struct child_process cmd;
 702
 703         memset(&demux, 0, sizeof(demux));
 704         if (use_sideband) {
 705                 /* xd[] is talking with upload-pack; subprocess reads from
 706                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 707                  * through demux->out.
 708                  */
 709                 demux.proc = sideband_demux;
 710                 demux.data = xd;
 711                 demux.out = -1;
 712                 if (start_async(&demux))
 713                         die("fetch-pack: unable to fork off sideband"
 714                             " demultiplexer");
 715         }
 716         else
 717                 demux.out = xd[0];
 718
 719         memset(&cmd, 0, sizeof(cmd));
 720         cmd.argv = argv;
 721         av = argv;
 722         *hdr_arg = 0;
 723         if (!args.keep_pack && unpack_limit) {
 724                 struct pack_header header;
 725
 726                 if (read_pack_header(demux.out, &header))
 727                         die("protocol error: bad pack header");
 728                 snprintf(hdr_arg, sizeof(hdr_arg),
 729                          "--pack_header=%"PRIu32",%"PRIu32,
 730                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 731                 if (ntohl(header.hdr_entries) < unpack_limit)
 732                         do_keep = 0;
 733                 else
 734                         do_keep = 1;
 735         }
 736
 737         if (do_keep) {
 738                 if (pack_lockfile)
 739                         cmd.out = -1;
 740                 *av++ = "index-pack";
 741                 *av++ = "--stdin";
 742                 if (!args.quiet && !args.no_progress)
 743                         *av++ = "-v";
 744                 if (args.use_thin_pack)
 745                         *av++ = "--fix-thin";
 746                 if (args.lock_pack || unpack_limit) {
 747                         int s = sprintf(keep_arg,
 748                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 749                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 750                                 strcpy(keep_arg + s, "localhost");
 751                         *av++ = keep_arg;
 752                 }
 753         }
 754         else {
 755                 *av++ = "unpack-objects";
 756                 if (args.quiet || args.no_progress)
 757                         *av++ = "-q";
 758         }
 759         if (*hdr_arg)
 760                 *av++ = hdr_arg;
 761         if (fetch_fsck_objects >= 0
 762             ? fetch_fsck_objects
 763             : transfer_fsck_objects >= 0
 764             ? transfer_fsck_objects
 765             : 0)
 766                 *av++ = "--strict";
 767         *av++ = NULL;
 768
 769         cmd.in = demux.out;
 770         cmd.git_cmd = 1;
 771         if (start_command(&cmd))
 772                 die("fetch-pack: unable to fork off %s", argv[0]);
 773         if (do_keep && pack_lockfile) {
 774                 *pack_lockfile = index_pack_lockfile(cmd.out);
 775                 close(cmd.out);
 776         }
 777
 778         if (finish_command(&cmd))
 779                 die("%s failed", argv[0]);
 780         if (use_sideband && finish_async(&demux))
 781                 die("error in sideband demultiplexer");
 782         return 0;
 783 }
 784
 785 static struct ref *do_fetch_pack(int fd[2],
 786                 const struct ref *orig_ref,
 787                 int nr_match,
 788                 char **match,
 789                 char **pack_lockfile)
 790 {
 791         struct ref *ref = copy_ref_list(orig_ref);
 792         unsigned char sha1[20];
 793         const char *agent_feature;
 794         int agent_len;
 795
 796         sort_ref_list(&ref, ref_compare_name);
 797
 798         if (is_repository_shallow() && !server_supports("shallow"))
 799                 die("Server does not support shallow clients");
 800         if (server_supports("multi_ack_detailed")) {
 801                 if (args.verbose)
 802                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 803                 multi_ack = 2;
 804                 if (server_supports("no-done")) {
 805                         if (args.verbose)
 806                                 fprintf(stderr, "Server supports no-done\n");
 807                         if (args.stateless_rpc)
 808                                 no_done = 1;
 809                 }
 810         }
 811         else if (server_supports("multi_ack")) {
 812                 if (args.verbose)
 813                         fprintf(stderr, "Server supports multi_ack\n");
 814                 multi_ack = 1;
 815         }
 816         if (server_supports("side-band-64k")) {
 817                 if (args.verbose)
 818                         fprintf(stderr, "Server supports side-band-64k\n");
 819                 use_sideband = 2;
 820         }
 821         else if (server_supports("side-band")) {
 822                 if (args.verbose)
 823                         fprintf(stderr, "Server supports side-band\n");
 824                 use_sideband = 1;
 825         }
 826         if (!server_supports("thin-pack"))
 827                 args.use_thin_pack = 0;
 828         if (!server_supports("no-progress"))
 829                 args.no_progress = 0;
 830         if (!server_supports("include-tag"))
 831                 args.include_tag = 0;
 832         if (server_supports("ofs-delta")) {
 833                 if (args.verbose)
 834                         fprintf(stderr, "Server supports ofs-delta\n");
 835         } else
 836                 prefer_ofs_delta = 0;
 837
 838         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 839                 agent_supported = 1;
 840                 if (args.verbose && agent_len)
 841                         fprintf(stderr, "Server version is %.*s\n",
 842                                 agent_len, agent_feature);
 843         }
 844
 845         if (everything_local(&ref, nr_match, match)) {
 846                 packet_flush(fd[1]);
 847                 goto all_done;
 848         }
 849         if (find_common(fd, sha1, ref) < 0)
 850                 if (!args.keep_pack)
 851                         /* When cloning, it is not unusual to have
 852                          * no common commit.
 853                          */
 854                         warning("no common commits");
 855
 856         if (args.stateless_rpc)
 857                 packet_flush(fd[1]);
 858         if (get_pack(fd, pack_lockfile))
 859                 die("git fetch-pack: fetch failed.");
 860
 861  all_done:
 862         return ref;
 863 }
 864
 865 static int remove_duplicates(int nr_heads, char **heads)
 866 {
 867         int src, dst;
 868
 869         if (!nr_heads)
 870                 return 0;
 871
 872         for (src = dst = 1; src < nr_heads; src++)
 873                 if (strcmp(heads[src], heads[dst-1]))
 874                         heads[dst++] = heads[src];
 875         return dst;
 876 }
 877
 878 static int fetch_pack_config(const char *var, const char *value, void *cb)
 879 {
 880         if (strcmp(var, "fetch.unpacklimit") == 0) {
 881                 fetch_unpack_limit = git_config_int(var, value);
 882                 return 0;
 883         }
 884
 885         if (strcmp(var, "transfer.unpacklimit") == 0) {
 886                 transfer_unpack_limit = git_config_int(var, value);
 887                 return 0;
 888         }
 889
 890         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 891                 prefer_ofs_delta = git_config_bool(var, value);
 892                 return 0;
 893         }
 894
 895         if (!strcmp(var, "fetch.fsckobjects")) {
 896                 fetch_fsck_objects = git_config_bool(var, value);
 897                 return 0;
 898         }
 899
 900         if (!strcmp(var, "transfer.fsckobjects")) {
 901                 transfer_fsck_objects = git_config_bool(var, value);
 902                 return 0;
 903         }
 904
 905         return git_default_config(var, value, cb);
 906 }
 907
 908 static struct lock_file lock;
 909
 910 static void fetch_pack_setup(void)
 911 {
 912         static int did_setup;
 913         if (did_setup)
 914                 return;
 915         git_config(fetch_pack_config, NULL);
 916         if (0 <= transfer_unpack_limit)
 917                 unpack_limit = transfer_unpack_limit;
 918         else if (0 <= fetch_unpack_limit)
 919                 unpack_limit = fetch_unpack_limit;
 920         did_setup = 1;
 921 }
 922
 923 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 924 {
 925         int i, ret;
 926         struct ref *ref = NULL;
 927         const char *dest = NULL;
 928         int alloc_heads = 0, nr_heads = 0;
 929         char **heads = NULL;
 930         int fd[2];
 931         char *pack_lockfile = NULL;
 932         char **pack_lockfile_ptr = NULL;
 933         struct child_process *conn;
 934
 935         packet_trace_identity("fetch-pack");
 936
 937         for (i = 1; i < argc && *argv[i] == '-'; i++) {
 938                 const char *arg = argv[i];
 939
 940                 if (!prefixcmp(arg, "--upload-pack=")) {
 941                         args.uploadpack = arg + 14;
 942                         continue;
 943                 }
 944                 if (!prefixcmp(arg, "--exec=")) {
 945                         args.uploadpack = arg + 7;
 946                         continue;
 947                 }
 948                 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 949                         args.quiet = 1;
 950                         continue;
 951                 }
 952                 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 953                         args.lock_pack = args.keep_pack;
 954                         args.keep_pack = 1;
 955                         continue;
 956                 }
 957                 if (!strcmp("--thin", arg)) {
 958                         args.use_thin_pack = 1;
 959                         continue;
 960                 }
 961                 if (!strcmp("--include-tag", arg)) {
 962                         args.include_tag = 1;
 963                         continue;
 964                 }
 965                 if (!strcmp("--all", arg)) {
 966                         args.fetch_all = 1;
 967                         continue;
 968                 }
 969                 if (!strcmp("--stdin", arg)) {
 970                         args.stdin_refs = 1;
 971                         continue;
 972                 }
 973                 if (!strcmp("-v", arg)) {
 974                         args.verbose = 1;
 975                         continue;
 976                 }
 977                 if (!prefixcmp(arg, "--depth=")) {
 978                         args.depth = strtol(arg + 8, NULL, 0);
 979                         continue;
 980                 }
 981                 if (!strcmp("--no-progress", arg)) {
 982                         args.no_progress = 1;
 983                         continue;
 984                 }
 985                 if (!strcmp("--stateless-rpc", arg)) {
 986                         args.stateless_rpc = 1;
 987                         continue;
 988                 }
 989                 if (!strcmp("--lock-pack", arg)) {
 990                         args.lock_pack = 1;
 991                         pack_lockfile_ptr = &pack_lockfile;
 992                         continue;
 993                 }
 994                 usage(fetch_pack_usage);
 995         }
 996
 997         if (i < argc)
 998                 dest = argv[i++];
 999         else
1000                 usage(fetch_pack_usage);
1001
1002         /*
1003          * Copy refs from cmdline to growable list, then append any
1004          * refs from the standard input:
1005          */
1006         ALLOC_GROW(heads, argc - i, alloc_heads);
1007         for (; i < argc; i++)
1008                 heads[nr_heads++] = xstrdup(argv[i]);
1009         if (args.stdin_refs) {
1010                 if (args.stateless_rpc) {
1011                         /* in stateless RPC mode we use pkt-line to read
1012                          * from stdin, until we get a flush packet
1013                          */
1014                         static char line[1000];
1015                         for (;;) {
1016                                 int n = packet_read_line(0, line, sizeof(line));
1017                                 if (!n)
1018                                         break;
1019                                 if (line[n-1] == '\n')
1020                                         n--;
1021                                 ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
1022                                 heads[nr_heads++] = xmemdupz(line, n);
1023                         }
1024                 }
1025                 else {
1026                         /* read from stdin one ref per line, until EOF */
1027                         struct strbuf line = STRBUF_INIT;
1028                         while (strbuf_getline(&line, stdin, '\n') != EOF) {
1029                                 ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
1030                                 heads[nr_heads++] = strbuf_detach(&line, NULL);
1031                         }
1032                         strbuf_release(&line);
1033                 }
1034         }
1035
1036         if (args.stateless_rpc) {
1037                 conn = NULL;
1038                 fd[0] = 0;
1039                 fd[1] = 1;
1040         } else {
1041                 conn = git_connect(fd, dest, args.uploadpack,
1042                                    args.verbose ? CONNECT_VERBOSE : 0);
1043         }
1044
1045         get_remote_heads(fd[0], &ref, 0, NULL);
1046
1047         ref = fetch_pack(&args, fd, conn, ref, dest,
1048                 nr_heads, heads, pack_lockfile_ptr);
1049         if (pack_lockfile) {
1050                 printf("lock %s\n", pack_lockfile);
1051                 fflush(stdout);
1052         }
1053         close(fd[0]);
1054         close(fd[1]);
1055         if (finish_connect(conn))
1056                 ref = NULL;
1057         ret = !ref;
1058
1059         if (!ret && nr_heads) {
1060                 /* If the heads to pull were given, we should have
1061                  * consumed all of them by matching the remote.
1062                  * Otherwise, 'git fetch remote no-such-ref' would
1063                  * silently succeed without issuing an error.
1064                  */
1065                 for (i = 0; i < nr_heads; i++)
1066                         if (heads[i] && heads[i][0]) {
1067                                 error("no such remote ref %s", heads[i]);
1068                                 ret = 1;
1069                         }
1070         }
1071         while (ref) {
1072                 printf("%s %s\n",
1073                        sha1_to_hex(ref->old_sha1), ref->name);
1074                 ref = ref->next;
1075         }
1076
1077         return ret;
1078 }
1079
1080 static int compare_heads(const void *a, const void *b)
1081 {
1082         return strcmp(*(const char **)a, *(const char **)b);
1083 }
1084
1085 struct ref *fetch_pack(struct fetch_pack_args *my_args,
1086                        int fd[], struct child_process *conn,
1087                        const struct ref *ref,
1088                 const char *dest,
1089                 int nr_heads,
1090                 char **heads,
1091                 char **pack_lockfile)
1092 {
1093         struct stat st;
1094         struct ref *ref_cpy;
1095
1096         fetch_pack_setup();
1097         if (&args != my_args)
1098                 memcpy(&args, my_args, sizeof(args));
1099         if (args.depth > 0) {
1100                 if (stat(git_path("shallow"), &st))
1101                         st.st_mtime = 0;
1102         }
1103
1104         if (heads && nr_heads) {
1105                 qsort(heads, nr_heads, sizeof(*heads), compare_heads);
1106                 nr_heads = remove_duplicates(nr_heads, heads);
1107         }
1108
1109         if (!ref) {
1110                 packet_flush(fd[1]);
1111                 die("no matching remote head");
1112         }
1113         ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
1114
1115         if (args.depth > 0) {
1116                 struct cache_time mtime;
1117                 struct strbuf sb = STRBUF_INIT;
1118                 char *shallow = git_path("shallow");
1119                 int fd;
1120
1121                 mtime.sec = st.st_mtime;
1122                 mtime.nsec = ST_MTIME_NSEC(st);
1123                 if (stat(shallow, &st)) {
1124                         if (mtime.sec)
1125                                 die("shallow file was removed during fetch");
1126                 } else if (st.st_mtime != mtime.sec
1127 #ifdef USE_NSEC
1128                                 || ST_MTIME_NSEC(st) != mtime.nsec
1129 #endif
1130                           )
1131                         die("shallow file was changed during fetch");
1132
1133                 fd = hold_lock_file_for_update(&lock, shallow,
1134                                                LOCK_DIE_ON_ERROR);
1135                 if (!write_shallow_commits(&sb, 0)
1136                  || write_in_full(fd, sb.buf, sb.len) != sb.len) {
1137                         unlink_or_warn(shallow);
1138                         rollback_lock_file(&lock);
1139                 } else {
1140                         commit_lock_file(&lock);
1141                 }
1142                 strbuf_release(&sb);
1143         }
1144
1145         reprepare_packed_git();
1146         return ref_cpy;
1147 }