libexec/got-fetch-pack/got-fetch-pack.c

   1 /*
   2  * Copyright (c) 2019 Ori Bernstein <ori@openbsd.org>
   3  *
   4  * Permission to use, copy, modify, and distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 #include "got_compat.h"
  18
  19 #include <sys/types.h>
  20 #include <sys/queue.h>
  21 #include <sys/uio.h>
  22 #include <sys/time.h>
  23 #include <sys/stat.h>
  24
  25 #include <stdint.h>
  26 #include <errno.h>
  27 #include <limits.h>
  28 #include <poll.h>
  29 #include <signal.h>
  30 #include <stdio.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33 #include <ctype.h>
  34 #include <fcntl.h>
  35 #include <unistd.h>
  36 #include <zlib.h>
  37 #include <err.h>
  38
  39 #include "got_error.h"
  40 #include "got_object.h"
  41 #include "got_path.h"
  42 #include "got_version.h"
  43 #include "got_fetch.h"
  44 #include "got_reference.h"
  45
  46 #include "got_lib_hash.h"
  47 #include "got_lib_delta.h"
  48 #include "got_lib_object.h"
  49 #include "got_lib_object_parse.h"
  50 #include "got_lib_privsep.h"
  51 #include "got_lib_pack.h"
  52 #include "got_lib_pkt.h"
  53 #include "got_lib_poll.h"
  54 #include "got_lib_gitproto.h"
  55 #include "got_lib_ratelimit.h"
  56
  57 #ifndef MIN
  58 #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
  59 #endif
  60
  61 #ifndef nitems
  62 #define nitems(_a)      (sizeof((_a)) / sizeof((_a)[0]))
  63 #endif
  64
  65 struct got_object *indexed;
  66 static int chattygot;
  67
  68 static const struct got_capability got_capabilities[] = {
  69         { GOT_CAPA_AGENT, "got/" GOT_VERSION_STR },
  70         { GOT_CAPA_OFS_DELTA, NULL },
  71         { GOT_CAPA_SIDE_BAND_64K, NULL },
  72 };
  73
  74 static void
  75 match_remote_ref(struct got_pathlist_head *have_refs,
  76     struct got_object_id *my_id, const char *refname)
  77 {
  78         struct got_pathlist_entry *pe;
  79
  80         /* XXX zero-hash signifies we don't have this ref;
  81          * we should use a flag instead */
  82         memset(my_id, 0, sizeof(*my_id));
  83
  84         TAILQ_FOREACH(pe, have_refs, entry) {
  85                 struct got_object_id *id = pe->data;
  86                 if (strcmp(pe->path, refname) == 0) {
  87                         memcpy(my_id, id, sizeof(*my_id));
  88                         break;
  89                 }
  90         }
  91 }
  92
  93 static int
  94 match_branch(const char *branch, const char *wanted_branch)
  95 {
  96         if (strncmp(branch, "refs/heads/", 11) != 0)
  97                 return 0;
  98
  99         if (strncmp(wanted_branch, "refs/heads/", 11) == 0)
 100                 wanted_branch += 11;
 101
 102         return (strcmp(branch + 11, wanted_branch) == 0);
 103 }
 104
 105 static int
 106 match_wanted_ref(const char *refname, const char *wanted_ref)
 107 {
 108         if (strncmp(refname, "refs/", 5) != 0)
 109                 return 0;
 110         refname += 5;
 111
 112         /*
 113          * Prevent fetching of references that won't make any
 114          * sense outside of the remote repository's context.
 115          */
 116         if (strncmp(refname, "got/", 4) == 0)
 117                 return 0;
 118         if (strncmp(refname, "remotes/", 8) == 0)
 119                 return 0;
 120
 121         if (strncmp(wanted_ref, "refs/", 5) == 0)
 122                 wanted_ref += 5;
 123
 124         /* Allow prefix match. */
 125         if (got_path_is_child(refname, wanted_ref, strlen(wanted_ref)))
 126                 return 1;
 127
 128         /* Allow exact match. */
 129         return (strcmp(refname, wanted_ref) == 0);
 130 }
 131
 132 static const struct got_error *
 133 send_fetch_server_progress(struct imsgbuf *ibuf, const char *msg, size_t msglen)
 134 {
 135         if (msglen > MAX_IMSGSIZE - IMSG_HEADER_SIZE)
 136                 return got_error(GOT_ERR_NO_SPACE);
 137
 138         if (msglen == 0)
 139                 return NULL;
 140
 141         if (imsg_compose(ibuf, GOT_IMSG_FETCH_SERVER_PROGRESS, 0, 0, -1,
 142             msg, msglen) == -1)
 143                 return got_error_from_errno(
 144                     "imsg_compose FETCH_SERVER_PROGRESS");
 145
 146         return got_privsep_flush_imsg(ibuf);
 147 }
 148
 149 static const struct got_error *
 150 send_fetch_download_progress(struct imsgbuf *ibuf, off_t bytes,
 151     struct got_ratelimit *rl)
 152 {
 153         const struct got_error *err;
 154         int elapsed = 0;
 155
 156         if (rl) {
 157                 err = got_ratelimit_check(&elapsed, rl);
 158                 if (err || !elapsed)
 159                         return err;
 160         }
 161
 162         if (imsg_compose(ibuf, GOT_IMSG_FETCH_DOWNLOAD_PROGRESS, 0, 0, -1,
 163             &bytes, sizeof(bytes)) == -1)
 164                 return got_error_from_errno(
 165                     "imsg_compose FETCH_DOWNLOAD_PROGRESS");
 166
 167         return got_privsep_flush_imsg(ibuf);
 168 }
 169
 170 static const struct got_error *
 171 send_fetch_done(struct imsgbuf *ibuf, uint8_t *pack_sha1)
 172 {
 173         if (imsg_compose(ibuf, GOT_IMSG_FETCH_DONE, 0, 0, -1,
 174             pack_sha1, SHA1_DIGEST_LENGTH) == -1)
 175                 return got_error_from_errno("imsg_compose FETCH");
 176         return got_privsep_flush_imsg(ibuf);
 177 }
 178
 179 static const struct got_error *
 180 fetch_progress(struct imsgbuf *ibuf, const char *buf, size_t len)
 181 {
 182         size_t i;
 183
 184         if (len == 0)
 185                 return NULL;
 186
 187         /*
 188          * Truncate messages which exceed the maximum imsg payload size.
 189          * Server may send up to 64k.
 190          */
 191         if (len > MAX_IMSGSIZE - IMSG_HEADER_SIZE)
 192                 len = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
 193
 194         /* Only allow printable ASCII. */
 195         for (i = 0; i < len; i++) {
 196                 if (isprint((unsigned char)buf[i]) ||
 197                     isspace((unsigned char)buf[i]))
 198                         continue;
 199                 return got_error_msg(GOT_ERR_BAD_PACKET,
 200                     "non-printable progress message received from server");
 201         }
 202
 203         return send_fetch_server_progress(ibuf, buf, len);
 204 }
 205
 206 static const struct got_error *
 207 fetch_error(const char *buf, size_t len)
 208 {
 209         static char msg[1024];
 210         size_t i;
 211
 212         for (i = 0; i < len && i < sizeof(msg) - 1; i++) {
 213                 if (!isprint((unsigned char)buf[i]))
 214                         return got_error_msg(GOT_ERR_BAD_PACKET,
 215                             "non-printable error message received from server");
 216                 msg[i] = buf[i];
 217         }
 218         msg[i] = '\0';
 219         return got_error_msg(GOT_ERR_FETCH_FAILED, msg);
 220 }
 221
 222 static const struct got_error *
 223 send_fetch_symrefs(struct imsgbuf *ibuf, struct got_pathlist_head *symrefs)
 224 {
 225         struct ibuf *wbuf;
 226         size_t len, nsymrefs = 0;
 227         struct got_pathlist_entry *pe;
 228
 229         len = sizeof(struct got_imsg_fetch_symrefs);
 230         TAILQ_FOREACH(pe, symrefs, entry) {
 231                 const char *target = pe->data;
 232                 len += sizeof(struct got_imsg_fetch_symref) +
 233                     pe->path_len + strlen(target);
 234                 nsymrefs++;
 235         }
 236
 237         if (len >= MAX_IMSGSIZE - IMSG_HEADER_SIZE)
 238                 return got_error(GOT_ERR_NO_SPACE);
 239
 240         wbuf = imsg_create(ibuf, GOT_IMSG_FETCH_SYMREFS, 0, 0, len);
 241         if (wbuf == NULL)
 242                 return got_error_from_errno("imsg_create FETCH_SYMREFS");
 243
 244         /* Keep in sync with struct got_imsg_fetch_symrefs definition! */
 245         if (imsg_add(wbuf, &nsymrefs, sizeof(nsymrefs)) == -1)
 246                 return got_error_from_errno("imsg_add FETCH_SYMREFS");
 247
 248         TAILQ_FOREACH(pe, symrefs, entry) {
 249                 const char *name = pe->path;
 250                 size_t name_len = pe->path_len;
 251                 const char *target = pe->data;
 252                 size_t target_len = strlen(target);
 253
 254                 /* Keep in sync with struct got_imsg_fetch_symref definition! */
 255                 if (imsg_add(wbuf, &name_len, sizeof(name_len)) == -1)
 256                         return got_error_from_errno("imsg_add FETCH_SYMREFS");
 257                 if (imsg_add(wbuf, &target_len, sizeof(target_len)) == -1)
 258                         return got_error_from_errno("imsg_add FETCH_SYMREFS");
 259                 if (imsg_add(wbuf, name, name_len) == -1)
 260                         return got_error_from_errno("imsg_add FETCH_SYMREFS");
 261                 if (imsg_add(wbuf, target, target_len) == -1)
 262                         return got_error_from_errno("imsg_add FETCH_SYMREFS");
 263         }
 264
 265         imsg_close(ibuf, wbuf);
 266         return got_privsep_flush_imsg(ibuf);
 267 }
 268
 269 static const struct got_error *
 270 send_fetch_ref(struct imsgbuf *ibuf, struct got_object_id *refid,
 271     const char *refname)
 272 {
 273         struct ibuf *wbuf;
 274         size_t len, reflen = strlen(refname);
 275
 276         len = sizeof(struct got_imsg_fetch_ref) + reflen;
 277         if (len >= MAX_IMSGSIZE - IMSG_HEADER_SIZE)
 278                 return got_error(GOT_ERR_NO_SPACE);
 279
 280         wbuf = imsg_create(ibuf, GOT_IMSG_FETCH_REF, 0, 0, len);
 281         if (wbuf == NULL)
 282                 return got_error_from_errno("imsg_create FETCH_REF");
 283
 284         /* Keep in sync with struct got_imsg_fetch_ref definition! */
 285         if (imsg_add(wbuf, refid, sizeof(*refid)) == -1)
 286                 return got_error_from_errno("imsg_add FETCH_REF");
 287         if (imsg_add(wbuf, refname, reflen) == -1)
 288                 return got_error_from_errno("imsg_add FETCH_REF");
 289
 290         imsg_close(ibuf, wbuf);
 291         return got_privsep_flush_imsg(ibuf);
 292 }
 293
 294 static const struct got_error *
 295 fetch_ref(struct imsgbuf *ibuf, struct got_pathlist_head *have_refs,
 296     struct got_object_id *have, struct got_object_id *want,
 297     const char *refname, const char *id_str)
 298 {
 299         const struct got_error *err;
 300         char *theirs = NULL, *mine = NULL;
 301
 302         if (!got_parse_object_id(want, id_str, GOT_HASH_SHA1)) {
 303                 err = got_error(GOT_ERR_BAD_OBJ_ID_STR);
 304                 goto done;
 305         }
 306
 307         match_remote_ref(have_refs, have, refname);
 308         err = send_fetch_ref(ibuf, want, refname);
 309         if (err)
 310                 goto done;
 311
 312         if (chattygot)
 313                 fprintf(stderr, "%s: %s will be fetched\n",
 314                     getprogname(), refname);
 315         if (chattygot > 1) {
 316                 err = got_object_id_str(&theirs, want);
 317                 if (err)
 318                         goto done;
 319                 err = got_object_id_str(&mine, have);
 320                 if (err)
 321                         goto done;
 322                 fprintf(stderr, "%s: remote: %s\n%s: local:  %s\n",
 323                     getprogname(), theirs, getprogname(), mine);
 324         }
 325 done:
 326         free(theirs);
 327         free(mine);
 328         return err;
 329 }
 330
 331 static const struct got_error *
 332 fetch_pack(int fd, int packfd, uint8_t *pack_sha1,
 333     struct got_pathlist_head *have_refs, int fetch_all_branches,
 334     struct got_pathlist_head *wanted_branches,
 335     struct got_pathlist_head *wanted_refs, int list_refs_only,
 336     const char *worktree_branch, const char *remote_head,
 337     int no_head, struct imsgbuf *ibuf)
 338 {
 339         const struct got_error *err = NULL;
 340         char buf[GOT_PKT_MAX];
 341         char hashstr[SHA1_DIGEST_STRING_LENGTH];
 342         struct got_object_id *have, *want;
 343         int is_firstpkt = 1, nref = 0, refsz = 16;
 344         int i, n, nwant = 0, nhave = 0, acked = 0, eof = 0;
 345         off_t packsz = 0, last_reported_packsz = 0;
 346         char *id_str = NULL, *default_id_str = NULL, *refname = NULL;
 347         char *server_capabilities = NULL, *my_capabilities = NULL;
 348         const char *default_branch = NULL;
 349         struct got_pathlist_head symrefs;
 350         struct got_pathlist_entry *pe;
 351         int sent_my_capabilites = 0, have_sidebands = 0;
 352         int found_branch = 0;
 353         struct got_hash ctx;
 354         uint8_t sha1_buf[SHA1_DIGEST_LENGTH];
 355         size_t sha1_buf_len = 0;
 356         ssize_t w;
 357         struct got_ratelimit rl;
 358
 359         TAILQ_INIT(&symrefs);
 360         got_hash_init(&ctx, GOT_HASH_SHA1);
 361         got_ratelimit_init(&rl, 0, 500);
 362
 363         have = malloc(refsz * sizeof(have[0]));
 364         if (have == NULL)
 365                 return got_error_from_errno("malloc");
 366         want = malloc(refsz * sizeof(want[0]));
 367         if (want == NULL) {
 368                 err = got_error_from_errno("malloc");
 369                 goto done;
 370         }
 371         while (1) {
 372                 err = got_pkt_readpkt(&n, fd, buf, sizeof(buf), chattygot,
 373                     INFTIM);
 374                 if (err)
 375                         goto done;
 376                 if (n == 0)
 377                         break;
 378                 if (n >= 4 && strncmp(buf, "ERR ", 4) == 0) {
 379                         err = fetch_error(&buf[4], n - 4);
 380                         goto done;
 381                 }
 382                 free(id_str);
 383                 free(refname);
 384                 err = got_gitproto_parse_refline(&id_str, &refname,
 385                     &server_capabilities, buf, n);
 386                 if (err)
 387                         goto done;
 388
 389                 if (refsz == nref + 1) {
 390                         struct got_object_id *h, *w;
 391
 392                         refsz *= 2;
 393                         h = reallocarray(have, refsz, sizeof(have[0]));
 394                         if (h == NULL) {
 395                                 err = got_error_from_errno("reallocarray");
 396                                 goto done;
 397                         }
 398                         have = h;
 399                         w = reallocarray(want, refsz, sizeof(want[0]));
 400                         if (w == NULL) {
 401                                 err = got_error_from_errno("reallocarray");
 402                                 goto done;
 403                         }
 404                         want = w;
 405                 }
 406
 407                 if (is_firstpkt) {
 408                         if (chattygot && server_capabilities[0] != '\0')
 409                                 fprintf(stderr, "%s: server capabilities: %s\n",
 410                                     getprogname(), server_capabilities);
 411                         err = got_gitproto_match_capabilities(&my_capabilities,
 412                             &symrefs, server_capabilities,
 413                             got_capabilities, nitems(got_capabilities));
 414                         if (err)
 415                                 goto done;
 416                         if (chattygot)
 417                                 fprintf(stderr, "%s: my capabilities:%s\n",
 418                                     getprogname(), my_capabilities != NULL ?
 419                                     my_capabilities : "");
 420                         err = send_fetch_symrefs(ibuf, &symrefs);
 421                         if (err)
 422                                 goto done;
 423                         is_firstpkt = 0;
 424                         if (!fetch_all_branches) {
 425                                 TAILQ_FOREACH(pe, &symrefs, entry) {
 426                                         const char *name = pe->path;
 427                                         const char *symref_target = pe->data;
 428                                         if (strcmp(name, GOT_REF_HEAD) != 0)
 429                                                 continue;
 430                                         default_branch = symref_target;
 431                                         break;
 432                                 }
 433                         }
 434                         if (default_branch)
 435                                 continue;
 436                 }
 437                 if (strstr(refname, "^{}")) {
 438                         if (chattygot) {
 439                                 fprintf(stderr, "%s: ignoring %s\n",
 440                                     getprogname(), refname);
 441                         }
 442                         continue;
 443                 }
 444                 if (default_branch && default_id_str == NULL &&
 445                     strcmp(refname, default_branch) == 0) {
 446                         default_id_str = strdup(id_str);
 447                         if (default_id_str == NULL) {
 448                                 err = got_error_from_errno("strdup");
 449                                 goto done;
 450                         }
 451                 }
 452
 453                 if (list_refs_only || strncmp(refname, "refs/tags/", 10) == 0) {
 454                         err = fetch_ref(ibuf, have_refs, &have[nref],
 455                             &want[nref], refname, id_str);
 456                         if (err)
 457                                 goto done;
 458                         nref++;
 459                 } else if (strncmp(refname, "refs/heads/", 11) == 0) {
 460                         if (fetch_all_branches) {
 461                                 err = fetch_ref(ibuf, have_refs, &have[nref],
 462                                     &want[nref], refname, id_str);
 463                                 if (err)
 464                                         goto done;
 465                                 nref++;
 466                                 found_branch = 1;
 467                                 continue;
 468                         }
 469                         TAILQ_FOREACH(pe, wanted_branches, entry) {
 470                                 if (match_branch(refname, pe->path))
 471                                         break;
 472                         }
 473                         if (pe != NULL || (worktree_branch != NULL &&
 474                             match_branch(refname, worktree_branch))) {
 475                                 err = fetch_ref(ibuf, have_refs, &have[nref],
 476                                     &want[nref], refname, id_str);
 477                                 if (err)
 478                                         goto done;
 479                                 nref++;
 480                                 found_branch = 1;
 481                         } else if (chattygot) {
 482                                 fprintf(stderr, "%s: ignoring %s\n",
 483                                     getprogname(), refname);
 484                         }
 485                 } else {
 486                         TAILQ_FOREACH(pe, wanted_refs, entry) {
 487                                 if (match_wanted_ref(refname, pe->path))
 488                                         break;
 489                         }
 490                         if (pe != NULL) {
 491                                 err = fetch_ref(ibuf, have_refs, &have[nref],
 492                                     &want[nref], refname, id_str);
 493                                 if (err)
 494                                         goto done;
 495                                 nref++;
 496                         } else if (chattygot) {
 497                                 fprintf(stderr, "%s: ignoring %s\n",
 498                                     getprogname(), refname);
 499                         }
 500                 }
 501         }
 502
 503         if (list_refs_only)
 504                 goto done;
 505
 506         /*
 507          * If -b was not used and either none of the requested branches
 508          * (got.conf, worktree) were found or the client already has the
 509          * remote HEAD symref but its target changed, fetch remote's HEAD.
 510          */
 511         if (!no_head && default_branch && default_id_str &&
 512             strncmp(default_branch, "refs/heads/", 11) == 0) {
 513                 int remote_head_changed = 0;
 514
 515                 if (remote_head) {
 516                         if (strcmp(remote_head, default_branch + 11) != 0)
 517                                 remote_head_changed = 1;
 518                 }
 519
 520                 if (!found_branch || remote_head_changed) {
 521                         err = fetch_ref(ibuf, have_refs, &have[nref],
 522                             &want[nref], default_branch, default_id_str);
 523                         if (err)
 524                                 goto done;
 525                         nref++;
 526                 }
 527         }
 528
 529         /* Abort if we haven't found anything to fetch. */
 530         if (nref == 0) {
 531                 struct got_pathlist_entry *pe;
 532                 static char msg[PATH_MAX + 33];
 533
 534                 pe = TAILQ_FIRST(wanted_branches);
 535                 if (pe) {
 536                         snprintf(msg, sizeof(msg),
 537                             "branch \"%s\" not found on server", pe->path);
 538                         err = got_error_msg(GOT_ERR_FETCH_NO_BRANCH, msg);
 539                         goto done;
 540                 }
 541
 542                 pe = TAILQ_FIRST(wanted_refs);
 543                 if (pe) {
 544                         snprintf(msg, sizeof(msg),
 545                             "reference \"%s\" not found on server", pe->path);
 546                         err = got_error_msg(GOT_ERR_FETCH_NO_BRANCH, msg);
 547                         goto done;
 548                 }
 549
 550                 err = got_error(GOT_ERR_FETCH_NO_BRANCH);
 551                 goto done;
 552         }
 553
 554         for (i = 0; i < nref; i++) {
 555                 if (got_object_id_cmp(&have[i], &want[i]) == 0)
 556                         continue;
 557                 got_object_id_hex(&want[i], hashstr, sizeof(hashstr));
 558                 n = snprintf(buf, sizeof(buf), "want %s%s\n", hashstr,
 559                     sent_my_capabilites || my_capabilities == NULL ?
 560                     "" : my_capabilities);
 561                 if (n < 0 || (size_t)n >= sizeof(buf)) {
 562                         err = got_error(GOT_ERR_NO_SPACE);
 563                         goto done;
 564                 }
 565                 err = got_pkt_writepkt(fd, buf, n, chattygot);
 566                 if (err)
 567                         goto done;
 568                 sent_my_capabilites = 1;
 569                 nwant++;
 570         }
 571         err = got_pkt_flushpkt(fd, chattygot);
 572         if (err)
 573                 goto done;
 574
 575         if (nwant == 0)
 576                 goto done;
 577
 578         TAILQ_FOREACH(pe, have_refs, entry) {
 579                 struct got_object_id *id = pe->data;
 580                 got_object_id_hex(id, hashstr, sizeof(hashstr));
 581                 n = snprintf(buf, sizeof(buf), "have %s\n", hashstr);
 582                 if (n < 0 || (size_t)n >= sizeof(buf)) {
 583                         err = got_error(GOT_ERR_NO_SPACE);
 584                         goto done;
 585                 }
 586                 err = got_pkt_writepkt(fd, buf, n, chattygot);
 587                 if (err)
 588                         goto done;
 589                 nhave++;
 590         }
 591
 592         n = strlcpy(buf, "done\n", sizeof(buf));
 593         err = got_pkt_writepkt(fd, buf, n, chattygot);
 594         if (err)
 595                 goto done;
 596
 597         while (nhave > 0 && !acked) {
 598                 struct got_object_id common_id;
 599
 600                 /* The server should ACK the object IDs we need. */
 601                 err = got_pkt_readpkt(&n, fd, buf, sizeof(buf), chattygot,
 602                     INFTIM);
 603                 if (err)
 604                         goto done;
 605                 if (n >= 4 && strncmp(buf, "ERR ", 4) == 0) {
 606                         err = fetch_error(&buf[4], n - 4);
 607                         goto done;
 608                 }
 609                 if (n >= 4 && strncmp(buf, "NAK\n", 4) == 0) {
 610                         /*
 611                          * Server could not find a common ancestor.
 612                          * Perhaps it is an out-of-date mirror, or there
 613                          * is a repository with unrelated history.
 614                          */
 615                         break;
 616                 }
 617                 if (n < 4 + SHA1_DIGEST_STRING_LENGTH ||
 618                     strncmp(buf, "ACK ", 4) != 0) {
 619                         err = got_error_msg(GOT_ERR_BAD_PACKET,
 620                             "unexpected message from server");
 621                         goto done;
 622                 }
 623                 if (!got_parse_object_id(&common_id, buf + 4,
 624                     GOT_HASH_SHA1)) {
 625                         err = got_error_msg(GOT_ERR_BAD_PACKET,
 626                             "bad object ID in ACK packet from server");
 627                         goto done;
 628                 }
 629                 acked++;
 630         }
 631
 632         if (nhave == 0) {
 633                 err = got_pkt_readpkt(&n, fd, buf, sizeof(buf), chattygot,
 634                     INFTIM);
 635                 if (err)
 636                         goto done;
 637                 if (n != 4 || strncmp(buf, "NAK\n", n) != 0) {
 638                         err = got_error_msg(GOT_ERR_BAD_PACKET,
 639                             "unexpected message from server");
 640                         goto done;
 641                 }
 642         }
 643
 644         if (chattygot)
 645                 fprintf(stderr, "%s: fetching...\n", getprogname());
 646
 647         if (my_capabilities != NULL &&
 648             strstr(my_capabilities, GOT_CAPA_SIDE_BAND_64K) != NULL)
 649                 have_sidebands = 1;
 650
 651         while (!eof) {
 652                 ssize_t r = 0;
 653                 int datalen = -1;
 654
 655                 if (have_sidebands) {
 656                         err = got_pkt_readhdr(&datalen, fd, chattygot, INFTIM);
 657                         if (err)
 658                                 goto done;
 659                         if (datalen <= 0)
 660                                 break;
 661
 662                         /* Read sideband channel ID (one byte). */
 663                         r = read(fd, buf, 1);
 664                         if (r == -1) {
 665                                 err = got_error_from_errno("read");
 666                                 goto done;
 667                         }
 668                         if (r != 1) {
 669                                 err = got_error_msg(GOT_ERR_BAD_PACKET,
 670                                     "short packet");
 671                                 goto done;
 672                         }
 673                         if (datalen > sizeof(buf) - 5) {
 674                                 err = got_error_msg(GOT_ERR_BAD_PACKET,
 675                                     "bad packet length");
 676                                 goto done;
 677                         }
 678                         datalen--; /* sideband ID has been read */
 679                         if (buf[0] == GOT_SIDEBAND_PACKFILE_DATA) {
 680                                 /* Read packfile data. */
 681                                 err = got_pkt_readn(&r, fd, buf, datalen,
 682                                     INFTIM);
 683                                 if (err)
 684                                         goto done;
 685                                 if (r != datalen) {
 686                                         err = got_error_msg(GOT_ERR_BAD_PACKET,
 687                                             "packet too short");
 688                                         goto done;
 689                                 }
 690                         } else if (buf[0] == GOT_SIDEBAND_PROGRESS_INFO) {
 691                                 err = got_pkt_readn(&r, fd, buf, datalen,
 692                                     INFTIM);
 693                                 if (err)
 694                                         goto done;
 695                                 if (r != datalen) {
 696                                         err = got_error_msg(GOT_ERR_BAD_PACKET,
 697                                             "packet too short");
 698                                         goto done;
 699                                 }
 700                                 err = fetch_progress(ibuf, buf, r);
 701                                 if (err)
 702                                         goto done;
 703                                 continue;
 704                         } else if (buf[0] == GOT_SIDEBAND_ERROR_INFO) {
 705                                 err = got_pkt_readn(&r, fd, buf, datalen,
 706                                     INFTIM);
 707                                 if (err)
 708                                         goto done;
 709                                 if (r != datalen) {
 710                                         err = got_error_msg(GOT_ERR_BAD_PACKET,
 711                                             "packet too short");
 712                                         goto done;
 713                                 }
 714                                 err = fetch_error(buf, r);
 715                                 goto done;
 716                         } else if (buf[0] == 'A') {
 717                                 err = got_pkt_readn(&r, fd, buf, datalen,
 718                                     INFTIM);
 719                                 if (err)
 720                                         goto done;
 721                                 if (r != datalen) {
 722                                         err = got_error_msg(GOT_ERR_BAD_PACKET,
 723                                             "packet too short");
 724                                         goto done;
 725                                 }
 726                                 /*
 727                                  * Git server responds with ACK after 'done'
 728                                  * even though multi_ack is disabled?!?
 729                                  */
 730                                 buf[r] = '\0';
 731                                 if (strncmp(buf, "CK ", 3) == 0)
 732                                         continue; /* ignore */
 733                                 err = got_error_msg(GOT_ERR_BAD_PACKET,
 734                                     "unexpected message from server");
 735                                 goto done;
 736                         } else {
 737                                 err = got_error_msg(GOT_ERR_BAD_PACKET,
 738                                     "unknown side-band received from server");
 739                                 goto done;
 740                         }
 741                 } else {
 742                         /* No sideband channel. Every byte is packfile data. */
 743                         size_t n = 0;
 744                         err = got_poll_read_full_timeout(fd, &n,
 745                             buf, sizeof buf, 1, INFTIM);
 746                         if (err) {
 747                                 if (err->code != GOT_ERR_EOF)
 748                                         goto done;
 749                                 r = 0;
 750                                 eof = 1;
 751                         } else
 752                                 r = n;
 753                 }
 754
 755                 /*
 756                  * An expected SHA1 checksum sits at the end of the pack file.
 757                  * Since we don't know the file size ahead of time we have to
 758                  * keep SHA1_DIGEST_LENGTH bytes buffered and avoid mixing
 759                  * those bytes into our SHA1 checksum computation until we
 760                  * know for sure that additional pack file data bytes follow.
 761                  */
 762                 if (r < SHA1_DIGEST_LENGTH) {
 763                         if (sha1_buf_len < SHA1_DIGEST_LENGTH) {
 764                                 /*
 765                                  * If there's enough buffered + read data to
 766                                  * fill up the buffer then shift a sufficient
 767                                  * amount of bytes out at the front to make
 768                                  * room, mixing those bytes into the checksum.
 769                                  */
 770                                 if (sha1_buf_len > 0 &&
 771                                     sha1_buf_len + r > SHA1_DIGEST_LENGTH) {
 772                                         size_t nshift = MIN(sha1_buf_len + r -
 773                                             SHA1_DIGEST_LENGTH, sha1_buf_len);
 774                                         got_hash_update(&ctx, sha1_buf,
 775                                             nshift);
 776                                         memmove(sha1_buf, sha1_buf + nshift,
 777                                             sha1_buf_len - nshift);
 778                                         sha1_buf_len -= nshift;
 779                                 }
 780
 781                                 /* Buffer potential checksum bytes. */
 782                                 memcpy(sha1_buf + sha1_buf_len, buf, r);
 783                                 sha1_buf_len += r;
 784                         } else if (r > 0) {
 785                                 /*
 786                                  * Mix in previously buffered bytes which
 787                                  * are not part of the checksum after all.
 788                                  */
 789                                 got_hash_update(&ctx, sha1_buf, r);
 790
 791                                 /* Update potential checksum buffer. */
 792                                 memmove(sha1_buf, sha1_buf + r,
 793                                     sha1_buf_len - r);
 794                                 memcpy(sha1_buf + sha1_buf_len - r, buf, r);
 795                         }
 796                 } else {
 797                         /* Mix in any previously buffered bytes. */
 798                         got_hash_update(&ctx, sha1_buf, sha1_buf_len);
 799
 800                         /* Mix in bytes read minus potential checksum bytes. */
 801                         got_hash_update(&ctx, buf, r - SHA1_DIGEST_LENGTH);
 802
 803                         /* Buffer potential checksum bytes. */
 804                         memcpy(sha1_buf, buf + r - SHA1_DIGEST_LENGTH,
 805                             SHA1_DIGEST_LENGTH);
 806                         sha1_buf_len = SHA1_DIGEST_LENGTH;
 807                 }
 808
 809                 /* Write packfile data to temporary pack file. */
 810                 w = write(packfd, buf, r);
 811                 if (w == -1) {
 812                         err = got_error_from_errno("write");
 813                         goto done;
 814                 }
 815                 if (w != r) {
 816                         err = got_error(GOT_ERR_IO);
 817                         goto done;
 818                 }
 819                 packsz += w;
 820
 821                 /* Don't send too many progress privsep messages. */
 822                 if (packsz > last_reported_packsz + 1024) {
 823                         err = send_fetch_download_progress(ibuf, packsz, &rl);
 824                         if (err)
 825                                 goto done;
 826                         last_reported_packsz = packsz;
 827                 }
 828         }
 829         err = send_fetch_download_progress(ibuf, packsz, NULL);
 830         if (err)
 831                 goto done;
 832
 833         got_hash_final(&ctx, pack_sha1);
 834         if (sha1_buf_len != SHA1_DIGEST_LENGTH ||
 835             memcmp(pack_sha1, sha1_buf, sha1_buf_len) != 0) {
 836                 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 837                     "pack file checksum mismatch");
 838         }
 839 done:
 840         got_pathlist_free(&symrefs, GOT_PATHLIST_FREE_ALL);
 841         free(have);
 842         free(want);
 843         free(id_str);
 844         free(default_id_str);
 845         free(refname);
 846         free(server_capabilities);
 847         return err;
 848 }
 849
 850
 851 int
 852 main(int argc, char **argv)
 853 {
 854         const struct got_error *err = NULL;
 855         int fetchfd = -1, packfd = -1;
 856         uint8_t pack_sha1[SHA1_DIGEST_LENGTH];
 857         struct imsgbuf ibuf;
 858         struct imsg imsg;
 859         struct got_pathlist_head have_refs;
 860         struct got_pathlist_head wanted_branches;
 861         struct got_pathlist_head wanted_refs;
 862         struct got_imsg_fetch_request fetch_req;
 863         struct got_imsg_fetch_have_ref href;
 864         struct got_imsg_fetch_wanted_branch wbranch;
 865         struct got_imsg_fetch_wanted_ref wref;
 866         size_t datalen, i;
 867         char *remote_head = NULL, *worktree_branch = NULL;
 868 #if 0
 869         static int attached;
 870         while (!attached)
 871                 sleep (1);
 872 #endif
 873
 874         TAILQ_INIT(&have_refs);
 875         TAILQ_INIT(&wanted_branches);
 876         TAILQ_INIT(&wanted_refs);
 877
 878         imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
 879 #ifndef PROFILE
 880         /* revoke access to most system calls */
 881         if (pledge("stdio recvfd", NULL) == -1) {
 882                 err = got_error_from_errno("pledge");
 883                 got_privsep_send_error(&ibuf, err);
 884                 return 1;
 885         }
 886
 887         /* revoke fs access */
 888         if (landlock_no_fs() == -1) {
 889                 err = got_error_from_errno("landlock_no_fs");
 890                 got_privsep_send_error(&ibuf, err);
 891                 return 1;
 892         }
 893         if (cap_enter() == -1) {
 894                 err = got_error_from_errno("cap_enter");
 895                 got_privsep_send_error(&ibuf, err);
 896                 return 1;
 897         }
 898 #endif
 899         err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
 900         if (err) {
 901                 if (err->code == GOT_ERR_PRIVSEP_PIPE)
 902                         err = NULL;
 903                 goto done;
 904         }
 905         if (imsg.hdr.type == GOT_IMSG_STOP)
 906                 goto done;
 907         if (imsg.hdr.type != GOT_IMSG_FETCH_REQUEST) {
 908                 err = got_error(GOT_ERR_PRIVSEP_MSG);
 909                 goto done;
 910         }
 911         datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
 912         if (datalen < sizeof(fetch_req)) {
 913                 err = got_error(GOT_ERR_PRIVSEP_LEN);
 914                 goto done;
 915         }
 916         memcpy(&fetch_req, imsg.data, sizeof(fetch_req));
 917         fetchfd = imsg_get_fd(&imsg);
 918
 919         if (datalen != sizeof(fetch_req) +
 920             fetch_req.worktree_branch_len + fetch_req.remote_head_len) {
 921                 err = got_error(GOT_ERR_PRIVSEP_LEN);
 922                 goto done;
 923         }
 924
 925         if (fetch_req.worktree_branch_len != 0) {
 926                 worktree_branch = strndup(imsg.data +
 927                     sizeof(fetch_req), fetch_req.worktree_branch_len);
 928                 if (worktree_branch == NULL) {
 929                         err = got_error_from_errno("strndup");
 930                         goto done;
 931                 }
 932         }
 933
 934         if (fetch_req.remote_head_len != 0) {
 935                 remote_head = strndup(imsg.data + sizeof(fetch_req) +
 936                     fetch_req.worktree_branch_len, fetch_req.remote_head_len);
 937                 if (remote_head == NULL) {
 938                         err = got_error_from_errno("strndup");
 939                         goto done;
 940                 }
 941         }
 942
 943         imsg_free(&imsg);
 944
 945         if (fetch_req.verbosity > 0)
 946                 chattygot += fetch_req.verbosity;
 947
 948         for (i = 0; i < fetch_req.n_have_refs; i++) {
 949                 struct got_object_id *id;
 950                 char *refname;
 951
 952                 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
 953                 if (err) {
 954                         if (err->code == GOT_ERR_PRIVSEP_PIPE)
 955                                 err = NULL;
 956                         goto done;
 957                 }
 958                 if (imsg.hdr.type == GOT_IMSG_STOP)
 959                         goto done;
 960                 if (imsg.hdr.type != GOT_IMSG_FETCH_HAVE_REF) {
 961                         err = got_error(GOT_ERR_PRIVSEP_MSG);
 962                         goto done;
 963                 }
 964                 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
 965                 if (datalen < sizeof(href)) {
 966                         err = got_error(GOT_ERR_PRIVSEP_LEN);
 967                         goto done;
 968                 }
 969                 memcpy(&href, imsg.data, sizeof(href));
 970                 if (datalen - sizeof(href) < href.name_len) {
 971                         err = got_error(GOT_ERR_PRIVSEP_LEN);
 972                         goto done;
 973                 }
 974                 refname = strndup(imsg.data + sizeof(href), href.name_len);
 975                 if (refname == NULL) {
 976                         err = got_error_from_errno("strndup");
 977                         goto done;
 978                 }
 979
 980                 id = malloc(sizeof(*id));
 981                 if (id == NULL) {
 982                         free(refname);
 983                         err = got_error_from_errno("malloc");
 984                         goto done;
 985                 }
 986                 memcpy(id, &href.id, sizeof(*id));
 987                 err = got_pathlist_append(&have_refs, refname, id);
 988                 if (err) {
 989                         free(refname);
 990                         free(id);
 991                         goto done;
 992                 }
 993
 994                 imsg_free(&imsg);
 995         }
 996
 997         for (i = 0; i < fetch_req.n_wanted_branches; i++) {
 998                 char *refname;
 999
1000                 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1001                 if (err) {
1002                         if (err->code == GOT_ERR_PRIVSEP_PIPE)
1003                                 err = NULL;
1004                         goto done;
1005                 }
1006                 if (imsg.hdr.type == GOT_IMSG_STOP)
1007                         goto done;
1008                 if (imsg.hdr.type != GOT_IMSG_FETCH_WANTED_BRANCH) {
1009                         err = got_error(GOT_ERR_PRIVSEP_MSG);
1010                         goto done;
1011                 }
1012                 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1013                 if (datalen < sizeof(wbranch)) {
1014                         err = got_error(GOT_ERR_PRIVSEP_LEN);
1015                         goto done;
1016                 }
1017                 memcpy(&wbranch, imsg.data, sizeof(wbranch));
1018                 if (datalen - sizeof(wbranch) < wbranch.name_len) {
1019                         err = got_error(GOT_ERR_PRIVSEP_LEN);
1020                         goto done;
1021                 }
1022                 refname = strndup(imsg.data + sizeof(wbranch),
1023                     wbranch.name_len);
1024                 if (refname == NULL) {
1025                         err = got_error_from_errno("strndup");
1026                         goto done;
1027                 }
1028
1029                 err = got_pathlist_append(&wanted_branches, refname, NULL);
1030                 if (err) {
1031                         free(refname);
1032                         goto done;
1033                 }
1034
1035                 imsg_free(&imsg);
1036         }
1037
1038         for (i = 0; i < fetch_req.n_wanted_refs; i++) {
1039                 char *refname;
1040
1041                 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1042                 if (err) {
1043                         if (err->code == GOT_ERR_PRIVSEP_PIPE)
1044                                 err = NULL;
1045                         goto done;
1046                 }
1047                 if (imsg.hdr.type == GOT_IMSG_STOP)
1048                         goto done;
1049                 if (imsg.hdr.type != GOT_IMSG_FETCH_WANTED_REF) {
1050                         err = got_error(GOT_ERR_PRIVSEP_MSG);
1051                         goto done;
1052                 }
1053                 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1054                 if (datalen < sizeof(wref)) {
1055                         err = got_error(GOT_ERR_PRIVSEP_LEN);
1056                         goto done;
1057                 }
1058                 memcpy(&wref, imsg.data, sizeof(wref));
1059                 if (datalen - sizeof(wref) < wref.name_len) {
1060                         err = got_error(GOT_ERR_PRIVSEP_LEN);
1061                         goto done;
1062                 }
1063                 refname = strndup(imsg.data + sizeof(wref), wref.name_len);
1064                 if (refname == NULL) {
1065                         err = got_error_from_errno("strndup");
1066                         goto done;
1067                 }
1068
1069                 err = got_pathlist_append(&wanted_refs, refname, NULL);
1070                 if (err) {
1071                         free(refname);
1072                         goto done;
1073                 }
1074
1075                 imsg_free(&imsg);
1076         }
1077
1078         err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1079         if (err) {
1080                 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1081                         err = NULL;
1082                 goto done;
1083         }
1084         if (imsg.hdr.type == GOT_IMSG_STOP)
1085                 goto done;
1086         if (imsg.hdr.type != GOT_IMSG_FETCH_OUTFD) {
1087                 err = got_error(GOT_ERR_PRIVSEP_MSG);
1088                 goto done;
1089         }
1090         if (imsg.hdr.len - IMSG_HEADER_SIZE != 0) {
1091                 err = got_error(GOT_ERR_PRIVSEP_LEN);
1092                 goto done;
1093         }
1094         packfd = imsg_get_fd(&imsg);
1095
1096         err = fetch_pack(fetchfd, packfd, pack_sha1, &have_refs,
1097             fetch_req.fetch_all_branches, &wanted_branches,
1098             &wanted_refs, fetch_req.list_refs_only,
1099             worktree_branch, remote_head, fetch_req.no_head, &ibuf);
1100 done:
1101         free(worktree_branch);
1102         free(remote_head);
1103         got_pathlist_free(&have_refs, GOT_PATHLIST_FREE_ALL);
1104         got_pathlist_free(&wanted_branches, GOT_PATHLIST_FREE_PATH);
1105         if (fetchfd != -1 && close(fetchfd) == -1 && err == NULL)
1106                 err = got_error_from_errno("close");
1107         if (packfd != -1 && close(packfd) == -1 && err == NULL)
1108                 err = got_error_from_errno("close");
1109         if (err != NULL)
1110                 got_privsep_send_error(&ibuf, err);
1111         else
1112                 err = send_fetch_done(&ibuf, pack_sha1);
1113         if (err != NULL) {
1114                 fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
1115                 got_privsep_send_error(&ibuf, err);
1116         }
1117
1118         exit(0);
1119 }