bundle-uri.c

   1 #include "cache.h"
   2 #include "bundle-uri.h"
   3 #include "bundle.h"
   4 #include "object-store.h"
   5 #include "refs.h"
   6 #include "run-command.h"
   7 #include "hashmap.h"
   8 #include "pkt-line.h"
   9 #include "config.h"
  10 #include "remote.h"
  11
  12 static int compare_bundles(const void *hashmap_cmp_fn_data,
  13                            const struct hashmap_entry *he1,
  14                            const struct hashmap_entry *he2,
  15                            const void *id)
  16 {
  17         const struct remote_bundle_info *e1 =
  18                 container_of(he1, const struct remote_bundle_info, ent);
  19         const struct remote_bundle_info *e2 =
  20                 container_of(he2, const struct remote_bundle_info, ent);
  21
  22         return strcmp(e1->id, id ? (const char *)id : e2->id);
  23 }
  24
  25 void init_bundle_list(struct bundle_list *list)
  26 {
  27         memset(list, 0, sizeof(*list));
  28
  29         /* Implied defaults. */
  30         list->mode = BUNDLE_MODE_ALL;
  31         list->version = 1;
  32
  33         hashmap_init(&list->bundles, compare_bundles, NULL, 0);
  34 }
  35
  36 static int clear_remote_bundle_info(struct remote_bundle_info *bundle,
  37                                     void *data)
  38 {
  39         FREE_AND_NULL(bundle->id);
  40         FREE_AND_NULL(bundle->uri);
  41         FREE_AND_NULL(bundle->file);
  42         bundle->unbundled = 0;
  43         return 0;
  44 }
  45
  46 void clear_bundle_list(struct bundle_list *list)
  47 {
  48         if (!list)
  49                 return;
  50
  51         for_all_bundles_in_list(list, clear_remote_bundle_info, NULL);
  52         hashmap_clear_and_free(&list->bundles, struct remote_bundle_info, ent);
  53         free(list->baseURI);
  54 }
  55
  56 int for_all_bundles_in_list(struct bundle_list *list,
  57                             bundle_iterator iter,
  58                             void *data)
  59 {
  60         struct remote_bundle_info *info;
  61         struct hashmap_iter i;
  62
  63         hashmap_for_each_entry(&list->bundles, &i, info, ent) {
  64                 int result = iter(info, data);
  65
  66                 if (result)
  67                         return result;
  68         }
  69
  70         return 0;
  71 }
  72
  73 static int summarize_bundle(struct remote_bundle_info *info, void *data)
  74 {
  75         FILE *fp = data;
  76         fprintf(fp, "[bundle \"%s\"]\n", info->id);
  77         fprintf(fp, "\turi = %s\n", info->uri);
  78         return 0;
  79 }
  80
  81 void print_bundle_list(FILE *fp, struct bundle_list *list)
  82 {
  83         const char *mode;
  84
  85         switch (list->mode) {
  86         case BUNDLE_MODE_ALL:
  87                 mode = "all";
  88                 break;
  89
  90         case BUNDLE_MODE_ANY:
  91                 mode = "any";
  92                 break;
  93
  94         case BUNDLE_MODE_NONE:
  95         default:
  96                 mode = "<unknown>";
  97         }
  98
  99         fprintf(fp, "[bundle]\n");
 100         fprintf(fp, "\tversion = %d\n", list->version);
 101         fprintf(fp, "\tmode = %s\n", mode);
 102
 103         for_all_bundles_in_list(list, summarize_bundle, fp);
 104 }
 105
 106 /**
 107  * Given a key-value pair, update the state of the given bundle list.
 108  * Returns 0 if the key-value pair is understood. Returns -1 if the key
 109  * is not understood or the value is malformed.
 110  */
 111 static int bundle_list_update(const char *key, const char *value,
 112                               struct bundle_list *list)
 113 {
 114         struct strbuf id = STRBUF_INIT;
 115         struct remote_bundle_info lookup = REMOTE_BUNDLE_INFO_INIT;
 116         struct remote_bundle_info *bundle;
 117         const char *subsection, *subkey;
 118         size_t subsection_len;
 119
 120         if (parse_config_key(key, "bundle", &subsection, &subsection_len, &subkey))
 121                 return -1;
 122
 123         if (!subsection_len) {
 124                 if (!strcmp(subkey, "version")) {
 125                         int version;
 126                         if (!git_parse_int(value, &version))
 127                                 return -1;
 128                         if (version != 1)
 129                                 return -1;
 130
 131                         list->version = version;
 132                         return 0;
 133                 }
 134
 135                 if (!strcmp(subkey, "mode")) {
 136                         if (!strcmp(value, "all"))
 137                                 list->mode = BUNDLE_MODE_ALL;
 138                         else if (!strcmp(value, "any"))
 139                                 list->mode = BUNDLE_MODE_ANY;
 140                         else
 141                                 return -1;
 142                         return 0;
 143                 }
 144
 145                 /* Ignore other unknown global keys. */
 146                 return 0;
 147         }
 148
 149         strbuf_add(&id, subsection, subsection_len);
 150
 151         /*
 152          * Check for an existing bundle with this <id>, or create one
 153          * if necessary.
 154          */
 155         lookup.id = id.buf;
 156         hashmap_entry_init(&lookup.ent, strhash(lookup.id));
 157         if (!(bundle = hashmap_get_entry(&list->bundles, &lookup, ent, NULL))) {
 158                 CALLOC_ARRAY(bundle, 1);
 159                 bundle->id = strbuf_detach(&id, NULL);
 160                 hashmap_entry_init(&bundle->ent, strhash(bundle->id));
 161                 hashmap_add(&list->bundles, &bundle->ent);
 162         }
 163         strbuf_release(&id);
 164
 165         if (!strcmp(subkey, "uri")) {
 166                 if (bundle->uri)
 167                         return -1;
 168                 bundle->uri = relative_url(list->baseURI, value, NULL);
 169                 return 0;
 170         }
 171
 172         /*
 173          * At this point, we ignore any information that we don't
 174          * understand, assuming it to be hints for a heuristic the client
 175          * does not currently understand.
 176          */
 177         return 0;
 178 }
 179
 180 static int config_to_bundle_list(const char *key, const char *value, void *data)
 181 {
 182         struct bundle_list *list = data;
 183         return bundle_list_update(key, value, list);
 184 }
 185
 186 int bundle_uri_parse_config_format(const char *uri,
 187                                    const char *filename,
 188                                    struct bundle_list *list)
 189 {
 190         int result;
 191         struct config_options opts = {
 192                 .error_action = CONFIG_ERROR_ERROR,
 193         };
 194
 195         if (!list->baseURI) {
 196                 struct strbuf baseURI = STRBUF_INIT;
 197                 strbuf_addstr(&baseURI, uri);
 198
 199                 /*
 200                  * If the URI does not end with a trailing slash, then
 201                  * remove the filename portion of the path. This is
 202                  * important for relative URIs.
 203                  */
 204                 strbuf_strip_file_from_path(&baseURI);
 205                 list->baseURI = strbuf_detach(&baseURI, NULL);
 206         }
 207         result = git_config_from_file_with_options(config_to_bundle_list,
 208                                                    filename, list,
 209                                                    &opts);
 210
 211         if (!result && list->mode == BUNDLE_MODE_NONE) {
 212                 warning(_("bundle list at '%s' has no mode"), uri);
 213                 result = 1;
 214         }
 215
 216         return result;
 217 }
 218
 219 static char *find_temp_filename(void)
 220 {
 221         int fd;
 222         struct strbuf name = STRBUF_INIT;
 223         /*
 224          * Find a temporary filename that is available. This is briefly
 225          * racy, but unlikely to collide.
 226          */
 227         fd = odb_mkstemp(&name, "bundles/tmp_uri_XXXXXX");
 228         if (fd < 0) {
 229                 warning(_("failed to create temporary file"));
 230                 return NULL;
 231         }
 232
 233         close(fd);
 234         unlink(name.buf);
 235         return strbuf_detach(&name, NULL);
 236 }
 237
 238 static int download_https_uri_to_file(const char *file, const char *uri)
 239 {
 240         int result = 0;
 241         struct child_process cp = CHILD_PROCESS_INIT;
 242         FILE *child_in = NULL, *child_out = NULL;
 243         struct strbuf line = STRBUF_INIT;
 244         int found_get = 0;
 245
 246         strvec_pushl(&cp.args, "git-remote-https", uri, NULL);
 247         cp.err = -1;
 248         cp.in = -1;
 249         cp.out = -1;
 250
 251         if (start_command(&cp))
 252                 return 1;
 253
 254         child_in = fdopen(cp.in, "w");
 255         if (!child_in) {
 256                 result = 1;
 257                 goto cleanup;
 258         }
 259
 260         child_out = fdopen(cp.out, "r");
 261         if (!child_out) {
 262                 result = 1;
 263                 goto cleanup;
 264         }
 265
 266         fprintf(child_in, "capabilities\n");
 267         fflush(child_in);
 268
 269         while (!strbuf_getline(&line, child_out)) {
 270                 if (!line.len)
 271                         break;
 272                 if (!strcmp(line.buf, "get"))
 273                         found_get = 1;
 274         }
 275         strbuf_release(&line);
 276
 277         if (!found_get) {
 278                 result = error(_("insufficient capabilities"));
 279                 goto cleanup;
 280         }
 281
 282         fprintf(child_in, "get %s %s\n\n", uri, file);
 283
 284 cleanup:
 285         if (child_in)
 286                 fclose(child_in);
 287         if (finish_command(&cp))
 288                 return 1;
 289         if (child_out)
 290                 fclose(child_out);
 291         return result;
 292 }
 293
 294 static int copy_uri_to_file(const char *filename, const char *uri)
 295 {
 296         const char *out;
 297
 298         if (starts_with(uri, "https:") ||
 299             starts_with(uri, "http:"))
 300                 return download_https_uri_to_file(filename, uri);
 301
 302         if (skip_prefix(uri, "file://", &out))
 303                 uri = out;
 304
 305         /* Copy as a file */
 306         return copy_file(filename, uri, 0);
 307 }
 308
 309 static int unbundle_from_file(struct repository *r, const char *file)
 310 {
 311         int result = 0;
 312         int bundle_fd;
 313         struct bundle_header header = BUNDLE_HEADER_INIT;
 314         struct string_list_item *refname;
 315         struct strbuf bundle_ref = STRBUF_INIT;
 316         size_t bundle_prefix_len;
 317
 318         if ((bundle_fd = read_bundle_header(file, &header)) < 0)
 319                 return 1;
 320
 321         /*
 322          * Skip the reachability walk here, since we will be adding
 323          * a reachable ref pointing to the new tips, which will reach
 324          * the prerequisite commits.
 325          */
 326         if ((result = unbundle(r, &header, bundle_fd, NULL,
 327                                VERIFY_BUNDLE_QUIET)))
 328                 return 1;
 329
 330         /*
 331          * Convert all refs/heads/ from the bundle into refs/bundles/
 332          * in the local repository.
 333          */
 334         strbuf_addstr(&bundle_ref, "refs/bundles/");
 335         bundle_prefix_len = bundle_ref.len;
 336
 337         for_each_string_list_item(refname, &header.references) {
 338                 struct object_id *oid = refname->util;
 339                 struct object_id old_oid;
 340                 const char *branch_name;
 341                 int has_old;
 342
 343                 if (!skip_prefix(refname->string, "refs/heads/", &branch_name))
 344                         continue;
 345
 346                 strbuf_setlen(&bundle_ref, bundle_prefix_len);
 347                 strbuf_addstr(&bundle_ref, branch_name);
 348
 349                 has_old = !read_ref(bundle_ref.buf, &old_oid);
 350                 update_ref("fetched bundle", bundle_ref.buf, oid,
 351                            has_old ? &old_oid : NULL,
 352                            REF_SKIP_OID_VERIFICATION,
 353                            UPDATE_REFS_MSG_ON_ERR);
 354         }
 355
 356         bundle_header_release(&header);
 357         return result;
 358 }
 359
 360 struct bundle_list_context {
 361         struct repository *r;
 362         struct bundle_list *list;
 363         enum bundle_list_mode mode;
 364         int count;
 365         int depth;
 366 };
 367
 368 /*
 369  * This early definition is necessary because we use indirect recursion:
 370  *
 371  * While iterating through a bundle list that was downloaded as part
 372  * of fetch_bundle_uri_internal(), iterator methods eventually call it
 373  * again, but with depth + 1.
 374  */
 375 static int fetch_bundle_uri_internal(struct repository *r,
 376                                      struct remote_bundle_info *bundle,
 377                                      int depth,
 378                                      struct bundle_list *list);
 379
 380 static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data)
 381 {
 382         int res;
 383         struct bundle_list_context *ctx = data;
 384
 385         if (ctx->mode == BUNDLE_MODE_ANY && ctx->count)
 386                 return 0;
 387
 388         res = fetch_bundle_uri_internal(ctx->r, bundle, ctx->depth + 1, ctx->list);
 389
 390         /*
 391          * Only increment count if the download succeeded. If our mode is
 392          * BUNDLE_MODE_ANY, then we will want to try other URIs in the
 393          * list in case they work instead.
 394          */
 395         if (!res)
 396                 ctx->count++;
 397
 398         /*
 399          * To be opportunistic as possible, we continue iterating and
 400          * download as many bundles as we can, so we can apply the ones
 401          * that work, even in BUNDLE_MODE_ALL mode.
 402          */
 403         return 0;
 404 }
 405
 406 static int download_bundle_list(struct repository *r,
 407                                 struct bundle_list *local_list,
 408                                 struct bundle_list *global_list,
 409                                 int depth)
 410 {
 411         struct bundle_list_context ctx = {
 412                 .r = r,
 413                 .list = global_list,
 414                 .depth = depth + 1,
 415                 .mode = local_list->mode,
 416         };
 417
 418         return for_all_bundles_in_list(local_list, download_bundle_to_file, &ctx);
 419 }
 420
 421 static int fetch_bundle_list_in_config_format(struct repository *r,
 422                                               struct bundle_list *global_list,
 423                                               struct remote_bundle_info *bundle,
 424                                               int depth)
 425 {
 426         int result;
 427         struct bundle_list list_from_bundle;
 428
 429         init_bundle_list(&list_from_bundle);
 430
 431         if ((result = bundle_uri_parse_config_format(bundle->uri,
 432                                                      bundle->file,
 433                                                      &list_from_bundle)))
 434                 goto cleanup;
 435
 436         if (list_from_bundle.mode == BUNDLE_MODE_NONE) {
 437                 warning(_("unrecognized bundle mode from URI '%s'"),
 438                         bundle->uri);
 439                 result = -1;
 440                 goto cleanup;
 441         }
 442
 443         if ((result = download_bundle_list(r, &list_from_bundle,
 444                                            global_list, depth)))
 445                 goto cleanup;
 446
 447 cleanup:
 448         clear_bundle_list(&list_from_bundle);
 449         return result;
 450 }
 451
 452 /**
 453  * This limits the recursion on fetch_bundle_uri_internal() when following
 454  * bundle lists.
 455  */
 456 static int max_bundle_uri_depth = 4;
 457
 458 /**
 459  * Recursively download all bundles advertised at the given URI
 460  * to files. If the file is a bundle, then add it to the given
 461  * 'list'. Otherwise, expect a bundle list and recurse on the
 462  * URIs in that list according to the list mode (ANY or ALL).
 463  */
 464 static int fetch_bundle_uri_internal(struct repository *r,
 465                                      struct remote_bundle_info *bundle,
 466                                      int depth,
 467                                      struct bundle_list *list)
 468 {
 469         int result = 0;
 470         struct remote_bundle_info *bcopy;
 471
 472         if (depth >= max_bundle_uri_depth) {
 473                 warning(_("exceeded bundle URI recursion limit (%d)"),
 474                         max_bundle_uri_depth);
 475                 return -1;
 476         }
 477
 478         if (!bundle->file &&
 479             !(bundle->file = find_temp_filename())) {
 480                 result = -1;
 481                 goto cleanup;
 482         }
 483
 484         if ((result = copy_uri_to_file(bundle->file, bundle->uri))) {
 485                 warning(_("failed to download bundle from URI '%s'"), bundle->uri);
 486                 goto cleanup;
 487         }
 488
 489         if ((result = !is_bundle(bundle->file, 1))) {
 490                 result = fetch_bundle_list_in_config_format(
 491                                 r, list, bundle, depth);
 492                 if (result)
 493                         warning(_("file at URI '%s' is not a bundle or bundle list"),
 494                                 bundle->uri);
 495                 goto cleanup;
 496         }
 497
 498         /* Copy the bundle and insert it into the global list. */
 499         CALLOC_ARRAY(bcopy, 1);
 500         bcopy->id = xstrdup(bundle->id);
 501         bcopy->file = xstrdup(bundle->file);
 502         hashmap_entry_init(&bcopy->ent, strhash(bcopy->id));
 503         hashmap_add(&list->bundles, &bcopy->ent);
 504
 505 cleanup:
 506         if (result && bundle->file)
 507                 unlink(bundle->file);
 508         return result;
 509 }
 510
 511 /**
 512  * This loop iterator breaks the loop with nonzero return code on the
 513  * first successful unbundling of a bundle.
 514  */
 515 static int attempt_unbundle(struct remote_bundle_info *info, void *data)
 516 {
 517         struct repository *r = data;
 518
 519         if (!info->file || info->unbundled)
 520                 return 0;
 521
 522         if (!unbundle_from_file(r, info->file)) {
 523                 info->unbundled = 1;
 524                 return 1;
 525         }
 526
 527         return 0;
 528 }
 529
 530 static int unbundle_all_bundles(struct repository *r,
 531                                 struct bundle_list *list)
 532 {
 533         /*
 534          * Iterate through all bundles looking for ones that can
 535          * successfully unbundle. If any succeed, then perhaps another
 536          * will succeed in the next attempt.
 537          *
 538          * Keep in mind that a non-zero result for the loop here means
 539          * the loop terminated early on a successful unbundling, which
 540          * signals that we can try again.
 541          */
 542         while (for_all_bundles_in_list(list, attempt_unbundle, r)) ;
 543
 544         return 0;
 545 }
 546
 547 static int unlink_bundle(struct remote_bundle_info *info, void *data)
 548 {
 549         if (info->file)
 550                 unlink_or_warn(info->file);
 551         return 0;
 552 }
 553
 554 int fetch_bundle_uri(struct repository *r, const char *uri)
 555 {
 556         int result;
 557         struct bundle_list list;
 558         struct remote_bundle_info bundle = {
 559                 .uri = xstrdup(uri),
 560                 .id = xstrdup(""),
 561         };
 562
 563         init_bundle_list(&list);
 564
 565         /* If a bundle is added to this global list, then it is required. */
 566         list.mode = BUNDLE_MODE_ALL;
 567
 568         if ((result = fetch_bundle_uri_internal(r, &bundle, 0, &list)))
 569                 goto cleanup;
 570
 571         result = unbundle_all_bundles(r, &list);
 572
 573 cleanup:
 574         for_all_bundles_in_list(&list, unlink_bundle, NULL);
 575         clear_bundle_list(&list);
 576         clear_remote_bundle_info(&bundle, NULL);
 577         return result;
 578 }
 579
 580 int fetch_bundle_list(struct repository *r, struct bundle_list *list)
 581 {
 582         int result;
 583         struct bundle_list global_list;
 584
 585         init_bundle_list(&global_list);
 586
 587         /* If a bundle is added to this global list, then it is required. */
 588         global_list.mode = BUNDLE_MODE_ALL;
 589
 590         if ((result = download_bundle_list(r, list, &global_list, 0)))
 591                 goto cleanup;
 592
 593         result = unbundle_all_bundles(r, &global_list);
 594
 595 cleanup:
 596         for_all_bundles_in_list(&global_list, unlink_bundle, NULL);
 597         clear_bundle_list(&global_list);
 598         return result;
 599 }
 600
 601 /**
 602  * API for serve.c.
 603  */
 604
 605 int bundle_uri_advertise(struct repository *r, struct strbuf *value UNUSED)
 606 {
 607         static int advertise_bundle_uri = -1;
 608
 609         if (advertise_bundle_uri != -1)
 610                 goto cached;
 611
 612         advertise_bundle_uri = 0;
 613         repo_config_get_maybe_bool(r, "uploadpack.advertisebundleuris", &advertise_bundle_uri);
 614
 615 cached:
 616         return advertise_bundle_uri;
 617 }
 618
 619 static int config_to_packet_line(const char *key, const char *value, void *data)
 620 {
 621         struct packet_reader *writer = data;
 622
 623         if (starts_with(key, "bundle."))
 624                 packet_write_fmt(writer->fd, "%s=%s", key, value);
 625
 626         return 0;
 627 }
 628
 629 int bundle_uri_command(struct repository *r,
 630                        struct packet_reader *request)
 631 {
 632         struct packet_writer writer;
 633         packet_writer_init(&writer, 1);
 634
 635         while (packet_reader_read(request) == PACKET_READ_NORMAL)
 636                 die(_("bundle-uri: unexpected argument: '%s'"), request->line);
 637         if (request->status != PACKET_READ_FLUSH)
 638                 die(_("bundle-uri: expected flush after arguments"));
 639
 640         /*
 641          * Read all "bundle.*" config lines to the client as key=value
 642          * packet lines.
 643          */
 644         git_config(config_to_packet_line, &writer);
 645
 646         packet_writer_flush(&writer);
 647
 648         return 0;
 649 }
 650
 651 /**
 652  * General API for {transport,connect}.c etc.
 653  */
 654 int bundle_uri_parse_line(struct bundle_list *list, const char *line)
 655 {
 656         int result;
 657         const char *equals;
 658         struct strbuf key = STRBUF_INIT;
 659
 660         if (!strlen(line))
 661                 return error(_("bundle-uri: got an empty line"));
 662
 663         equals = strchr(line, '=');
 664
 665         if (!equals)
 666                 return error(_("bundle-uri: line is not of the form 'key=value'"));
 667         if (line == equals || !*(equals + 1))
 668                 return error(_("bundle-uri: line has empty key or value"));
 669
 670         strbuf_add(&key, line, equals - line);
 671         result = bundle_list_update(key.buf, equals + 1, list);
 672         strbuf_release(&key);
 673
 674         return result;
 675 }