bundle-uri.c

   1 #include "cache.h"
   2 #include "bundle-uri.h"
   3 #include "bundle.h"
   4 #include "object-store.h"
   5 #include "refs.h"
   6 #include "run-command.h"
   7 #include "hashmap.h"
   8 #include "pkt-line.h"
   9 #include "config.h"
  10
  11 static int compare_bundles(const void *hashmap_cmp_fn_data,
  12                            const struct hashmap_entry *he1,
  13                            const struct hashmap_entry *he2,
  14                            const void *id)
  15 {
  16         const struct remote_bundle_info *e1 =
  17                 container_of(he1, const struct remote_bundle_info, ent);
  18         const struct remote_bundle_info *e2 =
  19                 container_of(he2, const struct remote_bundle_info, ent);
  20
  21         return strcmp(e1->id, id ? (const char *)id : e2->id);
  22 }
  23
  24 void init_bundle_list(struct bundle_list *list)
  25 {
  26         memset(list, 0, sizeof(*list));
  27
  28         /* Implied defaults. */
  29         list->mode = BUNDLE_MODE_ALL;
  30         list->version = 1;
  31
  32         hashmap_init(&list->bundles, compare_bundles, NULL, 0);
  33 }
  34
  35 static int clear_remote_bundle_info(struct remote_bundle_info *bundle,
  36                                     void *data)
  37 {
  38         FREE_AND_NULL(bundle->id);
  39         FREE_AND_NULL(bundle->uri);
  40         FREE_AND_NULL(bundle->file);
  41         bundle->unbundled = 0;
  42         return 0;
  43 }
  44
  45 void clear_bundle_list(struct bundle_list *list)
  46 {
  47         if (!list)
  48                 return;
  49
  50         for_all_bundles_in_list(list, clear_remote_bundle_info, NULL);
  51         hashmap_clear_and_free(&list->bundles, struct remote_bundle_info, ent);
  52 }
  53
  54 int for_all_bundles_in_list(struct bundle_list *list,
  55                             bundle_iterator iter,
  56                             void *data)
  57 {
  58         struct remote_bundle_info *info;
  59         struct hashmap_iter i;
  60
  61         hashmap_for_each_entry(&list->bundles, &i, info, ent) {
  62                 int result = iter(info, data);
  63
  64                 if (result)
  65                         return result;
  66         }
  67
  68         return 0;
  69 }
  70
  71 static int summarize_bundle(struct remote_bundle_info *info, void *data)
  72 {
  73         FILE *fp = data;
  74         fprintf(fp, "[bundle \"%s\"]\n", info->id);
  75         fprintf(fp, "\turi = %s\n", info->uri);
  76         return 0;
  77 }
  78
  79 void print_bundle_list(FILE *fp, struct bundle_list *list)
  80 {
  81         const char *mode;
  82
  83         switch (list->mode) {
  84         case BUNDLE_MODE_ALL:
  85                 mode = "all";
  86                 break;
  87
  88         case BUNDLE_MODE_ANY:
  89                 mode = "any";
  90                 break;
  91
  92         case BUNDLE_MODE_NONE:
  93         default:
  94                 mode = "<unknown>";
  95         }
  96
  97         fprintf(fp, "[bundle]\n");
  98         fprintf(fp, "\tversion = %d\n", list->version);
  99         fprintf(fp, "\tmode = %s\n", mode);
 100
 101         for_all_bundles_in_list(list, summarize_bundle, fp);
 102 }
 103
 104 /**
 105  * Given a key-value pair, update the state of the given bundle list.
 106  * Returns 0 if the key-value pair is understood. Returns -1 if the key
 107  * is not understood or the value is malformed.
 108  */
 109 static int bundle_list_update(const char *key, const char *value,
 110                               struct bundle_list *list)
 111 {
 112         struct strbuf id = STRBUF_INIT;
 113         struct remote_bundle_info lookup = REMOTE_BUNDLE_INFO_INIT;
 114         struct remote_bundle_info *bundle;
 115         const char *subsection, *subkey;
 116         size_t subsection_len;
 117
 118         if (parse_config_key(key, "bundle", &subsection, &subsection_len, &subkey))
 119                 return -1;
 120
 121         if (!subsection_len) {
 122                 if (!strcmp(subkey, "version")) {
 123                         int version;
 124                         if (!git_parse_int(value, &version))
 125                                 return -1;
 126                         if (version != 1)
 127                                 return -1;
 128
 129                         list->version = version;
 130                         return 0;
 131                 }
 132
 133                 if (!strcmp(subkey, "mode")) {
 134                         if (!strcmp(value, "all"))
 135                                 list->mode = BUNDLE_MODE_ALL;
 136                         else if (!strcmp(value, "any"))
 137                                 list->mode = BUNDLE_MODE_ANY;
 138                         else
 139                                 return -1;
 140                         return 0;
 141                 }
 142
 143                 /* Ignore other unknown global keys. */
 144                 return 0;
 145         }
 146
 147         strbuf_add(&id, subsection, subsection_len);
 148
 149         /*
 150          * Check for an existing bundle with this <id>, or create one
 151          * if necessary.
 152          */
 153         lookup.id = id.buf;
 154         hashmap_entry_init(&lookup.ent, strhash(lookup.id));
 155         if (!(bundle = hashmap_get_entry(&list->bundles, &lookup, ent, NULL))) {
 156                 CALLOC_ARRAY(bundle, 1);
 157                 bundle->id = strbuf_detach(&id, NULL);
 158                 hashmap_entry_init(&bundle->ent, strhash(bundle->id));
 159                 hashmap_add(&list->bundles, &bundle->ent);
 160         }
 161         strbuf_release(&id);
 162
 163         if (!strcmp(subkey, "uri")) {
 164                 if (bundle->uri)
 165                         return -1;
 166                 bundle->uri = xstrdup(value);
 167                 return 0;
 168         }
 169
 170         /*
 171          * At this point, we ignore any information that we don't
 172          * understand, assuming it to be hints for a heuristic the client
 173          * does not currently understand.
 174          */
 175         return 0;
 176 }
 177
 178 static int config_to_bundle_list(const char *key, const char *value, void *data)
 179 {
 180         struct bundle_list *list = data;
 181         return bundle_list_update(key, value, list);
 182 }
 183
 184 int bundle_uri_parse_config_format(const char *uri,
 185                                    const char *filename,
 186                                    struct bundle_list *list)
 187 {
 188         int result;
 189         struct config_options opts = {
 190                 .error_action = CONFIG_ERROR_ERROR,
 191         };
 192
 193         result = git_config_from_file_with_options(config_to_bundle_list,
 194                                                    filename, list,
 195                                                    &opts);
 196
 197         if (!result && list->mode == BUNDLE_MODE_NONE) {
 198                 warning(_("bundle list at '%s' has no mode"), uri);
 199                 result = 1;
 200         }
 201
 202         return result;
 203 }
 204
 205 static char *find_temp_filename(void)
 206 {
 207         int fd;
 208         struct strbuf name = STRBUF_INIT;
 209         /*
 210          * Find a temporary filename that is available. This is briefly
 211          * racy, but unlikely to collide.
 212          */
 213         fd = odb_mkstemp(&name, "bundles/tmp_uri_XXXXXX");
 214         if (fd < 0) {
 215                 warning(_("failed to create temporary file"));
 216                 return NULL;
 217         }
 218
 219         close(fd);
 220         unlink(name.buf);
 221         return strbuf_detach(&name, NULL);
 222 }
 223
 224 static int download_https_uri_to_file(const char *file, const char *uri)
 225 {
 226         int result = 0;
 227         struct child_process cp = CHILD_PROCESS_INIT;
 228         FILE *child_in = NULL, *child_out = NULL;
 229         struct strbuf line = STRBUF_INIT;
 230         int found_get = 0;
 231
 232         strvec_pushl(&cp.args, "git-remote-https", uri, NULL);
 233         cp.err = -1;
 234         cp.in = -1;
 235         cp.out = -1;
 236
 237         if (start_command(&cp))
 238                 return 1;
 239
 240         child_in = fdopen(cp.in, "w");
 241         if (!child_in) {
 242                 result = 1;
 243                 goto cleanup;
 244         }
 245
 246         child_out = fdopen(cp.out, "r");
 247         if (!child_out) {
 248                 result = 1;
 249                 goto cleanup;
 250         }
 251
 252         fprintf(child_in, "capabilities\n");
 253         fflush(child_in);
 254
 255         while (!strbuf_getline(&line, child_out)) {
 256                 if (!line.len)
 257                         break;
 258                 if (!strcmp(line.buf, "get"))
 259                         found_get = 1;
 260         }
 261         strbuf_release(&line);
 262
 263         if (!found_get) {
 264                 result = error(_("insufficient capabilities"));
 265                 goto cleanup;
 266         }
 267
 268         fprintf(child_in, "get %s %s\n\n", uri, file);
 269
 270 cleanup:
 271         if (child_in)
 272                 fclose(child_in);
 273         if (finish_command(&cp))
 274                 return 1;
 275         if (child_out)
 276                 fclose(child_out);
 277         return result;
 278 }
 279
 280 static int copy_uri_to_file(const char *filename, const char *uri)
 281 {
 282         const char *out;
 283
 284         if (starts_with(uri, "https:") ||
 285             starts_with(uri, "http:"))
 286                 return download_https_uri_to_file(filename, uri);
 287
 288         if (skip_prefix(uri, "file://", &out))
 289                 uri = out;
 290
 291         /* Copy as a file */
 292         return copy_file(filename, uri, 0);
 293 }
 294
 295 static int unbundle_from_file(struct repository *r, const char *file)
 296 {
 297         int result = 0;
 298         int bundle_fd;
 299         struct bundle_header header = BUNDLE_HEADER_INIT;
 300         struct string_list_item *refname;
 301         struct strbuf bundle_ref = STRBUF_INIT;
 302         size_t bundle_prefix_len;
 303
 304         if ((bundle_fd = read_bundle_header(file, &header)) < 0)
 305                 return 1;
 306
 307         /*
 308          * Skip the reachability walk here, since we will be adding
 309          * a reachable ref pointing to the new tips, which will reach
 310          * the prerequisite commits.
 311          */
 312         if ((result = unbundle(r, &header, bundle_fd, NULL,
 313                                VERIFY_BUNDLE_QUIET)))
 314                 return 1;
 315
 316         /*
 317          * Convert all refs/heads/ from the bundle into refs/bundles/
 318          * in the local repository.
 319          */
 320         strbuf_addstr(&bundle_ref, "refs/bundles/");
 321         bundle_prefix_len = bundle_ref.len;
 322
 323         for_each_string_list_item(refname, &header.references) {
 324                 struct object_id *oid = refname->util;
 325                 struct object_id old_oid;
 326                 const char *branch_name;
 327                 int has_old;
 328
 329                 if (!skip_prefix(refname->string, "refs/heads/", &branch_name))
 330                         continue;
 331
 332                 strbuf_setlen(&bundle_ref, bundle_prefix_len);
 333                 strbuf_addstr(&bundle_ref, branch_name);
 334
 335                 has_old = !read_ref(bundle_ref.buf, &old_oid);
 336                 update_ref("fetched bundle", bundle_ref.buf, oid,
 337                            has_old ? &old_oid : NULL,
 338                            REF_SKIP_OID_VERIFICATION,
 339                            UPDATE_REFS_MSG_ON_ERR);
 340         }
 341
 342         bundle_header_release(&header);
 343         return result;
 344 }
 345
 346 struct bundle_list_context {
 347         struct repository *r;
 348         struct bundle_list *list;
 349         enum bundle_list_mode mode;
 350         int count;
 351         int depth;
 352 };
 353
 354 /*
 355  * This early definition is necessary because we use indirect recursion:
 356  *
 357  * While iterating through a bundle list that was downloaded as part
 358  * of fetch_bundle_uri_internal(), iterator methods eventually call it
 359  * again, but with depth + 1.
 360  */
 361 static int fetch_bundle_uri_internal(struct repository *r,
 362                                      struct remote_bundle_info *bundle,
 363                                      int depth,
 364                                      struct bundle_list *list);
 365
 366 static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data)
 367 {
 368         int res;
 369         struct bundle_list_context *ctx = data;
 370
 371         if (ctx->mode == BUNDLE_MODE_ANY && ctx->count)
 372                 return 0;
 373
 374         res = fetch_bundle_uri_internal(ctx->r, bundle, ctx->depth + 1, ctx->list);
 375
 376         /*
 377          * Only increment count if the download succeeded. If our mode is
 378          * BUNDLE_MODE_ANY, then we will want to try other URIs in the
 379          * list in case they work instead.
 380          */
 381         if (!res)
 382                 ctx->count++;
 383
 384         /*
 385          * To be opportunistic as possible, we continue iterating and
 386          * download as many bundles as we can, so we can apply the ones
 387          * that work, even in BUNDLE_MODE_ALL mode.
 388          */
 389         return 0;
 390 }
 391
 392 static int download_bundle_list(struct repository *r,
 393                                 struct bundle_list *local_list,
 394                                 struct bundle_list *global_list,
 395                                 int depth)
 396 {
 397         struct bundle_list_context ctx = {
 398                 .r = r,
 399                 .list = global_list,
 400                 .depth = depth + 1,
 401                 .mode = local_list->mode,
 402         };
 403
 404         return for_all_bundles_in_list(local_list, download_bundle_to_file, &ctx);
 405 }
 406
 407 static int fetch_bundle_list_in_config_format(struct repository *r,
 408                                               struct bundle_list *global_list,
 409                                               struct remote_bundle_info *bundle,
 410                                               int depth)
 411 {
 412         int result;
 413         struct bundle_list list_from_bundle;
 414
 415         init_bundle_list(&list_from_bundle);
 416
 417         if ((result = bundle_uri_parse_config_format(bundle->uri,
 418                                                      bundle->file,
 419                                                      &list_from_bundle)))
 420                 goto cleanup;
 421
 422         if (list_from_bundle.mode == BUNDLE_MODE_NONE) {
 423                 warning(_("unrecognized bundle mode from URI '%s'"),
 424                         bundle->uri);
 425                 result = -1;
 426                 goto cleanup;
 427         }
 428
 429         if ((result = download_bundle_list(r, &list_from_bundle,
 430                                            global_list, depth)))
 431                 goto cleanup;
 432
 433 cleanup:
 434         clear_bundle_list(&list_from_bundle);
 435         return result;
 436 }
 437
 438 /**
 439  * This limits the recursion on fetch_bundle_uri_internal() when following
 440  * bundle lists.
 441  */
 442 static int max_bundle_uri_depth = 4;
 443
 444 /**
 445  * Recursively download all bundles advertised at the given URI
 446  * to files. If the file is a bundle, then add it to the given
 447  * 'list'. Otherwise, expect a bundle list and recurse on the
 448  * URIs in that list according to the list mode (ANY or ALL).
 449  */
 450 static int fetch_bundle_uri_internal(struct repository *r,
 451                                      struct remote_bundle_info *bundle,
 452                                      int depth,
 453                                      struct bundle_list *list)
 454 {
 455         int result = 0;
 456         struct remote_bundle_info *bcopy;
 457
 458         if (depth >= max_bundle_uri_depth) {
 459                 warning(_("exceeded bundle URI recursion limit (%d)"),
 460                         max_bundle_uri_depth);
 461                 return -1;
 462         }
 463
 464         if (!bundle->file &&
 465             !(bundle->file = find_temp_filename())) {
 466                 result = -1;
 467                 goto cleanup;
 468         }
 469
 470         if ((result = copy_uri_to_file(bundle->file, bundle->uri))) {
 471                 warning(_("failed to download bundle from URI '%s'"), bundle->uri);
 472                 goto cleanup;
 473         }
 474
 475         if ((result = !is_bundle(bundle->file, 1))) {
 476                 result = fetch_bundle_list_in_config_format(
 477                                 r, list, bundle, depth);
 478                 if (result)
 479                         warning(_("file at URI '%s' is not a bundle or bundle list"),
 480                                 bundle->uri);
 481                 goto cleanup;
 482         }
 483
 484         /* Copy the bundle and insert it into the global list. */
 485         CALLOC_ARRAY(bcopy, 1);
 486         bcopy->id = xstrdup(bundle->id);
 487         bcopy->file = xstrdup(bundle->file);
 488         hashmap_entry_init(&bcopy->ent, strhash(bcopy->id));
 489         hashmap_add(&list->bundles, &bcopy->ent);
 490
 491 cleanup:
 492         if (result && bundle->file)
 493                 unlink(bundle->file);
 494         return result;
 495 }
 496
 497 /**
 498  * This loop iterator breaks the loop with nonzero return code on the
 499  * first successful unbundling of a bundle.
 500  */
 501 static int attempt_unbundle(struct remote_bundle_info *info, void *data)
 502 {
 503         struct repository *r = data;
 504
 505         if (!info->file || info->unbundled)
 506                 return 0;
 507
 508         if (!unbundle_from_file(r, info->file)) {
 509                 info->unbundled = 1;
 510                 return 1;
 511         }
 512
 513         return 0;
 514 }
 515
 516 static int unbundle_all_bundles(struct repository *r,
 517                                 struct bundle_list *list)
 518 {
 519         /*
 520          * Iterate through all bundles looking for ones that can
 521          * successfully unbundle. If any succeed, then perhaps another
 522          * will succeed in the next attempt.
 523          *
 524          * Keep in mind that a non-zero result for the loop here means
 525          * the loop terminated early on a successful unbundling, which
 526          * signals that we can try again.
 527          */
 528         while (for_all_bundles_in_list(list, attempt_unbundle, r)) ;
 529
 530         return 0;
 531 }
 532
 533 static int unlink_bundle(struct remote_bundle_info *info, void *data)
 534 {
 535         if (info->file)
 536                 unlink_or_warn(info->file);
 537         return 0;
 538 }
 539
 540 int fetch_bundle_uri(struct repository *r, const char *uri)
 541 {
 542         int result;
 543         struct bundle_list list;
 544         struct remote_bundle_info bundle = {
 545                 .uri = xstrdup(uri),
 546                 .id = xstrdup(""),
 547         };
 548
 549         init_bundle_list(&list);
 550
 551         /* If a bundle is added to this global list, then it is required. */
 552         list.mode = BUNDLE_MODE_ALL;
 553
 554         if ((result = fetch_bundle_uri_internal(r, &bundle, 0, &list)))
 555                 goto cleanup;
 556
 557         result = unbundle_all_bundles(r, &list);
 558
 559 cleanup:
 560         for_all_bundles_in_list(&list, unlink_bundle, NULL);
 561         clear_bundle_list(&list);
 562         clear_remote_bundle_info(&bundle, NULL);
 563         return result;
 564 }
 565
 566 /**
 567  * General API for {transport,connect}.c etc.
 568  */
 569 int bundle_uri_parse_line(struct bundle_list *list, const char *line)
 570 {
 571         int result;
 572         const char *equals;
 573         struct strbuf key = STRBUF_INIT;
 574
 575         if (!strlen(line))
 576                 return error(_("bundle-uri: got an empty line"));
 577
 578         equals = strchr(line, '=');
 579
 580         if (!equals)
 581                 return error(_("bundle-uri: line is not of the form 'key=value'"));
 582         if (line == equals || !*(equals + 1))
 583                 return error(_("bundle-uri: line has empty key or value"));
 584
 585         strbuf_add(&key, line, equals - line);
 586         result = bundle_list_update(key.buf, equals + 1, list);
 587         strbuf_release(&key);
 588
 589         return result;
 590 }