remote-curl.c

   1 #include "cache.h"
   2 #include "remote.h"
   3 #include "strbuf.h"
   4 #include "walker.h"
   5 #include "http.h"
   6 #include "exec_cmd.h"
   7 #include "run-command.h"
   8 #include "pkt-line.h"
   9 #include "sideband.h"
  10
  11 static struct remote *remote;
  12 static const char *url;
  13 static struct walker *walker;
  14
  15 struct options {
  16         int verbosity;
  17         unsigned long depth;
  18         unsigned progress : 1,
  19                 followtags : 1,
  20                 dry_run : 1,
  21                 thin : 1;
  22 };
  23 static struct options options;
  24
  25 static void init_walker(void)
  26 {
  27         if (!walker)
  28                 walker = get_http_walker(url, remote);
  29 }
  30
  31 static int set_option(const char *name, const char *value)
  32 {
  33         if (!strcmp(name, "verbosity")) {
  34                 char *end;
  35                 int v = strtol(value, &end, 10);
  36                 if (value == end || *end)
  37                         return -1;
  38                 options.verbosity = v;
  39                 return 0;
  40         }
  41         else if (!strcmp(name, "progress")) {
  42                 if (!strcmp(value, "true"))
  43                         options.progress = 1;
  44                 else if (!strcmp(value, "false"))
  45                         options.progress = 0;
  46                 else
  47                         return -1;
  48                 return 1 /* TODO implement later */;
  49         }
  50         else if (!strcmp(name, "depth")) {
  51                 char *end;
  52                 unsigned long v = strtoul(value, &end, 10);
  53                 if (value == end || *end)
  54                         return -1;
  55                 options.depth = v;
  56                 return 1 /* TODO implement later */;
  57         }
  58         else if (!strcmp(name, "followtags")) {
  59                 if (!strcmp(value, "true"))
  60                         options.followtags = 1;
  61                 else if (!strcmp(value, "false"))
  62                         options.followtags = 0;
  63                 else
  64                         return -1;
  65                 return 1 /* TODO implement later */;
  66         }
  67         else if (!strcmp(name, "dry-run")) {
  68                 if (!strcmp(value, "true"))
  69                         options.dry_run = 1;
  70                 else if (!strcmp(value, "false"))
  71                         options.dry_run = 0;
  72                 else
  73                         return -1;
  74                 return 0;
  75         }
  76         else {
  77                 return 1 /* unsupported */;
  78         }
  79 }
  80
  81 struct discovery {
  82         const char *service;
  83         char *buf_alloc;
  84         char *buf;
  85         size_t len;
  86         unsigned proto_git : 1;
  87 };
  88 static struct discovery *last_discovery;
  89
  90 static void free_discovery(struct discovery *d)
  91 {
  92         if (d) {
  93                 if (d == last_discovery)
  94                         last_discovery = NULL;
  95                 free(d->buf_alloc);
  96                 free(d);
  97         }
  98 }
  99
 100 static struct discovery* discover_refs(const char *service)
 101 {
 102         struct strbuf buffer = STRBUF_INIT;
 103         struct discovery *last = last_discovery;
 104         char *refs_url;
 105         int http_ret, is_http = 0;
 106
 107         if (last && !strcmp(service, last->service))
 108                 return last;
 109         free_discovery(last);
 110
 111         strbuf_addf(&buffer, "%s/info/refs", url);
 112         if (!prefixcmp(url, "http://") || !prefixcmp(url, "https://")) {
 113                 is_http = 1;
 114                 if (!strchr(url, '?'))
 115                         strbuf_addch(&buffer, '?');
 116                 else
 117                         strbuf_addch(&buffer, '&');
 118                 strbuf_addf(&buffer, "service=%s", service);
 119         }
 120         refs_url = strbuf_detach(&buffer, NULL);
 121
 122         init_walker();
 123         http_ret = http_get_strbuf(refs_url, &buffer, HTTP_NO_CACHE);
 124         switch (http_ret) {
 125         case HTTP_OK:
 126                 break;
 127         case HTTP_MISSING_TARGET:
 128                 die("%s not found: did you run git update-server-info on the"
 129                     " server?", refs_url);
 130         default:
 131                 http_error(refs_url, http_ret);
 132                 die("HTTP request failed");
 133         }
 134
 135         last= xcalloc(1, sizeof(*last_discovery));
 136         last->service = service;
 137         last->buf_alloc = strbuf_detach(&buffer, &last->len);
 138         last->buf = last->buf_alloc;
 139
 140         if (is_http && 5 <= last->len && last->buf[4] == '#') {
 141                 /* smart HTTP response; validate that the service
 142                  * pkt-line matches our request.
 143                  */
 144                 struct strbuf exp = STRBUF_INIT;
 145
 146                 if (packet_get_line(&buffer, &last->buf, &last->len) <= 0)
 147                         die("%s has invalid packet header", refs_url);
 148                 if (buffer.len && buffer.buf[buffer.len - 1] == '\n')
 149                         strbuf_setlen(&buffer, buffer.len - 1);
 150
 151                 strbuf_addf(&exp, "# service=%s", service);
 152                 if (strbuf_cmp(&exp, &buffer))
 153                         die("invalid server response; got '%s'", buffer.buf);
 154                 strbuf_release(&exp);
 155
 156                 /* The header can include additional metadata lines, up
 157                  * until a packet flush marker.  Ignore these now, but
 158                  * in the future we might start to scan them.
 159                  */
 160                 strbuf_reset(&buffer);
 161                 while (packet_get_line(&buffer, &last->buf, &last->len) > 0)
 162                         strbuf_reset(&buffer);
 163
 164                 last->proto_git = 1;
 165         }
 166
 167         free(refs_url);
 168         strbuf_release(&buffer);
 169         last_discovery = last;
 170         return last;
 171 }
 172
 173 static int write_discovery(int fd, void *data)
 174 {
 175         struct discovery *heads = data;
 176         int err = 0;
 177         if (write_in_full(fd, heads->buf, heads->len) != heads->len)
 178                 err = 1;
 179         close(fd);
 180         return err;
 181 }
 182
 183 static struct ref *parse_git_refs(struct discovery *heads)
 184 {
 185         struct ref *list = NULL;
 186         struct async async;
 187
 188         memset(&async, 0, sizeof(async));
 189         async.proc = write_discovery;
 190         async.data = heads;
 191
 192         if (start_async(&async))
 193                 die("cannot start thread to parse advertised refs");
 194         get_remote_heads(async.out, &list, 0, NULL, 0, NULL);
 195         close(async.out);
 196         if (finish_async(&async))
 197                 die("ref parsing thread failed");
 198         return list;
 199 }
 200
 201 static struct ref *parse_info_refs(struct discovery *heads)
 202 {
 203         char *data, *start, *mid;
 204         char *ref_name;
 205         int i = 0;
 206
 207         struct ref *refs = NULL;
 208         struct ref *ref = NULL;
 209         struct ref *last_ref = NULL;
 210
 211         data = heads->buf;
 212         start = NULL;
 213         mid = data;
 214         while (i < heads->len) {
 215                 if (!start) {
 216                         start = &data[i];
 217                 }
 218                 if (data[i] == '\t')
 219                         mid = &data[i];
 220                 if (data[i] == '\n') {
 221                         data[i] = 0;
 222                         ref_name = mid + 1;
 223                         ref = xmalloc(sizeof(struct ref) +
 224                                       strlen(ref_name) + 1);
 225                         memset(ref, 0, sizeof(struct ref));
 226                         strcpy(ref->name, ref_name);
 227                         get_sha1_hex(start, ref->old_sha1);
 228                         if (!refs)
 229                                 refs = ref;
 230                         if (last_ref)
 231                                 last_ref->next = ref;
 232                         last_ref = ref;
 233                         start = NULL;
 234                 }
 235                 i++;
 236         }
 237
 238         init_walker();
 239         ref = alloc_ref("HEAD");
 240         if (!walker->fetch_ref(walker, ref) &&
 241             !resolve_remote_symref(ref, refs)) {
 242                 ref->next = refs;
 243                 refs = ref;
 244         } else {
 245                 free(ref);
 246         }
 247
 248         return refs;
 249 }
 250
 251 static struct ref *get_refs(int for_push)
 252 {
 253         struct discovery *heads;
 254
 255         if (for_push)
 256                 heads = discover_refs("git-receive-pack");
 257         else
 258                 heads = discover_refs("git-upload-pack");
 259
 260         if (heads->proto_git)
 261                 return parse_git_refs(heads);
 262         return parse_info_refs(heads);
 263 }
 264
 265 static void output_refs(struct ref *refs)
 266 {
 267         struct ref *posn;
 268         for (posn = refs; posn; posn = posn->next) {
 269                 if (posn->symref)
 270                         printf("@%s %s\n", posn->symref, posn->name);
 271                 else
 272                         printf("%s %s\n", sha1_to_hex(posn->old_sha1), posn->name);
 273         }
 274         printf("\n");
 275         fflush(stdout);
 276         free_refs(refs);
 277 }
 278
 279 struct rpc_state {
 280         const char *service_name;
 281         const char **argv;
 282         char *service_url;
 283         char *hdr_content_type;
 284         char *hdr_accept;
 285         char *buf;
 286         size_t alloc;
 287         size_t len;
 288         size_t pos;
 289         int in;
 290         int out;
 291         struct strbuf result;
 292 };
 293
 294 static size_t rpc_out(void *ptr, size_t eltsize,
 295                 size_t nmemb, void *buffer_)
 296 {
 297         size_t max = eltsize * nmemb;
 298         struct rpc_state *rpc = buffer_;
 299         size_t avail = rpc->len - rpc->pos;
 300
 301         if (!avail) {
 302                 avail = packet_read_line(rpc->out, rpc->buf, rpc->alloc);
 303                 if (!avail)
 304                         return 0;
 305                 rpc->pos = 0;
 306                 rpc->len = avail;
 307         }
 308
 309         if (max < avail);
 310                 avail = max;
 311         memcpy(ptr, rpc->buf + rpc->pos, avail);
 312         rpc->pos += avail;
 313         return avail;
 314 }
 315
 316 static size_t rpc_in(const void *ptr, size_t eltsize,
 317                 size_t nmemb, void *buffer_)
 318 {
 319         size_t size = eltsize * nmemb;
 320         struct rpc_state *rpc = buffer_;
 321         write_or_die(rpc->in, ptr, size);
 322         return size;
 323 }
 324
 325 static int post_rpc(struct rpc_state *rpc)
 326 {
 327         struct active_request_slot *slot;
 328         struct slot_results results;
 329         struct curl_slist *headers = NULL;
 330         int err = 0, large_request = 0;
 331
 332         /* Try to load the entire request, if we can fit it into the
 333          * allocated buffer space we can use HTTP/1.0 and avoid the
 334          * chunked encoding mess.
 335          */
 336         while (1) {
 337                 size_t left = rpc->alloc - rpc->len;
 338                 char *buf = rpc->buf + rpc->len;
 339                 int n;
 340
 341                 if (left < LARGE_PACKET_MAX) {
 342                         large_request = 1;
 343                         break;
 344                 }
 345
 346                 n = packet_read_line(rpc->out, buf, left);
 347                 if (!n)
 348                         break;
 349                 rpc->len += n;
 350         }
 351
 352         slot = get_active_slot();
 353         slot->results = &results;
 354
 355         curl_easy_setopt(slot->curl, CURLOPT_POST, 1);
 356         curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
 357         curl_easy_setopt(slot->curl, CURLOPT_URL, rpc->service_url);
 358
 359         headers = curl_slist_append(headers, rpc->hdr_content_type);
 360         headers = curl_slist_append(headers, rpc->hdr_accept);
 361
 362         if (large_request) {
 363                 /* The request body is large and the size cannot be predicted.
 364                  * We must use chunked encoding to send it.
 365                  */
 366                 headers = curl_slist_append(headers, "Expect: 100-continue");
 367                 headers = curl_slist_append(headers, "Transfer-Encoding: chunked");
 368                 curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, rpc_out);
 369                 curl_easy_setopt(slot->curl, CURLOPT_INFILE, rpc);
 370                 if (options.verbosity > 1) {
 371                         fprintf(stderr, "POST %s (chunked)\n", rpc->service_name);
 372                         fflush(stderr);
 373                 }
 374
 375         } else {
 376                 /* We know the complete request size in advance, use the
 377                  * more normal Content-Length approach.
 378                  */
 379                 curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, rpc->buf);
 380                 curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, rpc->len);
 381                 if (options.verbosity > 1) {
 382                         fprintf(stderr, "POST %s (%lu bytes)\n",
 383                                 rpc->service_name, (unsigned long)rpc->len);
 384                         fflush(stderr);
 385                 }
 386         }
 387
 388         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
 389         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, rpc_in);
 390         curl_easy_setopt(slot->curl, CURLOPT_FILE, rpc);
 391
 392         slot->curl_result = curl_easy_perform(slot->curl);
 393         finish_active_slot(slot);
 394
 395         if (results.curl_result != CURLE_OK) {
 396                 err |= error("RPC failed; result=%d, HTTP code = %ld",
 397                         results.curl_result, results.http_code);
 398         }
 399
 400         curl_slist_free_all(headers);
 401         return err;
 402 }
 403
 404 static int rpc_service(struct rpc_state *rpc, struct discovery *heads)
 405 {
 406         const char *svc = rpc->service_name;
 407         struct strbuf buf = STRBUF_INIT;
 408         struct child_process client;
 409         int err = 0;
 410
 411         init_walker();
 412         memset(&client, 0, sizeof(client));
 413         client.in = -1;
 414         client.out = -1;
 415         client.git_cmd = 1;
 416         client.argv = rpc->argv;
 417         if (start_command(&client))
 418                 exit(1);
 419         if (heads)
 420                 write_or_die(client.in, heads->buf, heads->len);
 421
 422         rpc->alloc = http_post_buffer;
 423         rpc->buf = xmalloc(rpc->alloc);
 424         rpc->in = client.in;
 425         rpc->out = client.out;
 426         strbuf_init(&rpc->result, 0);
 427
 428         strbuf_addf(&buf, "%s/%s", url, svc);
 429         rpc->service_url = strbuf_detach(&buf, NULL);
 430
 431         strbuf_addf(&buf, "Content-Type: application/x-%s-request", svc);
 432         rpc->hdr_content_type = strbuf_detach(&buf, NULL);
 433
 434         strbuf_addf(&buf, "Accept: application/x-%s-response", svc);
 435         rpc->hdr_accept = strbuf_detach(&buf, NULL);
 436
 437         while (!err) {
 438                 int n = packet_read_line(rpc->out, rpc->buf, rpc->alloc);
 439                 if (!n)
 440                         break;
 441                 rpc->pos = 0;
 442                 rpc->len = n;
 443                 err |= post_rpc(rpc);
 444         }
 445         strbuf_read(&rpc->result, client.out, 0);
 446
 447         close(client.in);
 448         close(client.out);
 449         client.in = -1;
 450         client.out = -1;
 451
 452         err |= finish_command(&client);
 453         free(rpc->service_url);
 454         free(rpc->hdr_content_type);
 455         free(rpc->hdr_accept);
 456         free(rpc->buf);
 457         strbuf_release(&buf);
 458         return err;
 459 }
 460
 461 static int fetch_dumb(int nr_heads, struct ref **to_fetch)
 462 {
 463         char **targets = xmalloc(nr_heads * sizeof(char*));
 464         int ret, i;
 465
 466         for (i = 0; i < nr_heads; i++)
 467                 targets[i] = xstrdup(sha1_to_hex(to_fetch[i]->old_sha1));
 468
 469         init_walker();
 470         walker->get_all = 1;
 471         walker->get_tree = 1;
 472         walker->get_history = 1;
 473         walker->get_verbosely = options.verbosity >= 3;
 474         walker->get_recover = 0;
 475         ret = walker_fetch(walker, nr_heads, targets, NULL, NULL);
 476
 477         for (i = 0; i < nr_heads; i++)
 478                 free(targets[i]);
 479         free(targets);
 480
 481         return ret ? error("Fetch failed.") : 0;
 482 }
 483
 484 static void parse_fetch(struct strbuf *buf)
 485 {
 486         struct ref **to_fetch = NULL;
 487         struct ref *list_head = NULL;
 488         struct ref **list = &list_head;
 489         int alloc_heads = 0, nr_heads = 0;
 490
 491         do {
 492                 if (!prefixcmp(buf->buf, "fetch ")) {
 493                         char *p = buf->buf + strlen("fetch ");
 494                         char *name;
 495                         struct ref *ref;
 496                         unsigned char old_sha1[20];
 497
 498                         if (strlen(p) < 40 || get_sha1_hex(p, old_sha1))
 499                                 die("protocol error: expected sha/ref, got %s'", p);
 500                         if (p[40] == ' ')
 501                                 name = p + 41;
 502                         else if (!p[40])
 503                                 name = "";
 504                         else
 505                                 die("protocol error: expected sha/ref, got %s'", p);
 506
 507                         ref = alloc_ref(name);
 508                         hashcpy(ref->old_sha1, old_sha1);
 509
 510                         *list = ref;
 511                         list = &ref->next;
 512
 513                         ALLOC_GROW(to_fetch, nr_heads + 1, alloc_heads);
 514                         to_fetch[nr_heads++] = ref;
 515                 }
 516                 else
 517                         die("http transport does not support %s", buf->buf);
 518
 519                 strbuf_reset(buf);
 520                 if (strbuf_getline(buf, stdin, '\n') == EOF)
 521                         return;
 522                 if (!*buf->buf)
 523                         break;
 524         } while (1);
 525
 526         if (fetch_dumb(nr_heads, to_fetch))
 527                 exit(128); /* error already reported */
 528         free_refs(list_head);
 529         free(to_fetch);
 530
 531         printf("\n");
 532         fflush(stdout);
 533         strbuf_reset(buf);
 534 }
 535
 536 static int push_dav(int nr_spec, char **specs)
 537 {
 538         const char **argv = xmalloc((10 + nr_spec) * sizeof(char*));
 539         int argc = 0, i;
 540
 541         argv[argc++] = "http-push";
 542         argv[argc++] = "--helper-status";
 543         if (options.dry_run)
 544                 argv[argc++] = "--dry-run";
 545         if (options.verbosity > 1)
 546                 argv[argc++] = "--verbose";
 547         argv[argc++] = url;
 548         for (i = 0; i < nr_spec; i++)
 549                 argv[argc++] = specs[i];
 550         argv[argc++] = NULL;
 551
 552         if (run_command_v_opt(argv, RUN_GIT_CMD))
 553                 die("git-%s failed", argv[0]);
 554         free(argv);
 555         return 0;
 556 }
 557
 558 static int push_git(struct discovery *heads, int nr_spec, char **specs)
 559 {
 560         struct rpc_state rpc;
 561         const char **argv;
 562         int argc = 0, i, err;
 563
 564         argv = xmalloc((10 + nr_spec) * sizeof(char*));
 565         argv[argc++] = "send-pack";
 566         argv[argc++] = "--stateless-rpc";
 567         argv[argc++] = "--helper-status";
 568         if (options.thin)
 569                 argv[argc++] = "--thin";
 570         if (options.dry_run)
 571                 argv[argc++] = "--dry-run";
 572         if (options.verbosity > 1)
 573                 argv[argc++] = "--verbose";
 574         argv[argc++] = url;
 575         for (i = 0; i < nr_spec; i++)
 576                 argv[argc++] = specs[i];
 577         argv[argc++] = NULL;
 578
 579         memset(&rpc, 0, sizeof(rpc));
 580         rpc.service_name = "git-receive-pack",
 581         rpc.argv = argv;
 582
 583         err = rpc_service(&rpc, heads);
 584         if (rpc.result.len)
 585                 safe_write(1, rpc.result.buf, rpc.result.len);
 586         strbuf_release(&rpc.result);
 587         free(argv);
 588         return err;
 589 }
 590
 591 static int push(int nr_spec, char **specs)
 592 {
 593         struct discovery *heads = discover_refs("git-receive-pack");
 594         int ret;
 595
 596         if (heads->proto_git)
 597                 ret = push_git(heads, nr_spec, specs);
 598         else
 599                 ret = push_dav(nr_spec, specs);
 600         free_discovery(heads);
 601         return ret;
 602 }
 603
 604 static void parse_push(struct strbuf *buf)
 605 {
 606         char **specs = NULL;
 607         int alloc_spec = 0, nr_spec = 0, i;
 608
 609         do {
 610                 if (!prefixcmp(buf->buf, "push ")) {
 611                         ALLOC_GROW(specs, nr_spec + 1, alloc_spec);
 612                         specs[nr_spec++] = xstrdup(buf->buf + 5);
 613                 }
 614                 else
 615                         die("http transport does not support %s", buf->buf);
 616
 617                 strbuf_reset(buf);
 618                 if (strbuf_getline(buf, stdin, '\n') == EOF)
 619                         return;
 620                 if (!*buf->buf)
 621                         break;
 622         } while (1);
 623
 624         if (push(nr_spec, specs))
 625                 exit(128); /* error already reported */
 626         for (i = 0; i < nr_spec; i++)
 627                 free(specs[i]);
 628         free(specs);
 629
 630         printf("\n");
 631         fflush(stdout);
 632 }
 633
 634 int main(int argc, const char **argv)
 635 {
 636         struct strbuf buf = STRBUF_INIT;
 637
 638         git_extract_argv0_path(argv[0]);
 639         setup_git_directory();
 640         if (argc < 2) {
 641                 fprintf(stderr, "Remote needed\n");
 642                 return 1;
 643         }
 644
 645         options.verbosity = 1;
 646         options.progress = !!isatty(2);
 647         options.thin = 1;
 648
 649         remote = remote_get(argv[1]);
 650
 651         if (argc > 2) {
 652                 url = argv[2];
 653         } else {
 654                 url = remote->url[0];
 655         }
 656
 657         do {
 658                 if (strbuf_getline(&buf, stdin, '\n') == EOF)
 659                         break;
 660                 if (!prefixcmp(buf.buf, "fetch ")) {
 661                         parse_fetch(&buf);
 662
 663                 } else if (!strcmp(buf.buf, "list") || !prefixcmp(buf.buf, "list ")) {
 664                         int for_push = !!strstr(buf.buf + 4, "for-push");
 665                         output_refs(get_refs(for_push));
 666
 667                 } else if (!prefixcmp(buf.buf, "push ")) {
 668                         parse_push(&buf);
 669
 670                 } else if (!prefixcmp(buf.buf, "option ")) {
 671                         char *name = buf.buf + strlen("option ");
 672                         char *value = strchr(name, ' ');
 673                         int result;
 674
 675                         if (value)
 676                                 *value++ = '\0';
 677                         else
 678                                 value = "true";
 679
 680                         result = set_option(name, value);
 681                         if (!result)
 682                                 printf("ok\n");
 683                         else if (result < 0)
 684                                 printf("error invalid value\n");
 685                         else
 686                                 printf("unsupported\n");
 687                         fflush(stdout);
 688
 689                 } else if (!strcmp(buf.buf, "capabilities")) {
 690                         printf("fetch\n");
 691                         printf("option\n");
 692                         printf("push\n");
 693                         printf("\n");
 694                         fflush(stdout);
 695                 } else {
 696                         return 1;
 697                 }
 698                 strbuf_reset(&buf);
 699         } while (1);
 700         return 0;
 701 }