plugins/curl/curl.c

   1 /* nbdkit
   2  * Copyright (C) 2014-2023 Red Hat Inc.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions are
   6  * met:
   7  *
   8  * * Redistributions of source code must retain the above copyright
   9  * notice, this list of conditions and the following disclaimer.
  10  *
  11  * * Redistributions in binary form must reproduce the above copyright
  12  * notice, this list of conditions and the following disclaimer in the
  13  * documentation and/or other materials provided with the distribution.
  14  *
  15  * * Neither the name of Red Hat nor the names of its contributors may be
  16  * used to endorse or promote products derived from this software without
  17  * specific prior written permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  21  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
  22  * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
  23  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  26  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  27  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  28  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  29  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  30  * SUCH DAMAGE.
  31  */
  32
  33 #include <config.h>
  34
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <stdarg.h>
  38 #include <stdbool.h>
  39 #include <stdint.h>
  40 #include <inttypes.h>
  41 #include <limits.h>
  42 #include <string.h>
  43 #include <unistd.h>
  44 #include <errno.h>
  45 #include <assert.h>
  46
  47 #include <curl/curl.h>
  48
  49 #include <nbdkit-plugin.h>
  50
  51 #include "cleanup.h"
  52
  53 #include "curldefs.h"
  54
  55 /* Plugin configuration. */
  56 const char *url = NULL;         /* required */
  57
  58 const char *cainfo = NULL;
  59 const char *capath = NULL;
  60 unsigned connections = 4;
  61 char *cookie = NULL;
  62 const char *cookiefile = NULL;
  63 const char *cookiejar = NULL;
  64 const char *cookie_script = NULL;
  65 unsigned cookie_script_renew = 0;
  66 bool followlocation = true;
  67 struct curl_slist *headers = NULL;
  68 const char *header_script = NULL;
  69 unsigned header_script_renew = 0;
  70 char *password = NULL;
  71 #ifndef HAVE_CURLOPT_PROTOCOLS_STR
  72 long protocols = CURLPROTO_ALL;
  73 #else
  74 const char *protocols = NULL;
  75 #endif
  76 const char *proxy = NULL;
  77 char *proxy_password = NULL;
  78 const char *proxy_user = NULL;
  79 bool sslverify = true;
  80 const char *ssl_cipher_list = NULL;
  81 const char *ssl_version = NULL;
  82 const char *tls13_ciphers = NULL;
  83 bool tcp_keepalive = false;
  84 bool tcp_nodelay = true;
  85 uint32_t timeout = 0;
  86 const char *unix_socket_path = NULL;
  87 const char *user = NULL;
  88 const char *user_agent = NULL;
  89
  90 /* Use '-D curl.verbose=1' to set. */
  91 NBDKIT_DLL_PUBLIC int curl_debug_verbose = 0;
  92
  93 static void
  94 curl_load (void)
  95 {
  96   CURLcode r;
  97
  98   r = curl_global_init (CURL_GLOBAL_DEFAULT);
  99   if (r != CURLE_OK) {
 100     nbdkit_error ("libcurl initialization failed: %d", (int) r);
 101     exit (EXIT_FAILURE);
 102   }
 103 }
 104
 105 static void
 106 curl_unload (void)
 107 {
 108   free (cookie);
 109   if (headers)
 110     curl_slist_free_all (headers);
 111   free (password);
 112   free (proxy_password);
 113   scripts_unload ();
 114   free_all_handles ();
 115   curl_global_cleanup ();
 116 }
 117
 118 #ifndef HAVE_CURLOPT_PROTOCOLS_STR
 119 /* See <curl/curl.h> */
 120 static struct { const char *name; long bitmask; } curl_protocols[] = {
 121   { "http", CURLPROTO_HTTP },
 122   { "https", CURLPROTO_HTTPS },
 123   { "ftp", CURLPROTO_FTP },
 124   { "ftps", CURLPROTO_FTPS },
 125   { "scp", CURLPROTO_SCP },
 126   { "sftp", CURLPROTO_SFTP },
 127   { "telnet", CURLPROTO_TELNET },
 128   { "ldap", CURLPROTO_LDAP },
 129   { "ldaps", CURLPROTO_LDAPS },
 130   { "dict", CURLPROTO_DICT },
 131   { "file", CURLPROTO_FILE },
 132   { "tftp", CURLPROTO_TFTP },
 133   { "imap", CURLPROTO_IMAP },
 134   { "imaps", CURLPROTO_IMAPS },
 135   { "pop3", CURLPROTO_POP3 },
 136   { "pop3s", CURLPROTO_POP3S },
 137   { "smtp", CURLPROTO_SMTP },
 138   { "smtps", CURLPROTO_SMTPS },
 139   { "rtsp", CURLPROTO_RTSP },
 140   { "rtmp", CURLPROTO_RTMP },
 141   { "rtmpt", CURLPROTO_RTMPT },
 142   { "rtmpe", CURLPROTO_RTMPE },
 143   { "rtmpte", CURLPROTO_RTMPTE },
 144   { "rtmps", CURLPROTO_RTMPS },
 145   { "rtmpts", CURLPROTO_RTMPTS },
 146   { "gopher", CURLPROTO_GOPHER },
 147 #ifdef CURLPROTO_SMB
 148   { "smb", CURLPROTO_SMB },
 149 #endif
 150 #ifdef CURLPROTO_SMBS
 151   { "smbs", CURLPROTO_SMBS },
 152 #endif
 153 #ifdef CURLPROTO_MQTT
 154   { "mqtt", CURLPROTO_MQTT },
 155 #endif
 156   { NULL }
 157 };
 158
 159 /* Parse the protocols parameter. */
 160 static int
 161 parse_protocols (const char *value)
 162 {
 163   size_t n, i;
 164
 165   protocols = 0;
 166
 167   while (*value) {
 168     n = strcspn (value, ",");
 169     for (i = 0; curl_protocols[i].name != NULL; ++i) {
 170       if (strlen (curl_protocols[i].name) == n &&
 171           strncmp (value, curl_protocols[i].name, n) == 0) {
 172         protocols |= curl_protocols[i].bitmask;
 173         goto found;
 174       }
 175     }
 176     nbdkit_error ("protocols: protocol name not found: %.*s", (int) n, value);
 177     return -1;
 178
 179   found:
 180     value += n;
 181     if (*value == ',')
 182       value++;
 183   }
 184
 185   if (protocols == 0) {
 186     nbdkit_error ("protocols: empty list of protocols is not allowed");
 187     return -1;
 188   }
 189
 190   nbdkit_debug ("curl: protocols: %ld", protocols);
 191
 192   return 0;
 193 }
 194 #endif /* !HAVE_CURLOPT_PROTOCOLS_STR */
 195
 196 /* Called for each key=value passed on the command line. */
 197 static int
 198 curl_config (const char *key, const char *value)
 199 {
 200   int r;
 201
 202   if (strcmp (key, "cainfo") == 0) {
 203     cainfo = value;
 204   }
 205
 206   else if (strcmp (key, "capath") == 0) {
 207     capath =  value;
 208   }
 209
 210   else if (strcmp (key, "connections") == 0) {
 211     if (nbdkit_parse_unsigned ("connections", value,
 212                                &connections) == -1)
 213       return -1;
 214     if (connections == 0) {
 215       nbdkit_error ("connections parameter must not be 0");
 216       return -1;
 217     }
 218   }
 219
 220   else if (strcmp (key, "cookie") == 0) {
 221     free (cookie);
 222     if (nbdkit_read_password (value, &cookie) == -1)
 223       return -1;
 224   }
 225
 226   else if (strcmp (key, "cookiefile") == 0) {
 227     /* Reject cookiefile=- because it will cause libcurl to try to
 228      * read from stdin when we connect.
 229      */
 230     if (strcmp (value, "-") == 0) {
 231       nbdkit_error ("cookiefile parameter cannot be \"-\"");
 232       return -1;
 233     }
 234     cookiefile = value;
 235   }
 236
 237   else if (strcmp (key, "cookiejar") == 0) {
 238     /* Reject cookiejar=- because it will cause libcurl to try to
 239      * write to stdout.
 240      */
 241     if (strcmp (value, "-") == 0) {
 242       nbdkit_error ("cookiejar parameter cannot be \"-\"");
 243       return -1;
 244     }
 245     cookiejar = value;
 246   }
 247
 248   else if (strcmp (key, "cookie-script") == 0) {
 249     cookie_script = value;
 250   }
 251
 252   else if (strcmp (key, "cookie-script-renew") == 0) {
 253     if (nbdkit_parse_unsigned ("cookie-script-renew", value,
 254                                &cookie_script_renew) == -1)
 255       return -1;
 256   }
 257
 258   else if (strcmp (key, "followlocation") == 0) {
 259     r = nbdkit_parse_bool (value);
 260     if (r == -1)
 261       return -1;
 262     followlocation = r;
 263   }
 264
 265   else if (strcmp (key, "header") == 0) {
 266     headers = curl_slist_append (headers, value);
 267     if (headers == NULL) {
 268       nbdkit_error ("curl_slist_append: %m");
 269       return -1;
 270     }
 271   }
 272
 273   else if (strcmp (key, "header-script") == 0) {
 274     header_script = value;
 275   }
 276
 277   else if (strcmp (key, "header-script-renew") == 0) {
 278     if (nbdkit_parse_unsigned ("header-script-renew", value,
 279                                &header_script_renew) == -1)
 280       return -1;
 281   }
 282
 283   else if (strcmp (key, "password") == 0) {
 284     free (password);
 285     if (nbdkit_read_password (value, &password) == -1)
 286       return -1;
 287   }
 288
 289   else if (strcmp (key, "protocols") == 0) {
 290 #ifndef HAVE_CURLOPT_PROTOCOLS_STR
 291     if (parse_protocols (value) == -1)
 292       return -1;
 293 #else
 294     protocols = value;
 295 #endif
 296   }
 297
 298   else if (strcmp (key, "proxy") == 0) {
 299     proxy = value;
 300   }
 301
 302   else if (strcmp (key, "proxy-password") == 0) {
 303     free (proxy_password);
 304     if (nbdkit_read_password (value, &proxy_password) == -1)
 305       return -1;
 306   }
 307
 308   else if (strcmp (key, "proxy-user") == 0)
 309     proxy_user = value;
 310
 311   else if (strcmp (key, "sslverify") == 0) {
 312     r = nbdkit_parse_bool (value);
 313     if (r == -1)
 314       return -1;
 315     sslverify = r;
 316   }
 317
 318   else if (strcmp (key, "ssl-version") == 0)
 319     ssl_version = value;
 320
 321   else if (strcmp (key, "ssl-cipher-list") == 0)
 322     ssl_cipher_list = value;
 323
 324   else if (strcmp (key, "tls13-ciphers") == 0)
 325     tls13_ciphers = value;
 326
 327   else if (strcmp (key, "tcp-keepalive") == 0) {
 328     r = nbdkit_parse_bool (value);
 329     if (r == -1)
 330       return -1;
 331     tcp_keepalive = r;
 332   }
 333
 334   else if (strcmp (key, "tcp-nodelay") == 0) {
 335     r = nbdkit_parse_bool (value);
 336     if (r == -1)
 337       return -1;
 338     tcp_nodelay = r;
 339   }
 340
 341   else if (strcmp (key, "timeout") == 0) {
 342     if (nbdkit_parse_uint32_t ("timeout", value, &timeout) == -1)
 343       return -1;
 344 #if LONG_MAX < UINT32_MAX
 345     /* C17 5.2.4.2.1 requires that LONG_MAX is at least 2^31 - 1.
 346      * However a large positive number might still exceed the limit.
 347      */
 348     if (timeout > LONG_MAX) {
 349       nbdkit_error ("timeout is too large");
 350       return -1;
 351     }
 352 #endif
 353   }
 354
 355   else if (strcmp (key, "unix-socket-path") == 0 ||
 356            strcmp (key, "unix_socket_path") == 0)
 357     unix_socket_path = value;
 358
 359   else if (strcmp (key, "url") == 0)
 360     url = value;
 361
 362   else if (strcmp (key, "user") == 0)
 363     user = value;
 364
 365   else if (strcmp (key, "user-agent") == 0)
 366     user_agent = value;
 367
 368   else {
 369     nbdkit_error ("unknown parameter '%s'", key);
 370     return -1;
 371   }
 372
 373   return 0;
 374 }
 375
 376 /* Check the user did pass a url parameter. */
 377 static int
 378 curl_config_complete (void)
 379 {
 380   if (url == NULL) {
 381     nbdkit_error ("you must supply the url=<URL> parameter "
 382                   "after the plugin name on the command line");
 383     return -1;
 384   }
 385
 386   if (headers && header_script) {
 387     nbdkit_error ("header and header-script cannot be used at the same time");
 388     return -1;
 389   }
 390
 391   if (!header_script && header_script_renew) {
 392     nbdkit_error ("header-script-renew cannot be used without header-script");
 393     return -1;
 394   }
 395
 396   if (cookie && cookie_script) {
 397     nbdkit_error ("cookie and cookie-script cannot be used at the same time");
 398     return -1;
 399   }
 400
 401   if (!cookie_script && cookie_script_renew) {
 402     nbdkit_error ("cookie-script-renew cannot be used without cookie-script");
 403     return -1;
 404   }
 405
 406   return 0;
 407 }
 408
 409 #define curl_config_help \
 410   "cainfo=<CAINFO>            Path to Certificate Authority file.\n" \
 411   "capath=<CAPATH>            Path to directory with CA certificates.\n" \
 412   "cookie=<COOKIE>            Set HTTP/HTTPS cookies.\n" \
 413   "cookiefile=                Enable cookie processing.\n" \
 414   "cookiefile=<FILENAME>      Read cookies from file.\n" \
 415   "cookiejar=<FILENAME>       Read and write cookies to jar.\n" \
 416   "cookie-script=<SCRIPT>     Script to set HTTP/HTTPS cookies.\n" \
 417   "cookie-script-renew=<SECS> Time to renew HTTP/HTTPS cookies.\n" \
 418   "followlocation=false       Do not follow redirects.\n" \
 419   "header=<HEADER>            Set HTTP/HTTPS header.\n" \
 420   "header-script=<SCRIPT>     Script to set HTTP/HTTPS headers.\n" \
 421   "header-script-renew=<SECS> Time to renew HTTP/HTTPS headers.\n" \
 422   "password=<PASSWORD>        The password for the user account.\n" \
 423   "protocols=PROTO,PROTO,..   Limit protocols allowed.\n" \
 424   "proxy=<PROXY>              Set proxy URL.\n" \
 425   "proxy-password=<PASSWORD>  The proxy password.\n" \
 426   "proxy-user=<USER>          The proxy user.\n" \
 427   "timeout=<TIMEOUT>          Set the timeout for requests (seconds).\n" \
 428   "sslverify=false            Do not verify SSL certificate of remote host.\n" \
 429   "ssl-version=<VERSION>      Specify preferred TLS/SSL version.\n " \
 430   "ssl-cipher-list=C1:C2:..   Specify TLS/SSL cipher suites to be used.\n" \
 431   "tls13-ciphers=C1:C2:..     Specify TLS 1.3 cipher suites to be used.\n" \
 432   "tcp-keepalive=true         Enable TCP keepalives.\n" \
 433   "tcp-nodelay=false          Disable Nagle’s algorithm.\n" \
 434   "unix-socket-path=<PATH>    Open Unix domain socket instead of TCP/IP.\n" \
 435   "url=<URL>       (required) The disk image URL to serve.\n" \
 436   "user=<USER>                The user to log in as.\n" \
 437   "user-agent=<USER-AGENT>    Send user-agent header for HTTP/HTTPS."
 438
 439 /* Translate CURLcode to nbdkit_error. */
 440 #define display_curl_error(ch, r, fs, ...)                      \
 441   do {                                                          \
 442     nbdkit_error ((fs ": %s: %s"), ## __VA_ARGS__,              \
 443                   curl_easy_strerror ((r)), (ch)->errbuf);      \
 444   } while (0)
 445
 446 /* Create the per-connection handle. */
 447 static void *
 448 curl_open (int readonly)
 449 {
 450   struct handle *h;
 451
 452   h = calloc (1, sizeof *h);
 453   if (h == NULL) {
 454     nbdkit_error ("calloc: %m");
 455     return NULL;
 456   }
 457   h->readonly = readonly;
 458
 459   return h;
 460 }
 461
 462 /* Free up the per-connection handle. */
 463 static void
 464 curl_close (void *handle)
 465 {
 466   struct handle *h = handle;
 467
 468   free (h);
 469 }
 470
 471 #define THREAD_MODEL NBDKIT_THREAD_MODEL_SERIALIZE_REQUESTS
 472
 473 /* Calls get_handle() ... put_handle() to get a handle for the length
 474  * of the current scope.
 475  */
 476 #define GET_HANDLE_FOR_CURRENT_SCOPE(ch)                        \
 477   CLEANUP_PUT_HANDLE struct curl_handle *ch = get_handle ();
 478 #define CLEANUP_PUT_HANDLE __attribute__((cleanup (cleanup_put_handle)))
 479 static void
 480 cleanup_put_handle (void *chp)
 481 {
 482   struct curl_handle *ch = * (struct curl_handle **) chp;
 483
 484   if (ch != NULL)
 485     put_handle (ch);
 486 }
 487
 488 /* Get the file size. */
 489 static int64_t
 490 curl_get_size (void *handle)
 491 {
 492   GET_HANDLE_FOR_CURRENT_SCOPE (ch);
 493   if (ch == NULL)
 494     return -1;
 495
 496   return ch->exportsize;
 497 }
 498
 499 /* Multi-conn is safe for read-only connections, but HTTP does not
 500  * have any concept of flushing so we cannot use it for read-write
 501  * connections.
 502  */
 503 static int
 504 curl_can_multi_conn (void *handle)
 505 {
 506   struct handle *h = handle;
 507
 508   return !! h->readonly;
 509 }
 510
 511 /* Read data from the remote server. */
 512 static int
 513 curl_pread (void *handle, void *buf, uint32_t count, uint64_t offset)
 514 {
 515   CURLcode r;
 516   char range[128];
 517
 518   GET_HANDLE_FOR_CURRENT_SCOPE (ch);
 519   if (ch == NULL)
 520     return -1;
 521
 522   /* Run the scripts if necessary and set headers in the handle. */
 523   if (do_scripts (ch) == -1) return -1;
 524
 525   /* Tell the write_cb where we want the data to be written.  write_cb
 526    * will update this if the data comes in multiple sections.
 527    */
 528   ch->write_buf = buf;
 529   ch->write_count = count;
 530
 531   curl_easy_setopt (ch->c, CURLOPT_HTTPGET, 1L);
 532
 533   /* Make an HTTP range request. */
 534   snprintf (range, sizeof range, "%" PRIu64 "-%" PRIu64,
 535             offset, offset + count);
 536   curl_easy_setopt (ch->c, CURLOPT_RANGE, range);
 537
 538   /* The assumption here is that curl will look after timeouts. */
 539   r = curl_easy_perform (ch->c);
 540   if (r != CURLE_OK) {
 541     display_curl_error (ch, r, "pread: curl_easy_perform");
 542     return -1;
 543   }
 544
 545   /* Could use curl_easy_getinfo here to obtain further information
 546    * about the connection.
 547    */
 548
 549   /* As far as I understand the cURL API, this should never happen. */
 550   assert (ch->write_count == 0);
 551
 552   return 0;
 553 }
 554
 555 /* Write data to the remote server. */
 556 static int
 557 curl_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset)
 558 {
 559   CURLcode r;
 560   char range[128];
 561
 562   GET_HANDLE_FOR_CURRENT_SCOPE (ch);
 563   if (ch == NULL)
 564     return -1;
 565
 566   /* Run the scripts if necessary and set headers in the handle. */
 567   if (do_scripts (ch) == -1) return -1;
 568
 569   /* Tell the read_cb where we want the data to be read from.  read_cb
 570    * will update this if the data comes in multiple sections.
 571    */
 572   ch->read_buf = buf;
 573   ch->read_count = count;
 574
 575   curl_easy_setopt (ch->c, CURLOPT_UPLOAD, 1L);
 576
 577   /* Make an HTTP range request. */
 578   snprintf (range, sizeof range, "%" PRIu64 "-%" PRIu64,
 579             offset, offset + count);
 580   curl_easy_setopt (ch->c, CURLOPT_RANGE, range);
 581
 582   /* The assumption here is that curl will look after timeouts. */
 583   r = curl_easy_perform (ch->c);
 584   if (r != CURLE_OK) {
 585     display_curl_error (ch, r, "pwrite: curl_easy_perform");
 586     return -1;
 587   }
 588
 589   /* Could use curl_easy_getinfo here to obtain further information
 590    * about the connection.
 591    */
 592
 593   /* As far as I understand the cURL API, this should never happen. */
 594   assert (ch->read_count == 0);
 595
 596   return 0;
 597 }
 598
 599 static struct nbdkit_plugin plugin = {
 600   .name              = "curl",
 601   .version           = PACKAGE_VERSION,
 602   .load              = curl_load,
 603   .unload            = curl_unload,
 604   .config            = curl_config,
 605   .config_complete   = curl_config_complete,
 606   .config_help       = curl_config_help,
 607   .magic_config_key  = "url",
 608   .open              = curl_open,
 609   .close             = curl_close,
 610   .get_size          = curl_get_size,
 611   .can_multi_conn    = curl_can_multi_conn,
 612   .pread             = curl_pread,
 613   .pwrite            = curl_pwrite,
 614 };
 615
 616 NBDKIT_REGISTER_PLUGIN(plugin)