plugins/curl/pool.c

   1 /* nbdkit
   2  * Copyright (C) 2014-2023 Red Hat Inc.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions are
   6  * met:
   7  *
   8  * * Redistributions of source code must retain the above copyright
   9  * notice, this list of conditions and the following disclaimer.
  10  *
  11  * * Redistributions in binary form must reproduce the above copyright
  12  * notice, this list of conditions and the following disclaimer in the
  13  * documentation and/or other materials provided with the distribution.
  14  *
  15  * * Neither the name of Red Hat nor the names of its contributors may be
  16  * used to endorse or promote products derived from this software without
  17  * specific prior written permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  21  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
  22  * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
  23  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  26  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  27  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  28  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  29  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  30  * SUCH DAMAGE.
  31  */
  32
  33 /* Curl handle pool.
  34  *
  35  * To get a libcurl handle, call get_handle().  When you hold the
  36  * handle, it is yours exclusively to use.  After you have finished
  37  * with the handle, put it back into the pool by calling put_handle().
  38  */
  39
  40 #include <config.h>
  41
  42 #include <stdio.h>
  43 #include <stdlib.h>
  44 #include <stdbool.h>
  45 #include <stdint.h>
  46 #include <inttypes.h>
  47 #include <string.h>
  48 #include <assert.h>
  49 #include <pthread.h>
  50
  51 #include <curl/curl.h>
  52
  53 #include <nbdkit-plugin.h>
  54
  55 #include "ascii-ctype.h"
  56 #include "ascii-string.h"
  57 #include "cleanup.h"
  58
  59 #include "curldefs.h"
  60
  61 /* Translate CURLcode to nbdkit_error. */
  62 #define display_curl_error(ch, r, fs, ...)                      \
  63   do {                                                          \
  64     nbdkit_error ((fs ": %s: %s"), ## __VA_ARGS__,              \
  65                   curl_easy_strerror ((r)), (ch)->errbuf);      \
  66   } while (0)
  67
  68 static struct curl_handle *allocate_handle (void);
  69 static int debug_cb (CURL *handle, curl_infotype type,
  70                      const char *data, size_t size, void *);
  71 static size_t header_cb (void *ptr, size_t size, size_t nmemb, void *opaque);
  72 static size_t write_cb (char *ptr, size_t size, size_t nmemb, void *opaque);
  73 static size_t read_cb (void *ptr, size_t size, size_t nmemb, void *opaque);
  74
  75 /* In the current implementation there is only one handle.  This lock
  76  * prevents it from being used multiple times.
  77  */
  78 static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
  79
  80 /* The single curl handle.  NULL means not yet allocated. */
  81 static struct curl_handle *the_ch;
  82
  83 /* Close and free all handles in the pool. */
  84 void
  85 free_all_handles (void)
  86 {
  87   if (the_ch) {
  88     curl_easy_cleanup (the_ch->c);
  89     if (the_ch->headers_copy)
  90       curl_slist_free_all (the_ch->headers_copy);
  91     free (the_ch);
  92   }
  93 }
  94
  95 /* Get a handle from the pool.
  96  *
  97  * It is owned exclusively by the caller until they call put_handle.
  98  */
  99 struct curl_handle *
 100 get_handle (void)
 101 {
 102   int r;
 103
 104   r = pthread_mutex_lock (&lock);
 105   assert (r == 0);
 106   if (!the_ch) {
 107     the_ch = allocate_handle ();
 108     if (!the_ch) {
 109       pthread_mutex_unlock (&lock);
 110       return NULL;
 111     }
 112   }
 113   return the_ch;
 114 }
 115
 116 /* Return the handle to the pool. */
 117 void
 118 put_handle (struct curl_handle *ch)
 119 {
 120   pthread_mutex_unlock (&lock);
 121 }
 122
 123 /* Allocate and initialize a new libcurl handle. */
 124 static struct curl_handle *
 125 allocate_handle (void)
 126 {
 127   struct curl_handle *ch;
 128   CURLcode r;
 129 #ifdef HAVE_CURLINFO_CONTENT_LENGTH_DOWNLOAD_T
 130   curl_off_t o;
 131 #else
 132   double d;
 133 #endif
 134
 135   ch = calloc (1, sizeof *ch);
 136   if (ch == NULL) {
 137     nbdkit_error ("calloc: %m");
 138     free (ch);
 139     return NULL;
 140   }
 141
 142   ch->c = curl_easy_init ();
 143   if (ch->c == NULL) {
 144     nbdkit_error ("curl_easy_init: failed: %m");
 145     goto err;
 146   }
 147
 148   if (curl_debug_verbose) {
 149     /* NB: Constants must be explicitly long because the parameter is
 150      * varargs.
 151      */
 152     curl_easy_setopt (ch->c, CURLOPT_VERBOSE, 1L);
 153     curl_easy_setopt (ch->c, CURLOPT_DEBUGFUNCTION, debug_cb);
 154   }
 155
 156   curl_easy_setopt (ch->c, CURLOPT_ERRORBUFFER, ch->errbuf);
 157
 158   r = CURLE_OK;
 159   if (unix_socket_path) {
 160 #if HAVE_CURLOPT_UNIX_SOCKET_PATH
 161     r = curl_easy_setopt (ch->c, CURLOPT_UNIX_SOCKET_PATH, unix_socket_path);
 162 #else
 163     r = CURLE_UNKNOWN_OPTION;
 164 #endif
 165   }
 166   if (r != CURLE_OK) {
 167     display_curl_error (ch, r, "curl_easy_setopt: CURLOPT_UNIX_SOCKET_PATH");
 168     goto err;
 169   }
 170
 171   /* Set the URL. */
 172   r = curl_easy_setopt (ch->c, CURLOPT_URL, url);
 173   if (r != CURLE_OK) {
 174     display_curl_error (ch, r, "curl_easy_setopt: CURLOPT_URL [%s]", url);
 175     goto err;
 176   }
 177
 178   /* Various options we always set.
 179    *
 180    * NB: Both here and below constants must be explicitly long because
 181    * the parameter is varargs.
 182    *
 183    * For use of CURLOPT_NOSIGNAL see:
 184    * https://curl.se/libcurl/c/CURLOPT_NOSIGNAL.html
 185    */
 186   curl_easy_setopt (ch->c, CURLOPT_NOSIGNAL, 1L);
 187   curl_easy_setopt (ch->c, CURLOPT_AUTOREFERER, 1L);
 188   if (followlocation)
 189     curl_easy_setopt (ch->c, CURLOPT_FOLLOWLOCATION, 1L);
 190   curl_easy_setopt (ch->c, CURLOPT_FAILONERROR, 1L);
 191
 192   /* Options. */
 193   if (cainfo) {
 194     if (strlen (cainfo) == 0)
 195       curl_easy_setopt (ch->c, CURLOPT_CAINFO, NULL);
 196     else
 197       curl_easy_setopt (ch->c, CURLOPT_CAINFO, cainfo);
 198   }
 199   if (capath)
 200     curl_easy_setopt (ch->c, CURLOPT_CAPATH, capath);
 201   if (cookie)
 202     curl_easy_setopt (ch->c, CURLOPT_COOKIE, cookie);
 203   if (cookiefile)
 204     curl_easy_setopt (ch->c, CURLOPT_COOKIEFILE, cookiefile);
 205   if (cookiejar)
 206     curl_easy_setopt (ch->c, CURLOPT_COOKIEJAR, cookiejar);
 207   if (headers)
 208     curl_easy_setopt (ch->c, CURLOPT_HTTPHEADER, headers);
 209   if (password)
 210     curl_easy_setopt (ch->c, CURLOPT_PASSWORD, password);
 211 #ifndef HAVE_CURLOPT_PROTOCOLS_STR
 212   if (protocols != CURLPROTO_ALL) {
 213     curl_easy_setopt (ch->c, CURLOPT_PROTOCOLS, protocols);
 214     curl_easy_setopt (ch->c, CURLOPT_REDIR_PROTOCOLS, protocols);
 215   }
 216 #else /* HAVE_CURLOPT_PROTOCOLS_STR (new in 7.85.0) */
 217   if (protocols) {
 218     curl_easy_setopt (ch->c, CURLOPT_PROTOCOLS_STR, protocols);
 219     curl_easy_setopt (ch->c, CURLOPT_REDIR_PROTOCOLS_STR, protocols);
 220   }
 221 #endif /* HAVE_CURLOPT_PROTOCOLS_STR */
 222   if (proxy)
 223     curl_easy_setopt (ch->c, CURLOPT_PROXY, proxy);
 224   if (proxy_password)
 225     curl_easy_setopt (ch->c, CURLOPT_PROXYPASSWORD, proxy_password);
 226   if (proxy_user)
 227     curl_easy_setopt (ch->c, CURLOPT_PROXYUSERNAME, proxy_user);
 228   if (!sslverify) {
 229     curl_easy_setopt (ch->c, CURLOPT_SSL_VERIFYPEER, 0L);
 230     curl_easy_setopt (ch->c, CURLOPT_SSL_VERIFYHOST, 0L);
 231   }
 232   if (ssl_version) {
 233     if (strcmp (ssl_version, "tlsv1") == 0)
 234       curl_easy_setopt (ch->c, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
 235     else if (strcmp (ssl_version, "sslv2") == 0)
 236       curl_easy_setopt (ch->c, CURLOPT_SSLVERSION, CURL_SSLVERSION_SSLv2);
 237     else if (strcmp (ssl_version, "sslv3") == 0)
 238       curl_easy_setopt (ch->c, CURLOPT_SSLVERSION, CURL_SSLVERSION_SSLv3);
 239     else if (strcmp (ssl_version, "tlsv1.0") == 0)
 240       curl_easy_setopt (ch->c, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_0);
 241     else if (strcmp (ssl_version, "tlsv1.1") == 0)
 242       curl_easy_setopt (ch->c, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_1);
 243     else if (strcmp (ssl_version, "tlsv1.2") == 0)
 244       curl_easy_setopt (ch->c, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
 245     else if (strcmp (ssl_version, "tlsv1.3") == 0)
 246       curl_easy_setopt (ch->c, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_3);
 247     else {
 248       display_curl_error (ch, r, "curl_easy_setopt: CURLOPT_SSLVERSION [%s]",
 249                           ssl_version);
 250       goto err;
 251     }
 252
 253   }
 254   if (ssl_cipher_list)
 255     curl_easy_setopt (ch->c, CURLOPT_SSL_CIPHER_LIST, ssl_cipher_list);
 256   if (tls13_ciphers) {
 257 #if (LIBCURL_VERSION_MAJOR > 7) || \
 258     (LIBCURL_VERSION_MAJOR == 7 && LIBCURL_VERSION_MINOR >= 61)
 259     curl_easy_setopt (ch->c, CURLOPT_TLS13_CIPHERS, tls13_ciphers);
 260 #else
 261     /* This is not available before curl-7.61 */
 262     nbdkit_error ("tls13-ciphers is not supported in this build of "
 263                   "nbdkit-curl-plugin");
 264     goto err;
 265 #endif
 266   }
 267   if (tcp_keepalive)
 268     curl_easy_setopt (ch->c, CURLOPT_TCP_KEEPALIVE, 1L);
 269   if (!tcp_nodelay)
 270     curl_easy_setopt (ch->c, CURLOPT_TCP_NODELAY, 0L);
 271   if (timeout > 0)
 272     /* NB: The cast is required here because the parameter is varargs
 273      * treated as long, and not type safe.
 274      */
 275     curl_easy_setopt (ch->c, CURLOPT_TIMEOUT, (long) timeout);
 276   if (user)
 277     curl_easy_setopt (ch->c, CURLOPT_USERNAME, user);
 278   if (user_agent)
 279     curl_easy_setopt (ch->c, CURLOPT_USERAGENT, user_agent);
 280
 281   /* Get the file size and also whether the remote HTTP server
 282    * supports byte ranges.
 283    *
 284    * We must run the scripts if necessary and set headers in the
 285    * handle.
 286    */
 287   if (do_scripts (ch) == -1) goto err;
 288   ch->accept_range = false;
 289   curl_easy_setopt (ch->c, CURLOPT_NOBODY, 1L); /* No Body, not nobody! */
 290   curl_easy_setopt (ch->c, CURLOPT_HEADERFUNCTION, header_cb);
 291   curl_easy_setopt (ch->c, CURLOPT_HEADERDATA, ch);
 292   r = curl_easy_perform (ch->c);
 293   if (r != CURLE_OK) {
 294     display_curl_error (ch, r,
 295                         "problem doing HEAD request to fetch size of URL [%s]",
 296                         url);
 297     goto err;
 298   }
 299
 300 #ifdef HAVE_CURLINFO_CONTENT_LENGTH_DOWNLOAD_T
 301   r = curl_easy_getinfo (ch->c, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &o);
 302   if (r != CURLE_OK) {
 303     display_curl_error (ch, r,
 304                         "could not get length of remote file [%s]", url);
 305     goto err;
 306   }
 307
 308   if (o == -1) {
 309     nbdkit_error ("could not get length of remote file [%s], "
 310                   "is the URL correct?", url);
 311     goto err;
 312   }
 313
 314   ch->exportsize = o;
 315 #else
 316   r = curl_easy_getinfo (ch->c, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &d);
 317   if (r != CURLE_OK) {
 318     display_curl_error (ch, r,
 319                         "could not get length of remote file [%s]", url);
 320     goto err;
 321   }
 322
 323   if (d == -1) {
 324     nbdkit_error ("could not get length of remote file [%s], "
 325                   "is the URL correct?", url);
 326     goto err;
 327   }
 328
 329   ch->exportsize = d;
 330 #endif
 331   nbdkit_debug ("content length: %" PRIi64, ch->exportsize);
 332
 333   if (ascii_strncasecmp (url, "http://", strlen ("http://")) == 0 ||
 334       ascii_strncasecmp (url, "https://", strlen ("https://")) == 0) {
 335     if (!ch->accept_range) {
 336       nbdkit_error ("server does not support 'range' (byte range) requests");
 337       goto err;
 338     }
 339
 340     nbdkit_debug ("accept range supported (for HTTP/HTTPS)");
 341   }
 342
 343   /* Get set up for reading and writing. */
 344   curl_easy_setopt (ch->c, CURLOPT_HEADERFUNCTION, NULL);
 345   curl_easy_setopt (ch->c, CURLOPT_HEADERDATA, NULL);
 346   curl_easy_setopt (ch->c, CURLOPT_WRITEFUNCTION, write_cb);
 347   curl_easy_setopt (ch->c, CURLOPT_WRITEDATA, ch);
 348   /* These are only used if !readonly but we always register them. */
 349   curl_easy_setopt (ch->c, CURLOPT_READFUNCTION, read_cb);
 350   curl_easy_setopt (ch->c, CURLOPT_READDATA, ch);
 351
 352   return ch;
 353
 354  err:
 355   if (ch->c)
 356     curl_easy_cleanup (ch->c);
 357   free (ch);
 358   return NULL;
 359 }
 360
 361 /* When using CURLOPT_VERBOSE, this callback is used to redirect
 362  * messages to nbdkit_debug (instead of stderr).
 363  */
 364 static int
 365 debug_cb (CURL *handle, curl_infotype type,
 366           const char *data, size_t size, void *opaque)
 367 {
 368   size_t origsize = size;
 369   CLEANUP_FREE char *str;
 370
 371   /* The data parameter passed is NOT \0-terminated, but also it may
 372    * have \n or \r\n line endings.  The only sane way to deal with
 373    * this is to copy the string.  (The data strings may also be
 374    * multi-line, but we don't deal with that here).
 375    */
 376   str = malloc (size + 1);
 377   if (str == NULL)
 378     goto out;
 379   memcpy (str, data, size);
 380   str[size] = '\0';
 381
 382   while (size > 0 && (str[size-1] == '\n' || str[size-1] == '\r')) {
 383     str[size-1] = '\0';
 384     size--;
 385   }
 386
 387   switch (type) {
 388   case CURLINFO_TEXT:
 389     nbdkit_debug ("%s", str);
 390     break;
 391   case CURLINFO_HEADER_IN:
 392     nbdkit_debug ("S: %s", str);
 393     break;
 394   case CURLINFO_HEADER_OUT:
 395     nbdkit_debug ("C: %s", str);
 396     break;
 397   default:
 398     /* Assume everything else is binary data that we cannot print. */
 399     nbdkit_debug ("<data with size=%zu>", origsize);
 400   }
 401
 402  out:
 403   return 0;
 404 }
 405
 406 static size_t
 407 header_cb (void *ptr, size_t size, size_t nmemb, void *opaque)
 408 {
 409   struct curl_handle *ch = opaque;
 410   size_t realsize = size * nmemb;
 411   const char *header = ptr;
 412   const char *end = header + realsize;
 413   const char *accept_ranges = "accept-ranges:";
 414   const char *bytes = "bytes";
 415
 416   if (realsize >= strlen (accept_ranges) &&
 417       ascii_strncasecmp (header, accept_ranges, strlen (accept_ranges)) == 0) {
 418     const char *p = strchr (header, ':') + 1;
 419
 420     /* Skip whitespace between the header name and value. */
 421     while (p < end && *p && ascii_isspace (*p))
 422       p++;
 423
 424     if (end - p >= strlen (bytes)
 425         && strncmp (p, bytes, strlen (bytes)) == 0) {
 426       /* Check that there is nothing but whitespace after the value. */
 427       p += strlen (bytes);
 428       while (p < end && *p && ascii_isspace (*p))
 429         p++;
 430
 431       if (p == end || !*p)
 432         ch->accept_range = true;
 433     }
 434   }
 435
 436   return realsize;
 437 }
 438
 439 /* NB: The terminology used by libcurl is confusing!
 440  *
 441  * WRITEFUNCTION / write_cb is used when reading from the remote server
 442  * READFUNCTION / read_cb is used when writing to the remote server.
 443  *
 444  * We use the same terminology as libcurl here.
 445  */
 446
 447 static size_t
 448 write_cb (char *ptr, size_t size, size_t nmemb, void *opaque)
 449 {
 450   struct curl_handle *ch = opaque;
 451   size_t orig_realsize = size * nmemb;
 452   size_t realsize = orig_realsize;
 453
 454   assert (ch->write_buf);
 455
 456   /* Don't read more than the requested amount of data, even if the
 457    * server or libcurl sends more.
 458    */
 459   if (realsize > ch->write_count)
 460     realsize = ch->write_count;
 461
 462   memcpy (ch->write_buf, ptr, realsize);
 463
 464   ch->write_count -= realsize;
 465   ch->write_buf += realsize;
 466
 467   return orig_realsize;
 468 }
 469
 470 static size_t
 471 read_cb (void *ptr, size_t size, size_t nmemb, void *opaque)
 472 {
 473   struct curl_handle *ch = opaque;
 474   size_t realsize = size * nmemb;
 475
 476   assert (ch->read_buf);
 477   if (realsize > ch->read_count)
 478     realsize = ch->read_count;
 479
 480   memcpy (ptr, ch->read_buf, realsize);
 481
 482   ch->read_count -= realsize;
 483   ch->read_buf += realsize;
 484
 485   return realsize;
 486 }