plugins/curl/pool.c

   1 /* nbdkit
   2  * Copyright Red Hat
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions are
   6  * met:
   7  *
   8  * * Redistributions of source code must retain the above copyright
   9  * notice, this list of conditions and the following disclaimer.
  10  *
  11  * * Redistributions in binary form must reproduce the above copyright
  12  * notice, this list of conditions and the following disclaimer in the
  13  * documentation and/or other materials provided with the distribution.
  14  *
  15  * * Neither the name of Red Hat nor the names of its contributors may be
  16  * used to endorse or promote products derived from this software without
  17  * specific prior written permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  21  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
  22  * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
  23  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  26  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  27  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  28  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  29  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  30  * SUCH DAMAGE.
  31  */
  32
  33 /* Curl handle pool.
  34  *
  35  * To get a libcurl handle, call get_handle().  When you hold the
  36  * handle, it is yours exclusively to use.  After you have finished
  37  * with the handle, put it back into the pool by calling put_handle().
  38  */
  39
  40 #include <config.h>
  41
  42 #include <stdio.h>
  43 #include <stdlib.h>
  44 #include <stdbool.h>
  45 #include <stdint.h>
  46 #include <inttypes.h>
  47 #include <string.h>
  48 #include <assert.h>
  49 #include <pthread.h>
  50
  51 #include <curl/curl.h>
  52
  53 #include <nbdkit-plugin.h>
  54
  55 #include "ascii-ctype.h"
  56 #include "ascii-string.h"
  57 #include "cleanup.h"
  58 #include "vector.h"
  59
  60 #include "curldefs.h"
  61
  62 /* Use '-D curl.pool=1' to debug handle pool. */
  63 NBDKIT_DLL_PUBLIC int curl_debug_pool = 0;
  64
  65 /* Translate CURLcode to nbdkit_error. */
  66 #define display_curl_error(ch, r, fs, ...)                      \
  67   do {                                                          \
  68     nbdkit_error ((fs ": %s: %s"), ## __VA_ARGS__,              \
  69                   curl_easy_strerror ((r)), (ch)->errbuf);      \
  70   } while (0)
  71
  72 static struct curl_handle *allocate_handle (void);
  73 static void free_handle (struct curl_handle *);
  74 static int debug_cb (CURL *handle, curl_infotype type,
  75                      const char *data, size_t size, void *);
  76 static size_t header_cb (void *ptr, size_t size, size_t nmemb, void *opaque);
  77 static size_t write_cb (char *ptr, size_t size, size_t nmemb, void *opaque);
  78 static size_t read_cb (void *ptr, size_t size, size_t nmemb, void *opaque);
  79
  80 /* This lock protects access to the curl_handles vector below. */
  81 static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
  82
  83 /* List of curl handles.  This is allocated dynamically as more
  84  * handles are requested.  Currently it does not shrink.  It may grow
  85  * up to 'connections' in length.
  86  */
  87 DEFINE_VECTOR_TYPE (curl_handle_list, struct curl_handle *);
  88 static curl_handle_list curl_handles = empty_vector;
  89
  90 /* The condition is used when the curl handles vector is full and
  91  * we're waiting for a thread to put_handle.
  92  */
  93 static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
  94 static size_t in_use = 0, waiting = 0;
  95
  96 /* Close and free all handles in the pool. */
  97 void
  98 free_all_handles (void)
  99 {
 100   size_t i;
 101
 102   if (curl_debug_pool)
 103     nbdkit_debug ("free_all_handles: number of curl handles allocated: %zu",
 104                   curl_handles.len);
 105
 106   for (i = 0; i < curl_handles.len; ++i)
 107     free_handle (curl_handles.ptr[i]);
 108   curl_handle_list_reset (&curl_handles);
 109 }
 110
 111 /* Get a handle from the pool.
 112  *
 113  * It is owned exclusively by the caller until they call put_handle.
 114  */
 115 struct curl_handle *
 116 get_handle (void)
 117 {
 118   ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
 119   size_t i;
 120   struct curl_handle *ch;
 121
 122  again:
 123   /* Look for a handle which is not in_use. */
 124   for (i = 0; i < curl_handles.len; ++i) {
 125     ch = curl_handles.ptr[i];
 126     if (!ch->in_use) {
 127       ch->in_use = true;
 128       in_use++;
 129       if (curl_debug_pool)
 130         nbdkit_debug ("get_handle: %zu", ch->i);
 131       return ch;
 132     }
 133   }
 134
 135   /* If more connections are allowed, then allocate a new handle. */
 136   if (curl_handles.len < connections) {
 137     ch = allocate_handle ();
 138     if (ch == NULL)
 139       return NULL;
 140     if (curl_handle_list_append (&curl_handles, ch) == -1) {
 141       free_handle (ch);
 142       return NULL;
 143     }
 144     ch->i = curl_handles.len - 1;
 145     ch->in_use = true;
 146     in_use++;
 147     if (curl_debug_pool)
 148       nbdkit_debug ("get_handle: %zu", ch->i);
 149     return ch;
 150   }
 151
 152   /* Otherwise we have run out of connections so we must wait until
 153    * another thread calls put_handle.
 154    */
 155   assert (in_use == connections);
 156   waiting++;
 157   while (in_use == connections)
 158     pthread_cond_wait (&cond, &lock);
 159   waiting--;
 160
 161   goto again;
 162 }
 163
 164 /* Return the handle to the pool. */
 165 void
 166 put_handle (struct curl_handle *ch)
 167 {
 168   ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock);
 169
 170   if (curl_debug_pool)
 171     nbdkit_debug ("put_handle: %zu", ch->i);
 172
 173   ch->in_use = false;
 174   in_use--;
 175
 176   /* Signal the next thread which is waiting. */
 177   if (waiting > 0)
 178     pthread_cond_signal (&cond);
 179 }
 180
 181 /* Allocate and initialize a new libcurl handle. */
 182 static struct curl_handle *
 183 allocate_handle (void)
 184 {
 185   struct curl_handle *ch;
 186   CURLcode r;
 187 #ifdef HAVE_CURLINFO_CONTENT_LENGTH_DOWNLOAD_T
 188   curl_off_t o;
 189 #else
 190   double d;
 191 #endif
 192
 193   ch = calloc (1, sizeof *ch);
 194   if (ch == NULL) {
 195     nbdkit_error ("calloc: %m");
 196     free (ch);
 197     return NULL;
 198   }
 199
 200   ch->c = curl_easy_init ();
 201   if (ch->c == NULL) {
 202     nbdkit_error ("curl_easy_init: failed: %m");
 203     goto err;
 204   }
 205
 206   if (curl_debug_verbose) {
 207     /* NB: Constants must be explicitly long because the parameter is
 208      * varargs.
 209      */
 210     curl_easy_setopt (ch->c, CURLOPT_VERBOSE, 1L);
 211     curl_easy_setopt (ch->c, CURLOPT_DEBUGFUNCTION, debug_cb);
 212   }
 213
 214   curl_easy_setopt (ch->c, CURLOPT_ERRORBUFFER, ch->errbuf);
 215
 216   r = CURLE_OK;
 217   if (unix_socket_path) {
 218 #if HAVE_CURLOPT_UNIX_SOCKET_PATH
 219     r = curl_easy_setopt (ch->c, CURLOPT_UNIX_SOCKET_PATH, unix_socket_path);
 220 #else
 221     r = CURLE_UNKNOWN_OPTION;
 222 #endif
 223   }
 224   if (r != CURLE_OK) {
 225     display_curl_error (ch, r, "curl_easy_setopt: CURLOPT_UNIX_SOCKET_PATH");
 226     goto err;
 227   }
 228
 229   /* Set the URL. */
 230   r = curl_easy_setopt (ch->c, CURLOPT_URL, url);
 231   if (r != CURLE_OK) {
 232     display_curl_error (ch, r, "curl_easy_setopt: CURLOPT_URL [%s]", url);
 233     goto err;
 234   }
 235
 236   /* Various options we always set.
 237    *
 238    * NB: Both here and below constants must be explicitly long because
 239    * the parameter is varargs.
 240    *
 241    * For use of CURLOPT_NOSIGNAL see:
 242    * https://curl.se/libcurl/c/CURLOPT_NOSIGNAL.html
 243    */
 244   curl_easy_setopt (ch->c, CURLOPT_NOSIGNAL, 1L);
 245   curl_easy_setopt (ch->c, CURLOPT_AUTOREFERER, 1L);
 246   if (followlocation)
 247     curl_easy_setopt (ch->c, CURLOPT_FOLLOWLOCATION, 1L);
 248   curl_easy_setopt (ch->c, CURLOPT_FAILONERROR, 1L);
 249
 250   /* Options. */
 251   if (cainfo) {
 252     if (strlen (cainfo) == 0)
 253       curl_easy_setopt (ch->c, CURLOPT_CAINFO, NULL);
 254     else
 255       curl_easy_setopt (ch->c, CURLOPT_CAINFO, cainfo);
 256   }
 257   if (capath)
 258     curl_easy_setopt (ch->c, CURLOPT_CAPATH, capath);
 259   if (cookie)
 260     curl_easy_setopt (ch->c, CURLOPT_COOKIE, cookie);
 261   if (cookiefile)
 262     curl_easy_setopt (ch->c, CURLOPT_COOKIEFILE, cookiefile);
 263   if (cookiejar)
 264     curl_easy_setopt (ch->c, CURLOPT_COOKIEJAR, cookiejar);
 265   if (headers)
 266     curl_easy_setopt (ch->c, CURLOPT_HTTPHEADER, headers);
 267   if (http_version != CURL_HTTP_VERSION_NONE)
 268     curl_easy_setopt (ch->c, CURLOPT_HTTP_VERSION, (long) http_version);
 269
 270   if (password)
 271     curl_easy_setopt (ch->c, CURLOPT_PASSWORD, password);
 272 #ifndef HAVE_CURLOPT_PROTOCOLS_STR
 273   if (protocols != CURLPROTO_ALL) {
 274     curl_easy_setopt (ch->c, CURLOPT_PROTOCOLS, protocols);
 275     curl_easy_setopt (ch->c, CURLOPT_REDIR_PROTOCOLS, protocols);
 276   }
 277 #else /* HAVE_CURLOPT_PROTOCOLS_STR (new in 7.85.0) */
 278   if (protocols) {
 279     curl_easy_setopt (ch->c, CURLOPT_PROTOCOLS_STR, protocols);
 280     curl_easy_setopt (ch->c, CURLOPT_REDIR_PROTOCOLS_STR, protocols);
 281   }
 282 #endif /* HAVE_CURLOPT_PROTOCOLS_STR */
 283   if (proxy)
 284     curl_easy_setopt (ch->c, CURLOPT_PROXY, proxy);
 285   if (proxy_password)
 286     curl_easy_setopt (ch->c, CURLOPT_PROXYPASSWORD, proxy_password);
 287   if (proxy_user)
 288     curl_easy_setopt (ch->c, CURLOPT_PROXYUSERNAME, proxy_user);
 289   if (!sslverify) {
 290     curl_easy_setopt (ch->c, CURLOPT_SSL_VERIFYPEER, 0L);
 291     curl_easy_setopt (ch->c, CURLOPT_SSL_VERIFYHOST, 0L);
 292   }
 293   if (ssl_version != CURL_SSLVERSION_DEFAULT)
 294     curl_easy_setopt (ch->c, CURLOPT_SSLVERSION, (long) ssl_version);
 295   if (ssl_cipher_list)
 296     curl_easy_setopt (ch->c, CURLOPT_SSL_CIPHER_LIST, ssl_cipher_list);
 297   if (tls13_ciphers) {
 298 #if (LIBCURL_VERSION_MAJOR > 7) || \
 299     (LIBCURL_VERSION_MAJOR == 7 && LIBCURL_VERSION_MINOR >= 61)
 300     curl_easy_setopt (ch->c, CURLOPT_TLS13_CIPHERS, tls13_ciphers);
 301 #else
 302     /* This is not available before curl-7.61 */
 303     nbdkit_error ("tls13-ciphers is not supported in this build of "
 304                   "nbdkit-curl-plugin");
 305     goto err;
 306 #endif
 307   }
 308   if (tcp_keepalive)
 309     curl_easy_setopt (ch->c, CURLOPT_TCP_KEEPALIVE, 1L);
 310   if (!tcp_nodelay)
 311     curl_easy_setopt (ch->c, CURLOPT_TCP_NODELAY, 0L);
 312   if (timeout > 0)
 313     curl_easy_setopt (ch->c, CURLOPT_TIMEOUT, (long) timeout);
 314   if (user)
 315     curl_easy_setopt (ch->c, CURLOPT_USERNAME, user);
 316   if (user_agent)
 317     curl_easy_setopt (ch->c, CURLOPT_USERAGENT, user_agent);
 318
 319   /* Get the file size and also whether the remote HTTP server
 320    * supports byte ranges.
 321    *
 322    * We must run the scripts if necessary and set headers in the
 323    * handle.
 324    */
 325   if (do_scripts (ch) == -1) goto err;
 326   ch->accept_range = false;
 327   curl_easy_setopt (ch->c, CURLOPT_NOBODY, 1L); /* No Body, not nobody! */
 328   curl_easy_setopt (ch->c, CURLOPT_HEADERFUNCTION, header_cb);
 329   curl_easy_setopt (ch->c, CURLOPT_HEADERDATA, ch);
 330   r = curl_easy_perform (ch->c);
 331   if (r != CURLE_OK) {
 332     display_curl_error (ch, r,
 333                         "problem doing HEAD request to fetch size of URL [%s]",
 334                         url);
 335     goto err;
 336   }
 337
 338 #ifdef HAVE_CURLINFO_CONTENT_LENGTH_DOWNLOAD_T
 339   r = curl_easy_getinfo (ch->c, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &o);
 340   if (r != CURLE_OK) {
 341     display_curl_error (ch, r,
 342                         "could not get length of remote file [%s]", url);
 343     goto err;
 344   }
 345
 346   if (o == -1) {
 347     nbdkit_error ("could not get length of remote file [%s], "
 348                   "is the URL correct?", url);
 349     goto err;
 350   }
 351
 352   ch->exportsize = o;
 353 #else
 354   r = curl_easy_getinfo (ch->c, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &d);
 355   if (r != CURLE_OK) {
 356     display_curl_error (ch, r,
 357                         "could not get length of remote file [%s]", url);
 358     goto err;
 359   }
 360
 361   if (d == -1) {
 362     nbdkit_error ("could not get length of remote file [%s], "
 363                   "is the URL correct?", url);
 364     goto err;
 365   }
 366
 367   ch->exportsize = d;
 368 #endif
 369   nbdkit_debug ("content length: %" PRIi64, ch->exportsize);
 370
 371   if (ascii_strncasecmp (url, "http://", strlen ("http://")) == 0 ||
 372       ascii_strncasecmp (url, "https://", strlen ("https://")) == 0) {
 373     if (!ch->accept_range) {
 374       nbdkit_error ("server does not support 'range' (byte range) requests");
 375       goto err;
 376     }
 377
 378     nbdkit_debug ("accept range supported (for HTTP/HTTPS)");
 379   }
 380
 381   /* Get set up for reading and writing. */
 382   curl_easy_setopt (ch->c, CURLOPT_HEADERFUNCTION, NULL);
 383   curl_easy_setopt (ch->c, CURLOPT_HEADERDATA, NULL);
 384   curl_easy_setopt (ch->c, CURLOPT_WRITEFUNCTION, write_cb);
 385   curl_easy_setopt (ch->c, CURLOPT_WRITEDATA, ch);
 386   /* These are only used if !readonly but we always register them. */
 387   curl_easy_setopt (ch->c, CURLOPT_READFUNCTION, read_cb);
 388   curl_easy_setopt (ch->c, CURLOPT_READDATA, ch);
 389
 390   return ch;
 391
 392  err:
 393   if (ch->c)
 394     curl_easy_cleanup (ch->c);
 395   free (ch);
 396   return NULL;
 397 }
 398
 399 /* When using CURLOPT_VERBOSE, this callback is used to redirect
 400  * messages to nbdkit_debug (instead of stderr).
 401  */
 402 static int
 403 debug_cb (CURL *handle, curl_infotype type,
 404           const char *data, size_t size, void *opaque)
 405 {
 406   size_t origsize = size;
 407   CLEANUP_FREE char *str;
 408
 409   /* The data parameter passed is NOT \0-terminated, but also it may
 410    * have \n or \r\n line endings.  The only sane way to deal with
 411    * this is to copy the string.  (The data strings may also be
 412    * multi-line, but we don't deal with that here).
 413    */
 414   str = malloc (size + 1);
 415   if (str == NULL)
 416     goto out;
 417   memcpy (str, data, size);
 418   str[size] = '\0';
 419
 420   while (size > 0 && (str[size-1] == '\n' || str[size-1] == '\r')) {
 421     str[size-1] = '\0';
 422     size--;
 423   }
 424
 425   switch (type) {
 426   case CURLINFO_TEXT:
 427     nbdkit_debug ("%s", str);
 428     break;
 429   case CURLINFO_HEADER_IN:
 430     nbdkit_debug ("S: %s", str);
 431     break;
 432   case CURLINFO_HEADER_OUT:
 433     nbdkit_debug ("C: %s", str);
 434     break;
 435   default:
 436     /* Assume everything else is binary data that we cannot print. */
 437     nbdkit_debug ("<data with size=%zu>", origsize);
 438   }
 439
 440  out:
 441   return 0;
 442 }
 443
 444 static size_t
 445 header_cb (void *ptr, size_t size, size_t nmemb, void *opaque)
 446 {
 447   struct curl_handle *ch = opaque;
 448   size_t realsize = size * nmemb;
 449   const char *header = ptr;
 450   const char *end = header + realsize;
 451   const char *accept_ranges = "accept-ranges:";
 452   const char *bytes = "bytes";
 453
 454   if (realsize >= strlen (accept_ranges) &&
 455       ascii_strncasecmp (header, accept_ranges, strlen (accept_ranges)) == 0) {
 456     const char *p = strchr (header, ':') + 1;
 457
 458     /* Skip whitespace between the header name and value. */
 459     while (p < end && *p && ascii_isspace (*p))
 460       p++;
 461
 462     if (end - p >= strlen (bytes)
 463         && strncmp (p, bytes, strlen (bytes)) == 0) {
 464       /* Check that there is nothing but whitespace after the value. */
 465       p += strlen (bytes);
 466       while (p < end && *p && ascii_isspace (*p))
 467         p++;
 468
 469       if (p == end || !*p)
 470         ch->accept_range = true;
 471     }
 472   }
 473
 474   return realsize;
 475 }
 476
 477 /* NB: The terminology used by libcurl is confusing!
 478  *
 479  * WRITEFUNCTION / write_cb is used when reading from the remote server
 480  * READFUNCTION / read_cb is used when writing to the remote server.
 481  *
 482  * We use the same terminology as libcurl here.
 483  */
 484
 485 static size_t
 486 write_cb (char *ptr, size_t size, size_t nmemb, void *opaque)
 487 {
 488   struct curl_handle *ch = opaque;
 489   size_t orig_realsize = size * nmemb;
 490   size_t realsize = orig_realsize;
 491
 492   assert (ch->write_buf);
 493
 494   /* Don't read more than the requested amount of data, even if the
 495    * server or libcurl sends more.
 496    */
 497   if (realsize > ch->write_count)
 498     realsize = ch->write_count;
 499
 500   memcpy (ch->write_buf, ptr, realsize);
 501
 502   ch->write_count -= realsize;
 503   ch->write_buf += realsize;
 504
 505   return orig_realsize;
 506 }
 507
 508 static size_t
 509 read_cb (void *ptr, size_t size, size_t nmemb, void *opaque)
 510 {
 511   struct curl_handle *ch = opaque;
 512   size_t realsize = size * nmemb;
 513
 514   assert (ch->read_buf);
 515   if (realsize > ch->read_count)
 516     realsize = ch->read_count;
 517
 518   memcpy (ptr, ch->read_buf, realsize);
 519
 520   ch->read_count -= realsize;
 521   ch->read_buf += realsize;
 522
 523   return realsize;
 524 }
 525
 526 static void
 527 free_handle (struct curl_handle *ch)
 528 {
 529   curl_easy_cleanup (ch->c);
 530   if (ch->headers_copy)
 531     curl_slist_free_all (ch->headers_copy);
 532   free (ch);
 533 }