From 6c6b8c225ad3dfa07f20ca419063f5b1840bcade Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Sat, 4 Feb 2023 11:51:43 +0000 Subject: [PATCH] curl: Complete implementation of the curl handle pool This commit implements the curl handle pool. By default it can grow to up to 4 curl handles, shared between all NBD connections. You can change this using the new connections=N parameter. --- plugins/curl/curl.c | 11 +++++ plugins/curl/curldefs.h | 5 +- plugins/curl/nbdkit-curl-plugin.pod | 22 +++++++++ plugins/curl/pool.c | 97 +++++++++++++++++++++++++++++-------- 4 files changed, 113 insertions(+), 22 deletions(-) diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c index 105de29c..4634c700 100644 --- a/plugins/curl/curl.c +++ b/plugins/curl/curl.c @@ -57,6 +57,7 @@ const char *url = NULL; /* required */ const char *cainfo = NULL; const char *capath = NULL; +unsigned connections = 4; char *cookie = NULL; const char *cookiefile = NULL; const char *cookiejar = NULL; @@ -206,6 +207,16 @@ curl_config (const char *key, const char *value) capath = value; } + else if (strcmp (key, "connections") == 0) { + if (nbdkit_parse_unsigned ("connections", value, + &connections) == -1) + return -1; + if (connections == 0) { + nbdkit_error ("connections parameter must not be 0"); + return -1; + } + } + else if (strcmp (key, "cookie") == 0) { free (cookie); if (nbdkit_read_password (value, &cookie) == -1) diff --git a/plugins/curl/curldefs.h b/plugins/curl/curldefs.h index 533c28c5..d3c616f0 100644 --- a/plugins/curl/curldefs.h +++ b/plugins/curl/curldefs.h @@ -50,6 +50,7 @@ extern const char *url; extern const char *cainfo; extern const char *capath; +extern unsigned connections; extern char *cookie; extern const char *cookiefile; extern const char *cookiejar; @@ -84,7 +85,6 @@ extern int curl_debug_verbose; /* The per-connection handle. */ struct handle { int readonly; - struct curl_handle *ch; }; /* The libcurl handle and some associated fields and buffers. */ @@ -92,6 +92,9 @@ struct curl_handle { /* The underlying curl handle. */ CURL *c; + /* True if the handle is in use by a thread. */ + bool in_use; + /* These fields are used/initialized when we create the handle. */ bool accept_range; int64_t exportsize; diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod index 3c3d66a5..546f07ed 100644 --- a/plugins/curl/nbdkit-curl-plugin.pod +++ b/plugins/curl/nbdkit-curl-plugin.pod @@ -54,6 +54,15 @@ that libcurl is compiled with. Set CA certificates directory location for libcurl. See L for more information. +=item BN + +Open up to C curl connections to the web server. The default is 4. +Curl connections are shared between all NBD clients, so you may wish +to increase this if you expect many simultaneous NBD clients (or a +single client using many multi-conn connections). + +See L below. + =item BCOOKIE =item BFILENAME @@ -329,6 +338,19 @@ user-agent header. =back +=head1 NBD CONNECTIONS AND CURL HANDLES + +nbdkit E 1.32 used a simple model where a new NBD connection would +create a new libcurl handle. In practice this meant there was a +1-to-1 relationship between NBD connections and HTTP connections to +the remote web server (assuming http: or https: URL). + +nbdkit E 1.34 changed to using a fixed pool of libcurl handles +shared across all NBD connections. You can control the maximum number +of curl handles in the pool with the C parameter (default +4). Note that if there are more than 4 NBD connections, they will +share the 4 web server connections, unless you adjust C. + =head1 HEADER AND COOKIE SCRIPTS While the C
and C parameters can be used to specify diff --git a/plugins/curl/pool.c b/plugins/curl/pool.c index 00a7d381..de02fea5 100644 --- a/plugins/curl/pool.c +++ b/plugins/curl/pool.c @@ -55,6 +55,7 @@ #include "ascii-ctype.h" #include "ascii-string.h" #include "cleanup.h" +#include "vector.h" #include "curldefs.h" @@ -66,30 +67,41 @@ } while (0) static struct curl_handle *allocate_handle (void); +static void free_handle (struct curl_handle *); static int debug_cb (CURL *handle, curl_infotype type, const char *data, size_t size, void *); static size_t header_cb (void *ptr, size_t size, size_t nmemb, void *opaque); static size_t write_cb (char *ptr, size_t size, size_t nmemb, void *opaque); static size_t read_cb (void *ptr, size_t size, size_t nmemb, void *opaque); -/* In the current implementation there is only one handle. This lock - * prevents it from being used multiple times. - */ +/* This lock protects access to the curl_handles vector below. */ static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; -/* The single curl handle. NULL means not yet allocated. */ -static struct curl_handle *the_ch; +/* List of curl handles. This is allocated dynamically as more + * handles are requested. Currently it does not shrink. It may grow + * up to 'connections' in length. + */ +DEFINE_VECTOR_TYPE(curl_handle_list, struct curl_handle *) +static curl_handle_list curl_handles = empty_vector; + +/* The condition is used when the curl handles vector is full and + * we're waiting for a thread to put_handle. + */ +static pthread_cond_t cond = PTHREAD_COND_INITIALIZER; +static size_t in_use = 0, waiting = 0; /* Close and free all handles in the pool. */ void free_all_handles (void) { - if (the_ch) { - curl_easy_cleanup (the_ch->c); - if (the_ch->headers_copy) - curl_slist_free_all (the_ch->headers_copy); - free (the_ch); - } + size_t i; + + nbdkit_debug ("free_all_handles: number of curl handles allocated: %zu", + curl_handles.len); + + for (i = 0; i < curl_handles.len; ++i) + free_handle (curl_handles.ptr[i]); + curl_handle_list_reset (&curl_handles); } /* Get a handle from the pool. @@ -99,25 +111,59 @@ free_all_handles (void) struct curl_handle * get_handle (void) { - int r; - - r = pthread_mutex_lock (&lock); - assert (r == 0); - if (!the_ch) { - the_ch = allocate_handle (); - if (!the_ch) { - pthread_mutex_unlock (&lock); + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); + size_t i; + struct curl_handle *ch; + + again: + /* Look for a handle which is not in_use. */ + for (i = 0; i < curl_handles.len; ++i) { + ch = curl_handles.ptr[i]; + if (!ch->in_use) { + ch->in_use = true; + in_use++; + return ch; + } + } + + /* If more connections are allowed, then allocate a new handle. */ + if (curl_handles.len < connections) { + ch = allocate_handle (); + if (ch == NULL) + return NULL; + if (curl_handle_list_append (&curl_handles, ch) == -1) { + free_handle (ch); return NULL; } + ch->in_use = true; + in_use++; + return ch; } - return the_ch; + + /* Otherwise we have run out of connections so we must wait until + * another thread calls put_handle. + */ + assert (in_use == connections); + waiting++; + while (in_use == connections) + pthread_cond_wait (&cond, &lock); + waiting--; + + goto again; } /* Return the handle to the pool. */ void put_handle (struct curl_handle *ch) { - pthread_mutex_unlock (&lock); + ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lock); + + ch->in_use = false; + in_use--; + + /* Signal the next thread which is waiting. */ + if (waiting > 0) + pthread_cond_signal (&cond); } /* Allocate and initialize a new libcurl handle. */ @@ -484,3 +530,12 @@ read_cb (void *ptr, size_t size, size_t nmemb, void *opaque) return realsize; } + +static void +free_handle (struct curl_handle *ch) +{ + curl_easy_cleanup (ch->c); + if (ch->headers_copy) + curl_slist_free_all (ch->headers_copy); + free (ch); +} -- 2.11.4.GIT