2 * Copyright (C) 2017-2019 Red Hat Inc.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are
8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
11 * * Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * * Neither the name of Red Hat nor the names of its contributors may be
16 * used to endorse or promote products derived from this software without
17 * specific prior written permission.
19 * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
22 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
26 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
27 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
29 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
45 #include <netinet/in.h>
46 #include <netinet/tcp.h>
47 #include <sys/socket.h>
51 #include <semaphore.h>
56 #define NBDKIT_API_VERSION 2
58 #include <nbdkit-plugin.h>
59 #include "byte-swapping.h"
62 /* The per-transaction details */
67 struct transaction
*next
;
70 /* The per-connection handle */
72 /* These fields are read-only once initialized */
73 struct nbd_handle
*nbd
;
74 int fd
; /* Cache of nbd_aio_get_fd */
75 int fds
[2]; /* Pipe for kicking the reader thread */
79 pthread_mutex_t trans_lock
; /* Covers access to trans list */
80 struct transaction
*trans
; /* List of pending transactions */
83 /* Connect to server via absolute name of Unix socket */
84 static char *sockname
;
86 /* Connect to server via TCP socket */
87 static const char *hostname
;
88 static const char *port
;
90 /* Connect to server via URI */
91 static const char *uri
;
93 /* Name of export on remote server, default '', ignored for oldstyle */
94 static const char *export
;
96 /* Number of retries */
97 static unsigned long retry
;
99 /* True to share single server connection among all clients */
101 static struct handle
*shared_handle
;
103 /* Control TLS settings */
105 static char *tls_certificates
;
106 static int tls_verify
= -1;
107 static const char *tls_username
;
108 static char *tls_psk
;
110 static struct handle
*nbdplug_open_handle (int readonly
);
111 static void nbdplug_close_handle (struct handle
*h
);
114 nbdplug_unload (void)
117 nbdplug_close_handle (shared_handle
);
119 free (tls_certificates
);
123 /* Called for each key=value passed on the command line. This plugin
124 * accepts socket=<sockname>, hostname=<hostname>/port=<port>, or
125 * [uri=]<uri> (exactly one connection required), and optional
126 * parameters export=<name>, retry=<n>, shared=<bool> and various
130 nbdplug_config (const char *key
, const char *value
)
135 if (strcmp (key
, "socket") == 0) {
136 /* See FILENAMES AND PATHS in nbdkit-plugin(3) */
138 sockname
= nbdkit_absolute_path (value
);
142 else if (strcmp (key
, "hostname") == 0)
144 else if (strcmp (key
, "port") == 0)
146 else if (strcmp (key
, "uri") == 0)
148 else if (strcmp (key
, "export") == 0)
150 else if (strcmp (key
, "retry") == 0) {
152 retry
= strtoul (value
, &end
, 0);
153 if (value
== end
|| errno
) {
154 nbdkit_error ("could not parse retry as integer (%s)", value
);
158 else if (strcmp (key
, "shared") == 0) {
159 r
= nbdkit_parse_bool (value
);
164 else if (strcmp (key
, "tls") == 0) {
165 if (strcasecmp (value
, "require") == 0 ||
166 strcasecmp (value
, "required") == 0 ||
167 strcasecmp (value
, "force") == 0)
170 tls
= nbdkit_parse_bool (value
);
175 else if (strcmp (key
, "tls-certificates") == 0) {
176 free (tls_certificates
);
177 tls_certificates
= nbdkit_absolute_path (value
);
178 if (!tls_certificates
)
181 else if (strcmp (key
, "tls-verify") == 0) {
182 r
= nbdkit_parse_bool (value
);
187 else if (strcmp (key
, "tls-username") == 0)
188 tls_username
= value
;
189 else if (strcmp (key
, "tls-psk") == 0) {
191 tls_psk
= nbdkit_absolute_path (value
);
196 nbdkit_error ("unknown parameter '%s'", key
);
203 /* Check the user passed exactly one socket description. */
205 nbdplug_config_complete (void)
208 struct sockaddr_un sock
;
210 if (hostname
|| port
) {
211 nbdkit_error ("cannot mix Unix socket and TCP hostname/port parameters");
215 nbdkit_error ("cannot mix Unix socket and URI parameters");
218 if (strlen (sockname
) > sizeof sock
.sun_path
) {
219 nbdkit_error ("socket file name too large");
225 nbdkit_error ("cannot mix TCP hostname/port and URI parameters");
232 struct nbd_handle
*nbd
= nbd_create ();
235 nbdkit_error ("unable to query libnbd details: %s", nbd_get_error ());
238 if (!nbd_supports_uri (nbd
)) {
239 nbdkit_error ("libnbd was compiled without uri support");
245 nbdkit_error ("must supply socket=, hostname= or uri= of external NBD server");
253 tls
= tls_certificates
|| tls_verify
>= 0 || tls_username
|| tls_psk
;
255 struct nbd_handle
*nbd
= nbd_create ();
258 nbdkit_error ("unable to query libnbd details: %s", nbd_get_error ());
261 if (!nbd_supports_tls (nbd
)) {
262 nbdkit_error ("libnbd was compiled without tls support");
269 if (shared
&& (shared_handle
= nbdplug_open_handle (false)) == NULL
)
274 #define nbdplug_config_help \
275 "[uri=]<URI> URI of an NBD socket to connect to (if supported).\n" \
276 "socket=<SOCKNAME> The Unix socket to connect to.\n" \
277 "hostname=<HOST> The hostname for the TCP socket to connect to.\n" \
278 "port=<PORT> TCP port or service name to use (default 10809).\n" \
279 "export=<NAME> Export name to connect to (default \"\").\n" \
280 "retry=<N> Retry connection up to N seconds (default 0).\n" \
281 "shared=<BOOL> True to share one server connection among all clients,\n" \
282 " rather than a connection per client (default false).\n" \
283 "tls=<MODE> How to use TLS; one of 'off', 'on', or 'require'.\n" \
284 "tls-certificates=<DIR> Directory containing files for X.509 certificates.\n" \
285 "tls-verify=<BOOL> True (default for X.509) to validate server.\n" \
286 "tls-username=<NAME> Override username presented in X.509 TLS.\n" \
287 "tls-psk=<FILE> File containing Pre-Shared Key for TLS.\n" \
290 nbdplug_dump_plugin (void)
292 struct nbd_handle
*nbd
= nbd_create ();
295 nbdkit_error ("unable to query libnbd details: %s", nbd_get_error ());
298 printf ("libnbd_version=%s\n", nbd_get_version (nbd
));
299 printf ("libnbd_tls=%d\n", nbd_supports_tls (nbd
));
300 printf ("libnbd_uri=%d\n", nbd_supports_uri (nbd
));
304 #define THREAD_MODEL NBDKIT_THREAD_MODEL_PARALLEL
308 nbdplug_reader (void *handle
)
310 struct handle
*h
= handle
;
313 while (!nbd_aio_is_dead (h
->nbd
) && !nbd_aio_is_closed (h
->nbd
)) {
314 struct pollfd fds
[2] = {
319 struct transaction
*trans
, **prev
;
323 dir
= nbd_aio_get_direction (h
->nbd
);
324 nbdkit_debug ("polling, dir=%d", dir
);
325 if (dir
& LIBNBD_AIO_DIRECTION_READ
)
326 fds
[0].events
|= POLLIN
;
327 if (dir
& LIBNBD_AIO_DIRECTION_WRITE
)
328 fds
[0].events
|= POLLOUT
;
329 if (poll (fds
, 2, -1) == -1) {
330 nbdkit_error ("poll: %m");
334 if (dir
& LIBNBD_AIO_DIRECTION_READ
&& fds
[0].revents
& POLLIN
)
335 nbd_aio_notify_read (h
->nbd
);
336 else if (dir
& LIBNBD_AIO_DIRECTION_WRITE
&& fds
[0].revents
& POLLOUT
)
337 nbd_aio_notify_write (h
->nbd
);
339 /* Check if we were kicked because a command was started */
340 if (fds
[1].revents
& POLLIN
&& read (h
->fds
[0], &c
, 1) != 1) {
341 nbdkit_error ("failed to read pipe: %m");
345 ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&h
->trans_lock
);
349 r
= nbd_aio_command_completed (h
->nbd
, trans
->cookie
);
351 nbdkit_debug ("transaction %" PRId64
" failed: %s", trans
->cookie
,
353 trans
->err
= nbd_get_errno ();
358 nbdkit_debug ("cookie %" PRId64
" completed state machine, status %d",
359 trans
->cookie
, trans
->err
);
361 if (sem_post (&trans
->sem
)) {
362 nbdkit_error ("failed to post semaphore: %m");
372 /* Clean up any stranded in-flight requests */
373 nbdkit_debug ("state machine changed to %s", nbd_connection_state (h
->nbd
));
375 struct transaction
*trans
;
378 ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&h
->trans_lock
);
380 h
->trans
= trans
? trans
->next
: NULL
;
384 r
= nbd_aio_command_completed (h
->nbd
, trans
->cookie
);
386 nbdkit_debug ("transaction %" PRId64
" failed: %s", trans
->cookie
,
388 trans
->err
= nbd_get_errno ();
390 trans
->err
= ESHUTDOWN
;
393 trans
->err
= ESHUTDOWN
;
394 if (sem_post (&trans
->sem
)) {
395 nbdkit_error ("failed to post semaphore: %m");
399 nbdkit_debug ("exiting state machine thread");
403 /* Register a cookie and return a transaction. */
404 static struct transaction
*
405 nbdplug_register (struct handle
*h
, int64_t cookie
)
407 struct transaction
*trans
;
411 nbdkit_error ("command failed: %s", nbd_get_error ());
412 errno
= nbd_get_errno ();
416 nbdkit_debug ("cookie %" PRId64
" started by state machine", cookie
);
417 trans
= calloc (1, sizeof *trans
);
419 nbdkit_error ("unable to track transaction: %m");
423 /* While locked, kick the reader thread and add our transaction */
424 ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&h
->trans_lock
);
425 if (write (h
->fds
[1], &c
, 1) != 1) {
426 nbdkit_error ("write to pipe: %m");
430 if (sem_init (&trans
->sem
, 0, 0)) {
431 nbdkit_error ("unable to create semaphore: %m");
435 trans
->cookie
= cookie
;
436 trans
->next
= h
->trans
;
441 /* Perform the reply half of a transaction. */
443 nbdplug_reply (struct handle
*h
, struct transaction
*trans
)
452 while ((err
= sem_wait (&trans
->sem
)) == -1 && errno
== EINTR
)
455 nbdkit_debug ("failed to wait on semaphore: %m");
460 if (sem_destroy (&trans
->sem
))
467 /* Create the shared or per-connection handle. */
468 static struct handle
*
469 nbdplug_open_handle (int readonly
)
474 h
= calloc (1, sizeof *h
);
476 nbdkit_error ("malloc: %m");
480 nbdkit_error ("pipe: %m");
487 h
->nbd
= nbd_create ();
490 if (nbd_set_export_name (h
->nbd
, export
) == -1)
492 if (nbd_add_meta_context (h
->nbd
, "base:allocation") == -1)
494 if (nbd_set_tls (h
->nbd
, tls
) == -1)
496 if (tls_certificates
&&
497 nbd_set_tls_certificates (h
->nbd
, tls_certificates
) == -1)
499 if (tls_verify
>= 0 && nbd_set_tls_verify_peer (h
->nbd
, tls_verify
) == -1)
501 if (tls_username
&& nbd_set_tls_username (h
->nbd
, tls_username
) == -1)
503 if (tls_psk
&& nbd_set_tls_psk_file (h
->nbd
, tls_psk
) == -1)
506 r
= nbd_connect_uri (h
->nbd
, uri
);
508 r
= nbd_connect_unix (h
->nbd
, sockname
);
510 r
= nbd_connect_tcp (h
->nbd
, hostname
, port
);
513 nbdkit_debug ("connect failed; will try again: %s", nbd_get_error ());
520 h
->fd
= nbd_aio_get_fd (h
->nbd
);
527 /* Spawn a dedicated reader thread */
528 if ((errno
= pthread_mutex_init (&h
->trans_lock
, NULL
))) {
529 nbdkit_error ("failed to initialize transaction mutex: %m");
532 if ((errno
= pthread_create (&h
->reader
, NULL
, nbdplug_reader
, h
))) {
533 nbdkit_error ("failed to initialize reader thread: %m");
534 pthread_mutex_destroy (&h
->trans_lock
);
543 nbdkit_error ("failure while creating nbd handle: %s", nbd_get_error ());
550 /* Create the per-connection handle. */
552 nbdplug_open (int readonly
)
555 return shared_handle
;
556 return nbdplug_open_handle (readonly
);
559 /* Free up the shared or per-connection handle. */
561 nbdplug_close_handle (struct handle
*h
)
563 if (nbd_shutdown (h
->nbd
) == -1)
564 nbdkit_debug ("failed to clean up handle: %s", nbd_get_error ());
565 if ((errno
= pthread_join (h
->reader
, NULL
)))
566 nbdkit_debug ("failed to join reader thread: %m");
570 pthread_mutex_destroy (&h
->trans_lock
);
574 /* Free up the per-connection handle. */
576 nbdplug_close (void *handle
)
578 struct handle
*h
= handle
;
581 nbdplug_close_handle (h
);
586 /* Get the file size. */
588 nbdplug_get_size (void *handle
)
590 struct handle
*h
= handle
;
591 int64_t size
= nbd_get_size (h
->nbd
);
594 nbdkit_error ("failure to get size: %s", nbd_get_error ());
601 nbdplug_can_write (void *handle
)
603 struct handle
*h
= handle
;
604 int i
= nbd_read_only (h
->nbd
);
607 nbdkit_error ("failure to check readonly flag: %s", nbd_get_error ());
610 return !(i
|| h
->readonly
);
614 nbdplug_can_flush (void *handle
)
616 struct handle
*h
= handle
;
617 int i
= nbd_can_flush (h
->nbd
);
620 nbdkit_error ("failure to check flush flag: %s", nbd_get_error ());
627 nbdplug_is_rotational (void *handle
)
629 struct handle
*h
= handle
;
630 int i
= nbd_is_rotational (h
->nbd
);
633 nbdkit_error ("failure to check rotational flag: %s", nbd_get_error ());
640 nbdplug_can_trim (void *handle
)
642 struct handle
*h
= handle
;
643 int i
= nbd_can_trim (h
->nbd
);
646 nbdkit_error ("failure to check trim flag: %s", nbd_get_error ());
653 nbdplug_can_zero (void *handle
)
655 struct handle
*h
= handle
;
656 int i
= nbd_can_zero (h
->nbd
);
659 nbdkit_error ("failure to check zero flag: %s", nbd_get_error ());
666 nbdplug_can_fua (void *handle
)
668 struct handle
*h
= handle
;
669 int i
= nbd_can_fua (h
->nbd
);
672 nbdkit_error ("failure to check fua flag: %s", nbd_get_error ());
675 return i
? NBDKIT_FUA_NATIVE
: NBDKIT_FUA_NONE
;
679 nbdplug_can_multi_conn (void *handle
)
681 struct handle
*h
= handle
;
682 int i
= nbd_can_multi_conn (h
->nbd
);
685 nbdkit_error ("failure to check multi-conn flag: %s", nbd_get_error ());
692 nbdplug_can_cache (void *handle
)
694 struct handle
*h
= handle
;
695 int i
= nbd_can_cache (h
->nbd
);
698 nbdkit_error ("failure to check cache flag: %s", nbd_get_error ());
701 return i
? NBDKIT_CACHE_NATIVE
: NBDKIT_CACHE_NONE
;
705 nbdplug_can_extents (void *handle
)
707 struct handle
*h
= handle
;
708 int i
= nbd_can_meta_context (h
->nbd
, "base:allocation");
711 nbdkit_error ("failure to check extents ability: %s", nbd_get_error ());
717 /* Read data from the file. */
719 nbdplug_pread (void *handle
, void *buf
, uint32_t count
, uint64_t offset
,
722 struct handle
*h
= handle
;
723 struct transaction
*s
;
726 s
= nbdplug_register (h
, nbd_aio_pread (h
->nbd
, buf
, count
, offset
, 0));
727 return nbdplug_reply (h
, s
);
730 /* Write data to the file. */
732 nbdplug_pwrite (void *handle
, const void *buf
, uint32_t count
, uint64_t offset
,
735 struct handle
*h
= handle
;
736 struct transaction
*s
;
737 uint32_t f
= flags
& NBDKIT_FLAG_FUA
? LIBNBD_CMD_FLAG_FUA
: 0;
739 assert (!(flags
& ~NBDKIT_FLAG_FUA
));
740 s
= nbdplug_register (h
, nbd_aio_pwrite (h
->nbd
, buf
, count
, offset
, f
));
741 return nbdplug_reply (h
, s
);
744 /* Write zeroes to the file. */
746 nbdplug_zero (void *handle
, uint32_t count
, uint64_t offset
, uint32_t flags
)
748 struct handle
*h
= handle
;
749 struct transaction
*s
;
752 assert (!(flags
& ~(NBDKIT_FLAG_FUA
| NBDKIT_FLAG_MAY_TRIM
)));
754 if (!(flags
& NBDKIT_FLAG_MAY_TRIM
))
755 f
|= LIBNBD_CMD_FLAG_NO_HOLE
;
756 if (flags
& NBDKIT_FLAG_FUA
)
757 f
|= LIBNBD_CMD_FLAG_FUA
;
758 s
= nbdplug_register (h
, nbd_aio_zero (h
->nbd
, count
, offset
, f
));
759 return nbdplug_reply (h
, s
);
762 /* Trim a portion of the file. */
764 nbdplug_trim (void *handle
, uint32_t count
, uint64_t offset
, uint32_t flags
)
766 struct handle
*h
= handle
;
767 struct transaction
*s
;
768 uint32_t f
= flags
& NBDKIT_FLAG_FUA
? LIBNBD_CMD_FLAG_FUA
: 0;
770 assert (!(flags
& ~NBDKIT_FLAG_FUA
));
771 s
= nbdplug_register (h
, nbd_aio_trim (h
->nbd
, count
, offset
, f
));
772 return nbdplug_reply (h
, s
);
775 /* Flush the file to disk. */
777 nbdplug_flush (void *handle
, uint32_t flags
)
779 struct handle
*h
= handle
;
780 struct transaction
*s
;
783 s
= nbdplug_register (h
, nbd_aio_flush (h
->nbd
, 0));
784 return nbdplug_reply (h
, s
);
788 nbdplug_extent (void *opaque
, const char *metacontext
, uint64_t offset
,
789 uint32_t *entries
, size_t nr_entries
)
791 struct nbdkit_extents
*extents
= opaque
;
793 assert (strcmp (metacontext
, "base:allocation") == 0);
794 assert (nr_entries
% 2 == 0);
796 /* We rely on the fact that NBDKIT_EXTENT_* match NBD_STATE_* */
797 if (nbdkit_add_extent (extents
, offset
, entries
[0], entries
[1]) == -1)
799 offset
+= entries
[0];
806 /* Read extents of the file. */
808 nbdplug_extents (void *handle
, uint32_t count
, uint64_t offset
,
809 uint32_t flags
, struct nbdkit_extents
*extents
)
811 struct handle
*h
= handle
;
812 struct transaction
*s
;
813 uint32_t f
= flags
& NBDKIT_FLAG_REQ_ONE
? LIBNBD_CMD_FLAG_REQ_ONE
: 0;
815 assert (!(flags
& ~NBDKIT_FLAG_REQ_ONE
));
816 s
= nbdplug_register (h
, nbd_aio_block_status (h
->nbd
, count
, offset
,
817 extents
, nbdplug_extent
, f
));
818 return nbdplug_reply (h
, s
);
821 /* Cache a portion of the file. */
823 nbdplug_cache (void *handle
, uint32_t count
, uint64_t offset
, uint32_t flags
)
825 struct handle
*h
= handle
;
826 struct transaction
*s
;
829 s
= nbdplug_register (h
, nbd_aio_cache (h
->nbd
, count
, offset
, 0));
830 return nbdplug_reply (h
, s
);
833 static struct nbdkit_plugin plugin
= {
835 .longname
= "nbdkit nbd plugin",
836 .version
= PACKAGE_VERSION
,
837 .unload
= nbdplug_unload
,
838 .config
= nbdplug_config
,
839 .config_complete
= nbdplug_config_complete
,
840 .config_help
= nbdplug_config_help
,
841 .magic_config_key
= "uri",
842 .dump_plugin
= nbdplug_dump_plugin
,
843 .open
= nbdplug_open
,
844 .close
= nbdplug_close
,
845 .get_size
= nbdplug_get_size
,
846 .can_write
= nbdplug_can_write
,
847 .can_flush
= nbdplug_can_flush
,
848 .is_rotational
= nbdplug_is_rotational
,
849 .can_trim
= nbdplug_can_trim
,
850 .can_zero
= nbdplug_can_zero
,
851 .can_fua
= nbdplug_can_fua
,
852 .can_multi_conn
= nbdplug_can_multi_conn
,
853 .can_extents
= nbdplug_can_extents
,
854 .can_cache
= nbdplug_can_cache
,
855 .pread
= nbdplug_pread
,
856 .pwrite
= nbdplug_pwrite
,
857 .zero
= nbdplug_zero
,
858 .flush
= nbdplug_flush
,
859 .trim
= nbdplug_trim
,
860 .extents
= nbdplug_extents
,
861 .cache
= nbdplug_cache
,
862 .errno_is_preserved
= 1,
865 NBDKIT_REGISTER_PLUGIN (plugin
)