2 * GlusterFS backend for QEMU
4 * Copyright (C) 2012 Bharata B Rao <bharata@linux.vnet.ibm.com>
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
11 #include "qemu/osdep.h"
12 #include <glusterfs/api/glfs.h>
13 #include "block/block_int.h"
14 #include "qapi/error.h"
15 #include "qapi/qmp/qdict.h"
16 #include "qapi/qmp/qerror.h"
18 #include "qemu/error-report.h"
19 #include "qemu/option.h"
20 #include "qemu/cutils.h"
22 #define GLUSTER_OPT_FILENAME "filename"
23 #define GLUSTER_OPT_VOLUME "volume"
24 #define GLUSTER_OPT_PATH "path"
25 #define GLUSTER_OPT_TYPE "type"
26 #define GLUSTER_OPT_SERVER_PATTERN "server."
27 #define GLUSTER_OPT_HOST "host"
28 #define GLUSTER_OPT_PORT "port"
29 #define GLUSTER_OPT_TO "to"
30 #define GLUSTER_OPT_IPV4 "ipv4"
31 #define GLUSTER_OPT_IPV6 "ipv6"
32 #define GLUSTER_OPT_SOCKET "socket"
33 #define GLUSTER_OPT_DEBUG "debug"
34 #define GLUSTER_DEFAULT_PORT 24007
35 #define GLUSTER_DEBUG_DEFAULT 4
36 #define GLUSTER_DEBUG_MAX 9
37 #define GLUSTER_OPT_LOGFILE "logfile"
38 #define GLUSTER_LOGFILE_DEFAULT "-" /* handled in libgfapi as /dev/stderr */
40 #define GERR_INDEX_HINT "hint: check in 'server' array index '%d'\n"
42 typedef struct GlusterAIOCB
{
46 AioContext
*aio_context
;
49 typedef struct BDRVGlusterState
{
53 bool supports_seek_data
;
57 typedef struct BDRVGlusterReopenState
{
60 } BDRVGlusterReopenState
;
63 typedef struct GlfsPreopened
{
69 typedef struct ListElement
{
70 QLIST_ENTRY(ListElement
) list
;
74 static QLIST_HEAD(glfs_list
, ListElement
) glfs_list
;
76 static QemuOptsList qemu_gluster_create_opts
= {
77 .name
= "qemu-gluster-create-opts",
78 .head
= QTAILQ_HEAD_INITIALIZER(qemu_gluster_create_opts
.head
),
81 .name
= BLOCK_OPT_SIZE
,
82 .type
= QEMU_OPT_SIZE
,
83 .help
= "Virtual disk size"
86 .name
= BLOCK_OPT_PREALLOC
,
87 .type
= QEMU_OPT_STRING
,
88 .help
= "Preallocation mode (allowed values: off, full)"
91 .name
= GLUSTER_OPT_DEBUG
,
92 .type
= QEMU_OPT_NUMBER
,
93 .help
= "Gluster log level, valid range is 0-9",
96 .name
= GLUSTER_OPT_LOGFILE
,
97 .type
= QEMU_OPT_STRING
,
98 .help
= "Logfile path of libgfapi",
100 { /* end of list */ }
104 static QemuOptsList runtime_opts
= {
106 .head
= QTAILQ_HEAD_INITIALIZER(runtime_opts
.head
),
109 .name
= GLUSTER_OPT_FILENAME
,
110 .type
= QEMU_OPT_STRING
,
111 .help
= "URL to the gluster image",
114 .name
= GLUSTER_OPT_DEBUG
,
115 .type
= QEMU_OPT_NUMBER
,
116 .help
= "Gluster log level, valid range is 0-9",
119 .name
= GLUSTER_OPT_LOGFILE
,
120 .type
= QEMU_OPT_STRING
,
121 .help
= "Logfile path of libgfapi",
123 { /* end of list */ }
127 static QemuOptsList runtime_json_opts
= {
128 .name
= "gluster_json",
129 .head
= QTAILQ_HEAD_INITIALIZER(runtime_json_opts
.head
),
132 .name
= GLUSTER_OPT_VOLUME
,
133 .type
= QEMU_OPT_STRING
,
134 .help
= "name of gluster volume where VM image resides",
137 .name
= GLUSTER_OPT_PATH
,
138 .type
= QEMU_OPT_STRING
,
139 .help
= "absolute path to image file in gluster volume",
142 .name
= GLUSTER_OPT_DEBUG
,
143 .type
= QEMU_OPT_NUMBER
,
144 .help
= "Gluster log level, valid range is 0-9",
146 { /* end of list */ }
150 static QemuOptsList runtime_type_opts
= {
151 .name
= "gluster_type",
152 .head
= QTAILQ_HEAD_INITIALIZER(runtime_type_opts
.head
),
155 .name
= GLUSTER_OPT_TYPE
,
156 .type
= QEMU_OPT_STRING
,
159 { /* end of list */ }
163 static QemuOptsList runtime_unix_opts
= {
164 .name
= "gluster_unix",
165 .head
= QTAILQ_HEAD_INITIALIZER(runtime_unix_opts
.head
),
168 .name
= GLUSTER_OPT_SOCKET
,
169 .type
= QEMU_OPT_STRING
,
170 .help
= "socket file path)",
172 { /* end of list */ }
176 static QemuOptsList runtime_inet_opts
= {
177 .name
= "gluster_inet",
178 .head
= QTAILQ_HEAD_INITIALIZER(runtime_inet_opts
.head
),
181 .name
= GLUSTER_OPT_TYPE
,
182 .type
= QEMU_OPT_STRING
,
186 .name
= GLUSTER_OPT_HOST
,
187 .type
= QEMU_OPT_STRING
,
188 .help
= "host address (hostname/ipv4/ipv6 addresses)",
191 .name
= GLUSTER_OPT_PORT
,
192 .type
= QEMU_OPT_STRING
,
193 .help
= "port number on which glusterd is listening (default 24007)",
197 .type
= QEMU_OPT_NUMBER
,
198 .help
= "max port number, not supported by gluster",
202 .type
= QEMU_OPT_BOOL
,
203 .help
= "ipv4 bool value, not supported by gluster",
207 .type
= QEMU_OPT_BOOL
,
208 .help
= "ipv6 bool value, not supported by gluster",
210 { /* end of list */ }
214 static void glfs_set_preopened(const char *volume
, glfs_t
*fs
)
216 ListElement
*entry
= NULL
;
218 entry
= g_new(ListElement
, 1);
220 entry
->saved
.volume
= g_strdup(volume
);
222 entry
->saved
.fs
= fs
;
223 entry
->saved
.ref
= 1;
225 QLIST_INSERT_HEAD(&glfs_list
, entry
, list
);
228 static glfs_t
*glfs_find_preopened(const char *volume
)
230 ListElement
*entry
= NULL
;
232 QLIST_FOREACH(entry
, &glfs_list
, list
) {
233 if (strcmp(entry
->saved
.volume
, volume
) == 0) {
235 return entry
->saved
.fs
;
242 static void glfs_clear_preopened(glfs_t
*fs
)
244 ListElement
*entry
= NULL
;
251 QLIST_FOREACH_SAFE(entry
, &glfs_list
, list
, next
) {
252 if (entry
->saved
.fs
== fs
) {
253 if (--entry
->saved
.ref
) {
257 QLIST_REMOVE(entry
, list
);
259 glfs_fini(entry
->saved
.fs
);
260 g_free(entry
->saved
.volume
);
266 static int parse_volume_options(BlockdevOptionsGluster
*gconf
, char *path
)
275 p
= q
= path
+ strspn(path
, "/");
276 p
+= strcspn(p
, "/");
280 gconf
->volume
= g_strndup(q
, p
- q
);
287 gconf
->path
= g_strdup(p
);
292 * file=gluster[+transport]://[host[:port]]/volume/path[?socket=...]
294 * 'gluster' is the protocol.
296 * 'transport' specifies the transport type used to connect to gluster
297 * management daemon (glusterd). Valid transport types are
298 * tcp or unix. If a transport type isn't specified, then tcp type is assumed.
300 * 'host' specifies the host where the volume file specification for
301 * the given volume resides. This can be either hostname or ipv4 address.
302 * If transport type is 'unix', then 'host' field should not be specified.
303 * The 'socket' field needs to be populated with the path to unix domain
306 * 'port' is the port number on which glusterd is listening. This is optional
307 * and if not specified, QEMU will send 0 which will make gluster to use the
308 * default port. If the transport type is unix, then 'port' should not be
311 * 'volume' is the name of the gluster volume which contains the VM image.
313 * 'path' is the path to the actual VM image that resides on gluster volume.
317 * file=gluster://1.2.3.4/testvol/a.img
318 * file=gluster+tcp://1.2.3.4/testvol/a.img
319 * file=gluster+tcp://1.2.3.4:24007/testvol/dir/a.img
320 * file=gluster+tcp://host.domain.com:24007/testvol/dir/a.img
321 * file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket
323 static int qemu_gluster_parse_uri(BlockdevOptionsGluster
*gconf
,
324 const char *filename
)
326 SocketAddress
*gsconf
;
328 QueryParams
*qp
= NULL
;
329 bool is_unix
= false;
332 uri
= uri_parse(filename
);
337 gconf
->server
= g_new0(SocketAddressList
, 1);
338 gconf
->server
->value
= gsconf
= g_new0(SocketAddress
, 1);
341 if (!uri
->scheme
|| !strcmp(uri
->scheme
, "gluster")) {
342 gsconf
->type
= SOCKET_ADDRESS_TYPE_INET
;
343 } else if (!strcmp(uri
->scheme
, "gluster+tcp")) {
344 gsconf
->type
= SOCKET_ADDRESS_TYPE_INET
;
345 } else if (!strcmp(uri
->scheme
, "gluster+unix")) {
346 gsconf
->type
= SOCKET_ADDRESS_TYPE_UNIX
;
348 } else if (!strcmp(uri
->scheme
, "gluster+rdma")) {
349 gsconf
->type
= SOCKET_ADDRESS_TYPE_INET
;
350 warn_report("rdma feature is not supported, falling back to tcp");
356 ret
= parse_volume_options(gconf
, uri
->path
);
361 qp
= query_params_parse(uri
->query
);
362 if (qp
->n
> 1 || (is_unix
&& !qp
->n
) || (!is_unix
&& qp
->n
)) {
368 if (uri
->server
|| uri
->port
) {
372 if (strcmp(qp
->p
[0].name
, "socket")) {
376 gsconf
->u
.q_unix
.path
= g_strdup(qp
->p
[0].value
);
378 gsconf
->u
.inet
.host
= g_strdup(uri
->server
? uri
->server
: "localhost");
380 gsconf
->u
.inet
.port
= g_strdup_printf("%d", uri
->port
);
382 gsconf
->u
.inet
.port
= g_strdup_printf("%d", GLUSTER_DEFAULT_PORT
);
388 query_params_free(qp
);
394 static struct glfs
*qemu_gluster_glfs_init(BlockdevOptionsGluster
*gconf
,
400 SocketAddressList
*server
;
401 unsigned long long port
;
403 glfs
= glfs_find_preopened(gconf
->volume
);
408 glfs
= glfs_new(gconf
->volume
);
413 glfs_set_preopened(gconf
->volume
, glfs
);
415 for (server
= gconf
->server
; server
; server
= server
->next
) {
416 switch (server
->value
->type
) {
417 case SOCKET_ADDRESS_TYPE_UNIX
:
418 ret
= glfs_set_volfile_server(glfs
, "unix",
419 server
->value
->u
.q_unix
.path
, 0);
421 case SOCKET_ADDRESS_TYPE_INET
:
422 if (parse_uint_full(server
->value
->u
.inet
.port
, &port
, 10) < 0 ||
424 error_setg(errp
, "'%s' is not a valid port number",
425 server
->value
->u
.inet
.port
);
429 ret
= glfs_set_volfile_server(glfs
, "tcp",
430 server
->value
->u
.inet
.host
,
433 case SOCKET_ADDRESS_TYPE_VSOCK
:
434 case SOCKET_ADDRESS_TYPE_FD
:
444 ret
= glfs_set_logging(glfs
, gconf
->logfile
, gconf
->debug
);
449 ret
= glfs_init(glfs
);
451 error_setg(errp
, "Gluster connection for volume %s, path %s failed"
452 " to connect", gconf
->volume
, gconf
->path
);
453 for (server
= gconf
->server
; server
; server
= server
->next
) {
454 if (server
->value
->type
== SOCKET_ADDRESS_TYPE_UNIX
) {
455 error_append_hint(errp
, "hint: failed on socket %s ",
456 server
->value
->u
.q_unix
.path
);
458 error_append_hint(errp
, "hint: failed on host %s and port %s ",
459 server
->value
->u
.inet
.host
,
460 server
->value
->u
.inet
.port
);
464 error_append_hint(errp
, "Please refer to gluster logs for more info\n");
466 /* glfs_init sometimes doesn't set errno although docs suggest that */
478 glfs_clear_preopened(glfs
);
485 * Convert the json formatted command line into qapi.
487 static int qemu_gluster_parse_json(BlockdevOptionsGluster
*gconf
,
488 QDict
*options
, Error
**errp
)
491 SocketAddress
*gsconf
= NULL
;
492 SocketAddressList
*curr
= NULL
;
493 QDict
*backing_options
= NULL
;
494 Error
*local_err
= NULL
;
497 int i
, type
, num_servers
;
499 /* create opts info from runtime_json_opts list */
500 opts
= qemu_opts_create(&runtime_json_opts
, NULL
, 0, &error_abort
);
501 qemu_opts_absorb_qdict(opts
, options
, &local_err
);
506 num_servers
= qdict_array_entries(options
, GLUSTER_OPT_SERVER_PATTERN
);
507 if (num_servers
< 1) {
508 error_setg(&local_err
, QERR_MISSING_PARAMETER
, "server");
512 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_VOLUME
);
514 error_setg(&local_err
, QERR_MISSING_PARAMETER
, GLUSTER_OPT_VOLUME
);
517 gconf
->volume
= g_strdup(ptr
);
519 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_PATH
);
521 error_setg(&local_err
, QERR_MISSING_PARAMETER
, GLUSTER_OPT_PATH
);
524 gconf
->path
= g_strdup(ptr
);
527 for (i
= 0; i
< num_servers
; i
++) {
528 str
= g_strdup_printf(GLUSTER_OPT_SERVER_PATTERN
"%d.", i
);
529 qdict_extract_subqdict(options
, &backing_options
, str
);
531 /* create opts info from runtime_type_opts list */
532 opts
= qemu_opts_create(&runtime_type_opts
, NULL
, 0, &error_abort
);
533 qemu_opts_absorb_qdict(opts
, backing_options
, &local_err
);
538 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_TYPE
);
540 error_setg(&local_err
, QERR_MISSING_PARAMETER
, GLUSTER_OPT_TYPE
);
541 error_append_hint(&local_err
, GERR_INDEX_HINT
, i
);
545 gsconf
= g_new0(SocketAddress
, 1);
546 if (!strcmp(ptr
, "tcp")) {
547 ptr
= "inet"; /* accept legacy "tcp" */
549 type
= qapi_enum_parse(&SocketAddressType_lookup
, ptr
, -1, NULL
);
550 if (type
!= SOCKET_ADDRESS_TYPE_INET
551 && type
!= SOCKET_ADDRESS_TYPE_UNIX
) {
552 error_setg(&local_err
,
553 "Parameter '%s' may be 'inet' or 'unix'",
555 error_append_hint(&local_err
, GERR_INDEX_HINT
, i
);
561 if (gsconf
->type
== SOCKET_ADDRESS_TYPE_INET
) {
562 /* create opts info from runtime_inet_opts list */
563 opts
= qemu_opts_create(&runtime_inet_opts
, NULL
, 0, &error_abort
);
564 qemu_opts_absorb_qdict(opts
, backing_options
, &local_err
);
569 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_HOST
);
571 error_setg(&local_err
, QERR_MISSING_PARAMETER
,
573 error_append_hint(&local_err
, GERR_INDEX_HINT
, i
);
576 gsconf
->u
.inet
.host
= g_strdup(ptr
);
577 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_PORT
);
579 error_setg(&local_err
, QERR_MISSING_PARAMETER
,
581 error_append_hint(&local_err
, GERR_INDEX_HINT
, i
);
584 gsconf
->u
.inet
.port
= g_strdup(ptr
);
586 /* defend for unsupported fields in InetSocketAddress,
587 * i.e. @ipv4, @ipv6 and @to
589 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_TO
);
591 gsconf
->u
.inet
.has_to
= true;
593 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_IPV4
);
595 gsconf
->u
.inet
.has_ipv4
= true;
597 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_IPV6
);
599 gsconf
->u
.inet
.has_ipv6
= true;
601 if (gsconf
->u
.inet
.has_to
) {
602 error_setg(&local_err
, "Parameter 'to' not supported");
605 if (gsconf
->u
.inet
.has_ipv4
|| gsconf
->u
.inet
.has_ipv6
) {
606 error_setg(&local_err
, "Parameters 'ipv4/ipv6' not supported");
611 /* create opts info from runtime_unix_opts list */
612 opts
= qemu_opts_create(&runtime_unix_opts
, NULL
, 0, &error_abort
);
613 qemu_opts_absorb_qdict(opts
, backing_options
, &local_err
);
618 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_SOCKET
);
620 error_setg(&local_err
, QERR_MISSING_PARAMETER
,
622 error_append_hint(&local_err
, GERR_INDEX_HINT
, i
);
625 gsconf
->u
.q_unix
.path
= g_strdup(ptr
);
629 if (gconf
->server
== NULL
) {
630 gconf
->server
= g_new0(SocketAddressList
, 1);
631 gconf
->server
->value
= gsconf
;
632 curr
= gconf
->server
;
634 curr
->next
= g_new0(SocketAddressList
, 1);
635 curr
->next
->value
= gsconf
;
640 QDECREF(backing_options
);
641 backing_options
= NULL
;
649 error_propagate(errp
, local_err
);
650 qapi_free_SocketAddress(gsconf
);
653 QDECREF(backing_options
);
658 static struct glfs
*qemu_gluster_init(BlockdevOptionsGluster
*gconf
,
659 const char *filename
,
660 QDict
*options
, Error
**errp
)
664 ret
= qemu_gluster_parse_uri(gconf
, filename
);
666 error_setg(errp
, "invalid URI");
667 error_append_hint(errp
, "Usage: file=gluster[+transport]://"
668 "[host[:port]]volume/path[?socket=...]"
670 "[,file.logfile=/path/filename.log]\n");
675 ret
= qemu_gluster_parse_json(gconf
, options
, errp
);
677 error_append_hint(errp
, "Usage: "
678 "-drive driver=qcow2,file.driver=gluster,"
679 "file.volume=testvol,file.path=/path/a.qcow2"
681 "[,file.logfile=/path/filename.log],"
682 "file.server.0.type=inet,"
683 "file.server.0.host=1.2.3.4,"
684 "file.server.0.port=24007,"
685 "file.server.1.transport=unix,"
686 "file.server.1.socket=/var/run/glusterd.socket ..."
694 return qemu_gluster_glfs_init(gconf
, errp
);
698 * AIO callback routine called from GlusterFS thread.
700 static void gluster_finish_aiocb(struct glfs_fd
*fd
, ssize_t ret
, void *arg
)
702 GlusterAIOCB
*acb
= (GlusterAIOCB
*)arg
;
704 if (!ret
|| ret
== acb
->size
) {
705 acb
->ret
= 0; /* Success */
706 } else if (ret
< 0) {
707 acb
->ret
= -errno
; /* Read/Write failed */
709 acb
->ret
= -EIO
; /* Partial read/write - fail it */
712 aio_co_schedule(acb
->aio_context
, acb
->coroutine
);
715 static void qemu_gluster_parse_flags(int bdrv_flags
, int *open_flags
)
717 assert(open_flags
!= NULL
);
719 *open_flags
|= O_BINARY
;
721 if (bdrv_flags
& BDRV_O_RDWR
) {
722 *open_flags
|= O_RDWR
;
724 *open_flags
|= O_RDONLY
;
727 if ((bdrv_flags
& BDRV_O_NOCACHE
)) {
728 *open_flags
|= O_DIRECT
;
733 * Do SEEK_DATA/HOLE to detect if it is functional. Older broken versions of
734 * gfapi incorrectly return the current offset when SEEK_DATA/HOLE is used.
735 * - Corrected versions return -1 and set errno to EINVAL.
736 * - Versions that support SEEK_DATA/HOLE correctly, will return -1 and set
737 * errno to ENXIO when SEEK_DATA is called with a position of EOF.
739 static bool qemu_gluster_test_seek(struct glfs_fd
*fd
)
743 #if defined SEEK_HOLE && defined SEEK_DATA
746 eof
= glfs_lseek(fd
, 0, SEEK_END
);
748 /* this should never occur */
752 /* this should always fail with ENXIO if SEEK_DATA is supported */
753 ret
= glfs_lseek(fd
, eof
, SEEK_DATA
);
756 return (ret
< 0) && (errno
== ENXIO
);
759 static int qemu_gluster_open(BlockDriverState
*bs
, QDict
*options
,
760 int bdrv_flags
, Error
**errp
)
762 BDRVGlusterState
*s
= bs
->opaque
;
765 BlockdevOptionsGluster
*gconf
= NULL
;
767 Error
*local_err
= NULL
;
768 const char *filename
, *logfile
;
770 opts
= qemu_opts_create(&runtime_opts
, NULL
, 0, &error_abort
);
771 qemu_opts_absorb_qdict(opts
, options
, &local_err
);
773 error_propagate(errp
, local_err
);
778 filename
= qemu_opt_get(opts
, GLUSTER_OPT_FILENAME
);
780 s
->debug
= qemu_opt_get_number(opts
, GLUSTER_OPT_DEBUG
,
781 GLUSTER_DEBUG_DEFAULT
);
784 } else if (s
->debug
> GLUSTER_DEBUG_MAX
) {
785 s
->debug
= GLUSTER_DEBUG_MAX
;
788 gconf
= g_new0(BlockdevOptionsGluster
, 1);
789 gconf
->debug
= s
->debug
;
790 gconf
->has_debug
= true;
792 logfile
= qemu_opt_get(opts
, GLUSTER_OPT_LOGFILE
);
793 s
->logfile
= g_strdup(logfile
? logfile
: GLUSTER_LOGFILE_DEFAULT
);
795 gconf
->logfile
= g_strdup(s
->logfile
);
796 gconf
->has_logfile
= true;
798 s
->glfs
= qemu_gluster_init(gconf
, filename
, options
, errp
);
804 #ifdef CONFIG_GLUSTERFS_XLATOR_OPT
805 /* Without this, if fsync fails for a recoverable reason (for instance,
806 * ENOSPC), gluster will dump its cache, preventing retries. This means
807 * almost certain data loss. Not all gluster versions support the
808 * 'resync-failed-syncs-after-fsync' key value, but there is no way to
809 * discover during runtime if it is supported (this api returns success for
810 * unknown key/value pairs) */
811 ret
= glfs_set_xlator_option(s
->glfs
, "*-write-behind",
812 "resync-failed-syncs-after-fsync",
815 error_setg_errno(errp
, errno
, "Unable to set xlator key/value pair");
821 qemu_gluster_parse_flags(bdrv_flags
, &open_flags
);
823 s
->fd
= glfs_open(s
->glfs
, gconf
->path
, open_flags
);
828 s
->supports_seek_data
= qemu_gluster_test_seek(s
->fd
);
832 qapi_free_BlockdevOptionsGluster(gconf
);
841 glfs_clear_preopened(s
->glfs
);
846 static int qemu_gluster_reopen_prepare(BDRVReopenState
*state
,
847 BlockReopenQueue
*queue
, Error
**errp
)
851 BDRVGlusterReopenState
*reop_s
;
852 BlockdevOptionsGluster
*gconf
;
855 assert(state
!= NULL
);
856 assert(state
->bs
!= NULL
);
858 s
= state
->bs
->opaque
;
860 state
->opaque
= g_new0(BDRVGlusterReopenState
, 1);
861 reop_s
= state
->opaque
;
863 qemu_gluster_parse_flags(state
->flags
, &open_flags
);
865 gconf
= g_new0(BlockdevOptionsGluster
, 1);
866 gconf
->debug
= s
->debug
;
867 gconf
->has_debug
= true;
868 gconf
->logfile
= g_strdup(s
->logfile
);
869 gconf
->has_logfile
= true;
870 reop_s
->glfs
= qemu_gluster_init(gconf
, state
->bs
->filename
, NULL
, errp
);
871 if (reop_s
->glfs
== NULL
) {
876 #ifdef CONFIG_GLUSTERFS_XLATOR_OPT
877 ret
= glfs_set_xlator_option(reop_s
->glfs
, "*-write-behind",
878 "resync-failed-syncs-after-fsync", "on");
880 error_setg_errno(errp
, errno
, "Unable to set xlator key/value pair");
886 reop_s
->fd
= glfs_open(reop_s
->glfs
, gconf
->path
, open_flags
);
887 if (reop_s
->fd
== NULL
) {
888 /* reops->glfs will be cleaned up in _abort */
894 /* state->opaque will be freed in either the _abort or _commit */
895 qapi_free_BlockdevOptionsGluster(gconf
);
899 static void qemu_gluster_reopen_commit(BDRVReopenState
*state
)
901 BDRVGlusterReopenState
*reop_s
= state
->opaque
;
902 BDRVGlusterState
*s
= state
->bs
->opaque
;
910 glfs_clear_preopened(s
->glfs
);
912 /* use the newly opened image / connection */
914 s
->glfs
= reop_s
->glfs
;
916 g_free(state
->opaque
);
917 state
->opaque
= NULL
;
923 static void qemu_gluster_reopen_abort(BDRVReopenState
*state
)
925 BDRVGlusterReopenState
*reop_s
= state
->opaque
;
927 if (reop_s
== NULL
) {
932 glfs_close(reop_s
->fd
);
935 glfs_clear_preopened(reop_s
->glfs
);
937 g_free(state
->opaque
);
938 state
->opaque
= NULL
;
943 #ifdef CONFIG_GLUSTERFS_ZEROFILL
944 static coroutine_fn
int qemu_gluster_co_pwrite_zeroes(BlockDriverState
*bs
,
947 BdrvRequestFlags flags
)
951 BDRVGlusterState
*s
= bs
->opaque
;
955 acb
.coroutine
= qemu_coroutine_self();
956 acb
.aio_context
= bdrv_get_aio_context(bs
);
958 ret
= glfs_zerofill_async(s
->fd
, offset
, size
, gluster_finish_aiocb
, &acb
);
963 qemu_coroutine_yield();
968 static int qemu_gluster_do_truncate(struct glfs_fd
*fd
, int64_t offset
,
969 PreallocMode prealloc
, Error
**errp
)
971 int64_t current_length
;
973 current_length
= glfs_lseek(fd
, 0, SEEK_END
);
974 if (current_length
< 0) {
975 error_setg_errno(errp
, errno
, "Failed to determine current size");
979 if (current_length
> offset
&& prealloc
!= PREALLOC_MODE_OFF
) {
980 error_setg(errp
, "Cannot use preallocation for shrinking files");
984 if (current_length
== offset
) {
989 #ifdef CONFIG_GLUSTERFS_FALLOCATE
990 case PREALLOC_MODE_FALLOC
:
991 if (glfs_fallocate(fd
, 0, current_length
, offset
- current_length
)) {
992 error_setg_errno(errp
, errno
, "Could not preallocate data");
996 #endif /* CONFIG_GLUSTERFS_FALLOCATE */
997 #ifdef CONFIG_GLUSTERFS_ZEROFILL
998 case PREALLOC_MODE_FULL
:
999 if (glfs_ftruncate(fd
, offset
)) {
1000 error_setg_errno(errp
, errno
, "Could not resize file");
1003 if (glfs_zerofill(fd
, current_length
, offset
- current_length
)) {
1004 error_setg_errno(errp
, errno
, "Could not zerofill the new area");
1008 #endif /* CONFIG_GLUSTERFS_ZEROFILL */
1009 case PREALLOC_MODE_OFF
:
1010 if (glfs_ftruncate(fd
, offset
)) {
1011 error_setg_errno(errp
, errno
, "Could not resize file");
1016 error_setg(errp
, "Unsupported preallocation mode: %s",
1017 PreallocMode_str(prealloc
));
1024 static int qemu_gluster_create(const char *filename
,
1025 QemuOpts
*opts
, Error
**errp
)
1027 BlockdevOptionsGluster
*gconf
;
1029 struct glfs_fd
*fd
= NULL
;
1031 PreallocMode prealloc
;
1032 int64_t total_size
= 0;
1034 Error
*local_err
= NULL
;
1036 gconf
= g_new0(BlockdevOptionsGluster
, 1);
1037 gconf
->debug
= qemu_opt_get_number_del(opts
, GLUSTER_OPT_DEBUG
,
1038 GLUSTER_DEBUG_DEFAULT
);
1039 if (gconf
->debug
< 0) {
1041 } else if (gconf
->debug
> GLUSTER_DEBUG_MAX
) {
1042 gconf
->debug
= GLUSTER_DEBUG_MAX
;
1044 gconf
->has_debug
= true;
1046 gconf
->logfile
= qemu_opt_get_del(opts
, GLUSTER_OPT_LOGFILE
);
1047 if (!gconf
->logfile
) {
1048 gconf
->logfile
= g_strdup(GLUSTER_LOGFILE_DEFAULT
);
1050 gconf
->has_logfile
= true;
1052 glfs
= qemu_gluster_init(gconf
, filename
, NULL
, errp
);
1058 total_size
= ROUND_UP(qemu_opt_get_size_del(opts
, BLOCK_OPT_SIZE
, 0),
1061 tmp
= qemu_opt_get_del(opts
, BLOCK_OPT_PREALLOC
);
1062 prealloc
= qapi_enum_parse(&PreallocMode_lookup
, tmp
, PREALLOC_MODE_OFF
,
1066 error_propagate(errp
, local_err
);
1071 fd
= glfs_creat(glfs
, gconf
->path
,
1072 O_WRONLY
| O_CREAT
| O_TRUNC
| O_BINARY
, S_IRUSR
| S_IWUSR
);
1078 ret
= qemu_gluster_do_truncate(fd
, total_size
, prealloc
, errp
);
1082 if (glfs_close(fd
) != 0 && ret
== 0) {
1086 qapi_free_BlockdevOptionsGluster(gconf
);
1087 glfs_clear_preopened(glfs
);
1091 static coroutine_fn
int qemu_gluster_co_rw(BlockDriverState
*bs
,
1092 int64_t sector_num
, int nb_sectors
,
1093 QEMUIOVector
*qiov
, int write
)
1097 BDRVGlusterState
*s
= bs
->opaque
;
1098 size_t size
= nb_sectors
* BDRV_SECTOR_SIZE
;
1099 off_t offset
= sector_num
* BDRV_SECTOR_SIZE
;
1103 acb
.coroutine
= qemu_coroutine_self();
1104 acb
.aio_context
= bdrv_get_aio_context(bs
);
1107 ret
= glfs_pwritev_async(s
->fd
, qiov
->iov
, qiov
->niov
, offset
, 0,
1108 gluster_finish_aiocb
, &acb
);
1110 ret
= glfs_preadv_async(s
->fd
, qiov
->iov
, qiov
->niov
, offset
, 0,
1111 gluster_finish_aiocb
, &acb
);
1118 qemu_coroutine_yield();
1122 static int qemu_gluster_truncate(BlockDriverState
*bs
, int64_t offset
,
1123 PreallocMode prealloc
, Error
**errp
)
1125 BDRVGlusterState
*s
= bs
->opaque
;
1126 return qemu_gluster_do_truncate(s
->fd
, offset
, prealloc
, errp
);
1129 static coroutine_fn
int qemu_gluster_co_readv(BlockDriverState
*bs
,
1134 return qemu_gluster_co_rw(bs
, sector_num
, nb_sectors
, qiov
, 0);
1137 static coroutine_fn
int qemu_gluster_co_writev(BlockDriverState
*bs
,
1142 return qemu_gluster_co_rw(bs
, sector_num
, nb_sectors
, qiov
, 1);
1145 static void qemu_gluster_close(BlockDriverState
*bs
)
1147 BDRVGlusterState
*s
= bs
->opaque
;
1154 glfs_clear_preopened(s
->glfs
);
1157 static coroutine_fn
int qemu_gluster_co_flush_to_disk(BlockDriverState
*bs
)
1161 BDRVGlusterState
*s
= bs
->opaque
;
1165 acb
.coroutine
= qemu_coroutine_self();
1166 acb
.aio_context
= bdrv_get_aio_context(bs
);
1168 ret
= glfs_fsync_async(s
->fd
, gluster_finish_aiocb
, &acb
);
1174 qemu_coroutine_yield();
1183 /* Some versions of Gluster (3.5.6 -> 3.5.8?) will not retain its cache
1184 * after a fsync failure, so we have no way of allowing the guest to safely
1185 * continue. Gluster versions prior to 3.5.6 don't retain the cache
1186 * either, but will invalidate the fd on error, so this is again our only
1189 * The 'resync-failed-syncs-after-fsync' xlator option for the
1190 * write-behind cache will cause later gluster versions to retain its
1191 * cache after error, so long as the fd remains open. However, we
1192 * currently have no way of knowing if this option is supported.
1194 * TODO: Once gluster provides a way for us to determine if the option
1195 * is supported, bypass the closure and setting drv to NULL. */
1196 qemu_gluster_close(bs
);
1201 #ifdef CONFIG_GLUSTERFS_DISCARD
1202 static coroutine_fn
int qemu_gluster_co_pdiscard(BlockDriverState
*bs
,
1203 int64_t offset
, int size
)
1207 BDRVGlusterState
*s
= bs
->opaque
;
1211 acb
.coroutine
= qemu_coroutine_self();
1212 acb
.aio_context
= bdrv_get_aio_context(bs
);
1214 ret
= glfs_discard_async(s
->fd
, offset
, size
, gluster_finish_aiocb
, &acb
);
1219 qemu_coroutine_yield();
1224 static int64_t qemu_gluster_getlength(BlockDriverState
*bs
)
1226 BDRVGlusterState
*s
= bs
->opaque
;
1229 ret
= glfs_lseek(s
->fd
, 0, SEEK_END
);
1237 static int64_t qemu_gluster_allocated_file_size(BlockDriverState
*bs
)
1239 BDRVGlusterState
*s
= bs
->opaque
;
1243 ret
= glfs_fstat(s
->fd
, &st
);
1247 return st
.st_blocks
* 512;
1251 static int qemu_gluster_has_zero_init(BlockDriverState
*bs
)
1253 /* GlusterFS volume could be backed by a block device */
1258 * Find allocation range in @bs around offset @start.
1259 * May change underlying file descriptor's file offset.
1260 * If @start is not in a hole, store @start in @data, and the
1261 * beginning of the next hole in @hole, and return 0.
1262 * If @start is in a non-trailing hole, store @start in @hole and the
1263 * beginning of the next non-hole in @data, and return 0.
1264 * If @start is in a trailing hole or beyond EOF, return -ENXIO.
1265 * If we can't find out, return a negative errno other than -ENXIO.
1267 * (Shamefully copied from file-posix.c, only miniscule adaptions.)
1269 static int find_allocation(BlockDriverState
*bs
, off_t start
,
1270 off_t
*data
, off_t
*hole
)
1272 BDRVGlusterState
*s
= bs
->opaque
;
1274 if (!s
->supports_seek_data
) {
1278 #if defined SEEK_HOLE && defined SEEK_DATA
1283 * D1. offs == start: start is in data
1284 * D2. offs > start: start is in a hole, next data at offs
1285 * D3. offs < 0, errno = ENXIO: either start is in a trailing hole
1286 * or start is beyond EOF
1287 * If the latter happens, the file has been truncated behind
1288 * our back since we opened it. All bets are off then.
1289 * Treating like a trailing hole is simplest.
1290 * D4. offs < 0, errno != ENXIO: we learned nothing
1292 offs
= glfs_lseek(s
->fd
, start
, SEEK_DATA
);
1294 return -errno
; /* D3 or D4 */
1298 /* This is not a valid return by lseek(). We are safe to just return
1299 * -EIO in this case, and we'll treat it like D4. Unfortunately some
1300 * versions of gluster server will return offs < start, so an assert
1301 * here will unnecessarily abort QEMU. */
1306 /* D2: in hole, next data at offs */
1312 /* D1: in data, end not yet known */
1316 * H1. offs == start: start is in a hole
1317 * If this happens here, a hole has been dug behind our back
1318 * since the previous lseek().
1319 * H2. offs > start: either start is in data, next hole at offs,
1320 * or start is in trailing hole, EOF at offs
1321 * Linux treats trailing holes like any other hole: offs ==
1322 * start. Solaris seeks to EOF instead: offs > start (blech).
1323 * If that happens here, a hole has been dug behind our back
1324 * since the previous lseek().
1325 * H3. offs < 0, errno = ENXIO: start is beyond EOF
1326 * If this happens, the file has been truncated behind our
1327 * back since we opened it. Treat it like a trailing hole.
1328 * H4. offs < 0, errno != ENXIO: we learned nothing
1329 * Pretend we know nothing at all, i.e. "forget" about D1.
1331 offs
= glfs_lseek(s
->fd
, start
, SEEK_HOLE
);
1333 return -errno
; /* D1 and (H3 or H4) */
1337 /* This is not a valid return by lseek(). We are safe to just return
1338 * -EIO in this case, and we'll treat it like H4. Unfortunately some
1339 * versions of gluster server will return offs < start, so an assert
1340 * here will unnecessarily abort QEMU. */
1346 * D1 and H2: either in data, next hole at offs, or it was in
1347 * data but is now in a trailing hole. In the latter case,
1348 * all bets are off. Treating it as if it there was data all
1349 * the way to EOF is safe, so simply do that.
1365 * Returns the allocation status of the specified sectors.
1367 * If 'sector_num' is beyond the end of the disk image the return value is 0
1368 * and 'pnum' is set to 0.
1370 * 'pnum' is set to the number of sectors (including and immediately following
1371 * the specified sector) that are known to be in the same
1372 * allocated/unallocated state.
1374 * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes
1375 * beyond the end of the disk image it will be clamped.
1377 * (Based on raw_co_get_block_status() from file-posix.c.)
1379 static int64_t coroutine_fn
qemu_gluster_co_get_block_status(
1380 BlockDriverState
*bs
, int64_t sector_num
, int nb_sectors
, int *pnum
,
1381 BlockDriverState
**file
)
1383 BDRVGlusterState
*s
= bs
->opaque
;
1384 off_t start
, data
= 0, hole
= 0;
1392 start
= sector_num
* BDRV_SECTOR_SIZE
;
1393 total_size
= bdrv_getlength(bs
);
1394 if (total_size
< 0) {
1396 } else if (start
>= total_size
) {
1399 } else if (start
+ nb_sectors
* BDRV_SECTOR_SIZE
> total_size
) {
1400 nb_sectors
= DIV_ROUND_UP(total_size
- start
, BDRV_SECTOR_SIZE
);
1403 ret
= find_allocation(bs
, start
, &data
, &hole
);
1404 if (ret
== -ENXIO
) {
1407 ret
= BDRV_BLOCK_ZERO
;
1408 } else if (ret
< 0) {
1409 /* No info available, so pretend there are no holes */
1411 ret
= BDRV_BLOCK_DATA
;
1412 } else if (data
== start
) {
1413 /* On a data extent, compute sectors to the end of the extent,
1414 * possibly including a partial sector at EOF. */
1415 *pnum
= MIN(nb_sectors
, DIV_ROUND_UP(hole
- start
, BDRV_SECTOR_SIZE
));
1416 ret
= BDRV_BLOCK_DATA
;
1418 /* On a hole, compute sectors to the beginning of the next extent. */
1419 assert(hole
== start
);
1420 *pnum
= MIN(nb_sectors
, (data
- start
) / BDRV_SECTOR_SIZE
);
1421 ret
= BDRV_BLOCK_ZERO
;
1426 return ret
| BDRV_BLOCK_OFFSET_VALID
| start
;
1430 static BlockDriver bdrv_gluster
= {
1431 .format_name
= "gluster",
1432 .protocol_name
= "gluster",
1433 .instance_size
= sizeof(BDRVGlusterState
),
1434 .bdrv_needs_filename
= false,
1435 .bdrv_file_open
= qemu_gluster_open
,
1436 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
1437 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
1438 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
1439 .bdrv_close
= qemu_gluster_close
,
1440 .bdrv_create
= qemu_gluster_create
,
1441 .bdrv_getlength
= qemu_gluster_getlength
,
1442 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
1443 .bdrv_truncate
= qemu_gluster_truncate
,
1444 .bdrv_co_readv
= qemu_gluster_co_readv
,
1445 .bdrv_co_writev
= qemu_gluster_co_writev
,
1446 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
1447 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
1448 #ifdef CONFIG_GLUSTERFS_DISCARD
1449 .bdrv_co_pdiscard
= qemu_gluster_co_pdiscard
,
1451 #ifdef CONFIG_GLUSTERFS_ZEROFILL
1452 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
1454 .bdrv_co_get_block_status
= qemu_gluster_co_get_block_status
,
1455 .create_opts
= &qemu_gluster_create_opts
,
1458 static BlockDriver bdrv_gluster_tcp
= {
1459 .format_name
= "gluster",
1460 .protocol_name
= "gluster+tcp",
1461 .instance_size
= sizeof(BDRVGlusterState
),
1462 .bdrv_needs_filename
= false,
1463 .bdrv_file_open
= qemu_gluster_open
,
1464 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
1465 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
1466 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
1467 .bdrv_close
= qemu_gluster_close
,
1468 .bdrv_create
= qemu_gluster_create
,
1469 .bdrv_getlength
= qemu_gluster_getlength
,
1470 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
1471 .bdrv_truncate
= qemu_gluster_truncate
,
1472 .bdrv_co_readv
= qemu_gluster_co_readv
,
1473 .bdrv_co_writev
= qemu_gluster_co_writev
,
1474 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
1475 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
1476 #ifdef CONFIG_GLUSTERFS_DISCARD
1477 .bdrv_co_pdiscard
= qemu_gluster_co_pdiscard
,
1479 #ifdef CONFIG_GLUSTERFS_ZEROFILL
1480 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
1482 .bdrv_co_get_block_status
= qemu_gluster_co_get_block_status
,
1483 .create_opts
= &qemu_gluster_create_opts
,
1486 static BlockDriver bdrv_gluster_unix
= {
1487 .format_name
= "gluster",
1488 .protocol_name
= "gluster+unix",
1489 .instance_size
= sizeof(BDRVGlusterState
),
1490 .bdrv_needs_filename
= true,
1491 .bdrv_file_open
= qemu_gluster_open
,
1492 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
1493 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
1494 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
1495 .bdrv_close
= qemu_gluster_close
,
1496 .bdrv_create
= qemu_gluster_create
,
1497 .bdrv_getlength
= qemu_gluster_getlength
,
1498 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
1499 .bdrv_truncate
= qemu_gluster_truncate
,
1500 .bdrv_co_readv
= qemu_gluster_co_readv
,
1501 .bdrv_co_writev
= qemu_gluster_co_writev
,
1502 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
1503 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
1504 #ifdef CONFIG_GLUSTERFS_DISCARD
1505 .bdrv_co_pdiscard
= qemu_gluster_co_pdiscard
,
1507 #ifdef CONFIG_GLUSTERFS_ZEROFILL
1508 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
1510 .bdrv_co_get_block_status
= qemu_gluster_co_get_block_status
,
1511 .create_opts
= &qemu_gluster_create_opts
,
1514 /* rdma is deprecated (actually never supported for volfile fetch).
1515 * Let's maintain it for the protocol compatibility, to make sure things
1516 * won't break immediately. For now, gluster+rdma will fall back to gluster+tcp
1517 * protocol with a warning.
1518 * TODO: remove gluster+rdma interface support
1520 static BlockDriver bdrv_gluster_rdma
= {
1521 .format_name
= "gluster",
1522 .protocol_name
= "gluster+rdma",
1523 .instance_size
= sizeof(BDRVGlusterState
),
1524 .bdrv_needs_filename
= true,
1525 .bdrv_file_open
= qemu_gluster_open
,
1526 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
1527 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
1528 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
1529 .bdrv_close
= qemu_gluster_close
,
1530 .bdrv_create
= qemu_gluster_create
,
1531 .bdrv_getlength
= qemu_gluster_getlength
,
1532 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
1533 .bdrv_truncate
= qemu_gluster_truncate
,
1534 .bdrv_co_readv
= qemu_gluster_co_readv
,
1535 .bdrv_co_writev
= qemu_gluster_co_writev
,
1536 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
1537 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
1538 #ifdef CONFIG_GLUSTERFS_DISCARD
1539 .bdrv_co_pdiscard
= qemu_gluster_co_pdiscard
,
1541 #ifdef CONFIG_GLUSTERFS_ZEROFILL
1542 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
1544 .bdrv_co_get_block_status
= qemu_gluster_co_get_block_status
,
1545 .create_opts
= &qemu_gluster_create_opts
,
1548 static void bdrv_gluster_init(void)
1550 bdrv_register(&bdrv_gluster_rdma
);
1551 bdrv_register(&bdrv_gluster_unix
);
1552 bdrv_register(&bdrv_gluster_tcp
);
1553 bdrv_register(&bdrv_gluster
);
1556 block_init(bdrv_gluster_init
);