2 * GlusterFS backend for QEMU
4 * Copyright (C) 2012 Bharata B Rao <bharata@linux.vnet.ibm.com>
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
10 #include "qemu/osdep.h"
11 #include <glusterfs/api/glfs.h>
12 #include "block/block_int.h"
13 #include "qapi/error.h"
14 #include "qapi/qmp/qerror.h"
16 #include "qemu/error-report.h"
17 #include "qemu/cutils.h"
19 #define GLUSTER_OPT_FILENAME "filename"
20 #define GLUSTER_OPT_VOLUME "volume"
21 #define GLUSTER_OPT_PATH "path"
22 #define GLUSTER_OPT_TYPE "type"
23 #define GLUSTER_OPT_SERVER_PATTERN "server."
24 #define GLUSTER_OPT_HOST "host"
25 #define GLUSTER_OPT_PORT "port"
26 #define GLUSTER_OPT_TO "to"
27 #define GLUSTER_OPT_IPV4 "ipv4"
28 #define GLUSTER_OPT_IPV6 "ipv6"
29 #define GLUSTER_OPT_SOCKET "socket"
30 #define GLUSTER_OPT_DEBUG "debug"
31 #define GLUSTER_DEFAULT_PORT 24007
32 #define GLUSTER_DEBUG_DEFAULT 4
33 #define GLUSTER_DEBUG_MAX 9
34 #define GLUSTER_OPT_LOGFILE "logfile"
35 #define GLUSTER_LOGFILE_DEFAULT "-" /* handled in libgfapi as /dev/stderr */
37 #define GERR_INDEX_HINT "hint: check in 'server' array index '%d'\n"
39 typedef struct GlusterAIOCB
{
43 AioContext
*aio_context
;
46 typedef struct BDRVGlusterState
{
50 bool supports_seek_data
;
54 typedef struct BDRVGlusterReopenState
{
57 } BDRVGlusterReopenState
;
60 typedef struct GlfsPreopened
{
66 typedef struct ListElement
{
67 QLIST_ENTRY(ListElement
) list
;
71 static QLIST_HEAD(glfs_list
, ListElement
) glfs_list
;
73 static QemuOptsList qemu_gluster_create_opts
= {
74 .name
= "qemu-gluster-create-opts",
75 .head
= QTAILQ_HEAD_INITIALIZER(qemu_gluster_create_opts
.head
),
78 .name
= BLOCK_OPT_SIZE
,
79 .type
= QEMU_OPT_SIZE
,
80 .help
= "Virtual disk size"
83 .name
= BLOCK_OPT_PREALLOC
,
84 .type
= QEMU_OPT_STRING
,
85 .help
= "Preallocation mode (allowed values: off, full)"
88 .name
= GLUSTER_OPT_DEBUG
,
89 .type
= QEMU_OPT_NUMBER
,
90 .help
= "Gluster log level, valid range is 0-9",
93 .name
= GLUSTER_OPT_LOGFILE
,
94 .type
= QEMU_OPT_STRING
,
95 .help
= "Logfile path of libgfapi",
101 static QemuOptsList runtime_opts
= {
103 .head
= QTAILQ_HEAD_INITIALIZER(runtime_opts
.head
),
106 .name
= GLUSTER_OPT_FILENAME
,
107 .type
= QEMU_OPT_STRING
,
108 .help
= "URL to the gluster image",
111 .name
= GLUSTER_OPT_DEBUG
,
112 .type
= QEMU_OPT_NUMBER
,
113 .help
= "Gluster log level, valid range is 0-9",
116 .name
= GLUSTER_OPT_LOGFILE
,
117 .type
= QEMU_OPT_STRING
,
118 .help
= "Logfile path of libgfapi",
120 { /* end of list */ }
124 static QemuOptsList runtime_json_opts
= {
125 .name
= "gluster_json",
126 .head
= QTAILQ_HEAD_INITIALIZER(runtime_json_opts
.head
),
129 .name
= GLUSTER_OPT_VOLUME
,
130 .type
= QEMU_OPT_STRING
,
131 .help
= "name of gluster volume where VM image resides",
134 .name
= GLUSTER_OPT_PATH
,
135 .type
= QEMU_OPT_STRING
,
136 .help
= "absolute path to image file in gluster volume",
139 .name
= GLUSTER_OPT_DEBUG
,
140 .type
= QEMU_OPT_NUMBER
,
141 .help
= "Gluster log level, valid range is 0-9",
143 { /* end of list */ }
147 static QemuOptsList runtime_type_opts
= {
148 .name
= "gluster_type",
149 .head
= QTAILQ_HEAD_INITIALIZER(runtime_type_opts
.head
),
152 .name
= GLUSTER_OPT_TYPE
,
153 .type
= QEMU_OPT_STRING
,
156 { /* end of list */ }
160 static QemuOptsList runtime_unix_opts
= {
161 .name
= "gluster_unix",
162 .head
= QTAILQ_HEAD_INITIALIZER(runtime_unix_opts
.head
),
165 .name
= GLUSTER_OPT_SOCKET
,
166 .type
= QEMU_OPT_STRING
,
167 .help
= "socket file path)",
169 { /* end of list */ }
173 static QemuOptsList runtime_tcp_opts
= {
174 .name
= "gluster_tcp",
175 .head
= QTAILQ_HEAD_INITIALIZER(runtime_tcp_opts
.head
),
178 .name
= GLUSTER_OPT_TYPE
,
179 .type
= QEMU_OPT_STRING
,
183 .name
= GLUSTER_OPT_HOST
,
184 .type
= QEMU_OPT_STRING
,
185 .help
= "host address (hostname/ipv4/ipv6 addresses)",
188 .name
= GLUSTER_OPT_PORT
,
189 .type
= QEMU_OPT_STRING
,
190 .help
= "port number on which glusterd is listening (default 24007)",
194 .type
= QEMU_OPT_NUMBER
,
195 .help
= "max port number, not supported by gluster",
199 .type
= QEMU_OPT_BOOL
,
200 .help
= "ipv4 bool value, not supported by gluster",
204 .type
= QEMU_OPT_BOOL
,
205 .help
= "ipv6 bool value, not supported by gluster",
207 { /* end of list */ }
211 static void glfs_set_preopened(const char *volume
, glfs_t
*fs
)
213 ListElement
*entry
= NULL
;
215 entry
= g_new(ListElement
, 1);
217 entry
->saved
.volume
= g_strdup(volume
);
219 entry
->saved
.fs
= fs
;
220 entry
->saved
.ref
= 1;
222 QLIST_INSERT_HEAD(&glfs_list
, entry
, list
);
225 static glfs_t
*glfs_find_preopened(const char *volume
)
227 ListElement
*entry
= NULL
;
229 QLIST_FOREACH(entry
, &glfs_list
, list
) {
230 if (strcmp(entry
->saved
.volume
, volume
) == 0) {
232 return entry
->saved
.fs
;
239 static void glfs_clear_preopened(glfs_t
*fs
)
241 ListElement
*entry
= NULL
;
247 QLIST_FOREACH(entry
, &glfs_list
, list
) {
248 if (entry
->saved
.fs
== fs
) {
249 if (--entry
->saved
.ref
) {
253 QLIST_REMOVE(entry
, list
);
255 glfs_fini(entry
->saved
.fs
);
256 g_free(entry
->saved
.volume
);
262 static int parse_volume_options(BlockdevOptionsGluster
*gconf
, char *path
)
271 p
= q
= path
+ strspn(path
, "/");
272 p
+= strcspn(p
, "/");
276 gconf
->volume
= g_strndup(q
, p
- q
);
283 gconf
->path
= g_strdup(p
);
288 * file=gluster[+transport]://[host[:port]]/volume/path[?socket=...]
290 * 'gluster' is the protocol.
292 * 'transport' specifies the transport type used to connect to gluster
293 * management daemon (glusterd). Valid transport types are
294 * tcp or unix. If a transport type isn't specified, then tcp type is assumed.
296 * 'host' specifies the host where the volume file specification for
297 * the given volume resides. This can be either hostname or ipv4 address.
298 * If transport type is 'unix', then 'host' field should not be specified.
299 * The 'socket' field needs to be populated with the path to unix domain
302 * 'port' is the port number on which glusterd is listening. This is optional
303 * and if not specified, QEMU will send 0 which will make gluster to use the
304 * default port. If the transport type is unix, then 'port' should not be
307 * 'volume' is the name of the gluster volume which contains the VM image.
309 * 'path' is the path to the actual VM image that resides on gluster volume.
313 * file=gluster://1.2.3.4/testvol/a.img
314 * file=gluster+tcp://1.2.3.4/testvol/a.img
315 * file=gluster+tcp://1.2.3.4:24007/testvol/dir/a.img
316 * file=gluster+tcp://host.domain.com:24007/testvol/dir/a.img
317 * file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket
319 static int qemu_gluster_parse_uri(BlockdevOptionsGluster
*gconf
,
320 const char *filename
)
322 GlusterServer
*gsconf
;
324 QueryParams
*qp
= NULL
;
325 bool is_unix
= false;
328 uri
= uri_parse(filename
);
333 gconf
->server
= g_new0(GlusterServerList
, 1);
334 gconf
->server
->value
= gsconf
= g_new0(GlusterServer
, 1);
337 if (!uri
->scheme
|| !strcmp(uri
->scheme
, "gluster")) {
338 gsconf
->type
= GLUSTER_TRANSPORT_TCP
;
339 } else if (!strcmp(uri
->scheme
, "gluster+tcp")) {
340 gsconf
->type
= GLUSTER_TRANSPORT_TCP
;
341 } else if (!strcmp(uri
->scheme
, "gluster+unix")) {
342 gsconf
->type
= GLUSTER_TRANSPORT_UNIX
;
344 } else if (!strcmp(uri
->scheme
, "gluster+rdma")) {
345 gsconf
->type
= GLUSTER_TRANSPORT_TCP
;
346 error_report("Warning: rdma feature is not supported, falling "
353 ret
= parse_volume_options(gconf
, uri
->path
);
358 qp
= query_params_parse(uri
->query
);
359 if (qp
->n
> 1 || (is_unix
&& !qp
->n
) || (!is_unix
&& qp
->n
)) {
365 if (uri
->server
|| uri
->port
) {
369 if (strcmp(qp
->p
[0].name
, "socket")) {
373 gsconf
->u
.q_unix
.path
= g_strdup(qp
->p
[0].value
);
375 gsconf
->u
.tcp
.host
= g_strdup(uri
->server
? uri
->server
: "localhost");
377 gsconf
->u
.tcp
.port
= g_strdup_printf("%d", uri
->port
);
379 gsconf
->u
.tcp
.port
= g_strdup_printf("%d", GLUSTER_DEFAULT_PORT
);
385 query_params_free(qp
);
391 static struct glfs
*qemu_gluster_glfs_init(BlockdevOptionsGluster
*gconf
,
397 GlusterServerList
*server
;
398 unsigned long long port
;
400 glfs
= glfs_find_preopened(gconf
->volume
);
405 glfs
= glfs_new(gconf
->volume
);
410 glfs_set_preopened(gconf
->volume
, glfs
);
412 for (server
= gconf
->server
; server
; server
= server
->next
) {
413 if (server
->value
->type
== GLUSTER_TRANSPORT_UNIX
) {
414 ret
= glfs_set_volfile_server(glfs
,
415 GlusterTransport_lookup
[server
->value
->type
],
416 server
->value
->u
.q_unix
.path
, 0);
418 if (parse_uint_full(server
->value
->u
.tcp
.port
, &port
, 10) < 0 ||
420 error_setg(errp
, "'%s' is not a valid port number",
421 server
->value
->u
.tcp
.port
);
425 ret
= glfs_set_volfile_server(glfs
,
426 GlusterTransport_lookup
[server
->value
->type
],
427 server
->value
->u
.tcp
.host
,
436 ret
= glfs_set_logging(glfs
, gconf
->logfile
, gconf
->debug_level
);
441 ret
= glfs_init(glfs
);
443 error_setg(errp
, "Gluster connection for volume %s, path %s failed"
444 " to connect", gconf
->volume
, gconf
->path
);
445 for (server
= gconf
->server
; server
; server
= server
->next
) {
446 if (server
->value
->type
== GLUSTER_TRANSPORT_UNIX
) {
447 error_append_hint(errp
, "hint: failed on socket %s ",
448 server
->value
->u
.q_unix
.path
);
450 error_append_hint(errp
, "hint: failed on host %s and port %s ",
451 server
->value
->u
.tcp
.host
,
452 server
->value
->u
.tcp
.port
);
456 error_append_hint(errp
, "Please refer to gluster logs for more info\n");
458 /* glfs_init sometimes doesn't set errno although docs suggest that */
470 glfs_clear_preopened(glfs
);
476 static int qapi_enum_parse(const char *opt
)
481 return GLUSTER_TRANSPORT__MAX
;
484 for (i
= 0; i
< GLUSTER_TRANSPORT__MAX
; i
++) {
485 if (!strcmp(opt
, GlusterTransport_lookup
[i
])) {
494 * Convert the json formatted command line into qapi.
496 static int qemu_gluster_parse_json(BlockdevOptionsGluster
*gconf
,
497 QDict
*options
, Error
**errp
)
500 GlusterServer
*gsconf
;
501 GlusterServerList
*curr
= NULL
;
502 QDict
*backing_options
= NULL
;
503 Error
*local_err
= NULL
;
509 /* create opts info from runtime_json_opts list */
510 opts
= qemu_opts_create(&runtime_json_opts
, NULL
, 0, &error_abort
);
511 qemu_opts_absorb_qdict(opts
, options
, &local_err
);
516 num_servers
= qdict_array_entries(options
, GLUSTER_OPT_SERVER_PATTERN
);
517 if (num_servers
< 1) {
518 error_setg(&local_err
, QERR_MISSING_PARAMETER
, "server");
522 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_VOLUME
);
524 error_setg(&local_err
, QERR_MISSING_PARAMETER
, GLUSTER_OPT_VOLUME
);
527 gconf
->volume
= g_strdup(ptr
);
529 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_PATH
);
531 error_setg(&local_err
, QERR_MISSING_PARAMETER
, GLUSTER_OPT_PATH
);
534 gconf
->path
= g_strdup(ptr
);
537 for (i
= 0; i
< num_servers
; i
++) {
538 str
= g_strdup_printf(GLUSTER_OPT_SERVER_PATTERN
"%d.", i
);
539 qdict_extract_subqdict(options
, &backing_options
, str
);
541 /* create opts info from runtime_type_opts list */
542 opts
= qemu_opts_create(&runtime_type_opts
, NULL
, 0, &error_abort
);
543 qemu_opts_absorb_qdict(opts
, backing_options
, &local_err
);
548 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_TYPE
);
549 gsconf
= g_new0(GlusterServer
, 1);
550 gsconf
->type
= qapi_enum_parse(ptr
);
552 error_setg(&local_err
, QERR_MISSING_PARAMETER
, GLUSTER_OPT_TYPE
);
553 error_append_hint(&local_err
, GERR_INDEX_HINT
, i
);
557 if (gsconf
->type
== GLUSTER_TRANSPORT__MAX
) {
558 error_setg(&local_err
, QERR_INVALID_PARAMETER_VALUE
,
559 GLUSTER_OPT_TYPE
, "tcp or unix");
560 error_append_hint(&local_err
, GERR_INDEX_HINT
, i
);
565 if (gsconf
->type
== GLUSTER_TRANSPORT_TCP
) {
566 /* create opts info from runtime_tcp_opts list */
567 opts
= qemu_opts_create(&runtime_tcp_opts
, NULL
, 0, &error_abort
);
568 qemu_opts_absorb_qdict(opts
, backing_options
, &local_err
);
573 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_HOST
);
575 error_setg(&local_err
, QERR_MISSING_PARAMETER
,
577 error_append_hint(&local_err
, GERR_INDEX_HINT
, i
);
580 gsconf
->u
.tcp
.host
= g_strdup(ptr
);
581 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_PORT
);
583 error_setg(&local_err
, QERR_MISSING_PARAMETER
,
585 error_append_hint(&local_err
, GERR_INDEX_HINT
, i
);
588 gsconf
->u
.tcp
.port
= g_strdup(ptr
);
590 /* defend for unsupported fields in InetSocketAddress,
591 * i.e. @ipv4, @ipv6 and @to
593 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_TO
);
595 gsconf
->u
.tcp
.has_to
= true;
597 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_IPV4
);
599 gsconf
->u
.tcp
.has_ipv4
= true;
601 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_IPV6
);
603 gsconf
->u
.tcp
.has_ipv6
= true;
605 if (gsconf
->u
.tcp
.has_to
) {
606 error_setg(&local_err
, "Parameter 'to' not supported");
609 if (gsconf
->u
.tcp
.has_ipv4
|| gsconf
->u
.tcp
.has_ipv6
) {
610 error_setg(&local_err
, "Parameters 'ipv4/ipv6' not supported");
615 /* create opts info from runtime_unix_opts list */
616 opts
= qemu_opts_create(&runtime_unix_opts
, NULL
, 0, &error_abort
);
617 qemu_opts_absorb_qdict(opts
, backing_options
, &local_err
);
622 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_SOCKET
);
624 error_setg(&local_err
, QERR_MISSING_PARAMETER
,
626 error_append_hint(&local_err
, GERR_INDEX_HINT
, i
);
629 gsconf
->u
.q_unix
.path
= g_strdup(ptr
);
633 if (gconf
->server
== NULL
) {
634 gconf
->server
= g_new0(GlusterServerList
, 1);
635 gconf
->server
->value
= gsconf
;
636 curr
= gconf
->server
;
638 curr
->next
= g_new0(GlusterServerList
, 1);
639 curr
->next
->value
= gsconf
;
643 qdict_del(backing_options
, str
);
651 error_propagate(errp
, local_err
);
654 qdict_del(backing_options
, str
);
661 static struct glfs
*qemu_gluster_init(BlockdevOptionsGluster
*gconf
,
662 const char *filename
,
663 QDict
*options
, Error
**errp
)
667 ret
= qemu_gluster_parse_uri(gconf
, filename
);
669 error_setg(errp
, "invalid URI");
670 error_append_hint(errp
, "Usage: file=gluster[+transport]://"
671 "[host[:port]]volume/path[?socket=...]"
673 "[,file.logfile=/path/filename.log]\n");
678 ret
= qemu_gluster_parse_json(gconf
, options
, errp
);
680 error_append_hint(errp
, "Usage: "
681 "-drive driver=qcow2,file.driver=gluster,"
682 "file.volume=testvol,file.path=/path/a.qcow2"
684 "[,file.logfile=/path/filename.log],"
685 "file.server.0.type=tcp,"
686 "file.server.0.host=1.2.3.4,"
687 "file.server.0.port=24007,"
688 "file.server.1.transport=unix,"
689 "file.server.1.socket=/var/run/glusterd.socket ..."
697 return qemu_gluster_glfs_init(gconf
, errp
);
700 static void qemu_gluster_complete_aio(void *opaque
)
702 GlusterAIOCB
*acb
= (GlusterAIOCB
*)opaque
;
704 qemu_coroutine_enter(acb
->coroutine
);
708 * AIO callback routine called from GlusterFS thread.
710 static void gluster_finish_aiocb(struct glfs_fd
*fd
, ssize_t ret
, void *arg
)
712 GlusterAIOCB
*acb
= (GlusterAIOCB
*)arg
;
714 if (!ret
|| ret
== acb
->size
) {
715 acb
->ret
= 0; /* Success */
716 } else if (ret
< 0) {
717 acb
->ret
= -errno
; /* Read/Write failed */
719 acb
->ret
= -EIO
; /* Partial read/write - fail it */
722 aio_bh_schedule_oneshot(acb
->aio_context
, qemu_gluster_complete_aio
, acb
);
725 static void qemu_gluster_parse_flags(int bdrv_flags
, int *open_flags
)
727 assert(open_flags
!= NULL
);
729 *open_flags
|= O_BINARY
;
731 if (bdrv_flags
& BDRV_O_RDWR
) {
732 *open_flags
|= O_RDWR
;
734 *open_flags
|= O_RDONLY
;
737 if ((bdrv_flags
& BDRV_O_NOCACHE
)) {
738 *open_flags
|= O_DIRECT
;
743 * Do SEEK_DATA/HOLE to detect if it is functional. Older broken versions of
744 * gfapi incorrectly return the current offset when SEEK_DATA/HOLE is used.
745 * - Corrected versions return -1 and set errno to EINVAL.
746 * - Versions that support SEEK_DATA/HOLE correctly, will return -1 and set
747 * errno to ENXIO when SEEK_DATA is called with a position of EOF.
749 static bool qemu_gluster_test_seek(struct glfs_fd
*fd
)
753 #if defined SEEK_HOLE && defined SEEK_DATA
756 eof
= glfs_lseek(fd
, 0, SEEK_END
);
758 /* this should never occur */
762 /* this should always fail with ENXIO if SEEK_DATA is supported */
763 ret
= glfs_lseek(fd
, eof
, SEEK_DATA
);
766 return (ret
< 0) && (errno
== ENXIO
);
769 static int qemu_gluster_open(BlockDriverState
*bs
, QDict
*options
,
770 int bdrv_flags
, Error
**errp
)
772 BDRVGlusterState
*s
= bs
->opaque
;
775 BlockdevOptionsGluster
*gconf
= NULL
;
777 Error
*local_err
= NULL
;
778 const char *filename
, *logfile
;
780 opts
= qemu_opts_create(&runtime_opts
, NULL
, 0, &error_abort
);
781 qemu_opts_absorb_qdict(opts
, options
, &local_err
);
783 error_propagate(errp
, local_err
);
788 filename
= qemu_opt_get(opts
, GLUSTER_OPT_FILENAME
);
790 s
->debug_level
= qemu_opt_get_number(opts
, GLUSTER_OPT_DEBUG
,
791 GLUSTER_DEBUG_DEFAULT
);
792 if (s
->debug_level
< 0) {
794 } else if (s
->debug_level
> GLUSTER_DEBUG_MAX
) {
795 s
->debug_level
= GLUSTER_DEBUG_MAX
;
798 gconf
= g_new0(BlockdevOptionsGluster
, 1);
799 gconf
->debug_level
= s
->debug_level
;
800 gconf
->has_debug_level
= true;
802 logfile
= qemu_opt_get(opts
, GLUSTER_OPT_LOGFILE
);
803 s
->logfile
= g_strdup(logfile
? logfile
: GLUSTER_LOGFILE_DEFAULT
);
805 gconf
->logfile
= g_strdup(s
->logfile
);
806 gconf
->has_logfile
= true;
808 s
->glfs
= qemu_gluster_init(gconf
, filename
, options
, errp
);
814 #ifdef CONFIG_GLUSTERFS_XLATOR_OPT
815 /* Without this, if fsync fails for a recoverable reason (for instance,
816 * ENOSPC), gluster will dump its cache, preventing retries. This means
817 * almost certain data loss. Not all gluster versions support the
818 * 'resync-failed-syncs-after-fsync' key value, but there is no way to
819 * discover during runtime if it is supported (this api returns success for
820 * unknown key/value pairs) */
821 ret
= glfs_set_xlator_option(s
->glfs
, "*-write-behind",
822 "resync-failed-syncs-after-fsync",
825 error_setg_errno(errp
, errno
, "Unable to set xlator key/value pair");
831 qemu_gluster_parse_flags(bdrv_flags
, &open_flags
);
833 s
->fd
= glfs_open(s
->glfs
, gconf
->path
, open_flags
);
838 s
->supports_seek_data
= qemu_gluster_test_seek(s
->fd
);
842 qapi_free_BlockdevOptionsGluster(gconf
);
851 glfs_clear_preopened(s
->glfs
);
856 static int qemu_gluster_reopen_prepare(BDRVReopenState
*state
,
857 BlockReopenQueue
*queue
, Error
**errp
)
861 BDRVGlusterReopenState
*reop_s
;
862 BlockdevOptionsGluster
*gconf
;
865 assert(state
!= NULL
);
866 assert(state
->bs
!= NULL
);
868 s
= state
->bs
->opaque
;
870 state
->opaque
= g_new0(BDRVGlusterReopenState
, 1);
871 reop_s
= state
->opaque
;
873 qemu_gluster_parse_flags(state
->flags
, &open_flags
);
875 gconf
= g_new0(BlockdevOptionsGluster
, 1);
876 gconf
->debug_level
= s
->debug_level
;
877 gconf
->has_debug_level
= true;
878 gconf
->logfile
= g_strdup(s
->logfile
);
879 gconf
->has_logfile
= true;
880 reop_s
->glfs
= qemu_gluster_init(gconf
, state
->bs
->filename
, NULL
, errp
);
881 if (reop_s
->glfs
== NULL
) {
886 #ifdef CONFIG_GLUSTERFS_XLATOR_OPT
887 ret
= glfs_set_xlator_option(reop_s
->glfs
, "*-write-behind",
888 "resync-failed-syncs-after-fsync", "on");
890 error_setg_errno(errp
, errno
, "Unable to set xlator key/value pair");
896 reop_s
->fd
= glfs_open(reop_s
->glfs
, gconf
->path
, open_flags
);
897 if (reop_s
->fd
== NULL
) {
898 /* reops->glfs will be cleaned up in _abort */
904 /* state->opaque will be freed in either the _abort or _commit */
905 qapi_free_BlockdevOptionsGluster(gconf
);
909 static void qemu_gluster_reopen_commit(BDRVReopenState
*state
)
911 BDRVGlusterReopenState
*reop_s
= state
->opaque
;
912 BDRVGlusterState
*s
= state
->bs
->opaque
;
920 glfs_clear_preopened(s
->glfs
);
922 /* use the newly opened image / connection */
924 s
->glfs
= reop_s
->glfs
;
926 g_free(state
->opaque
);
927 state
->opaque
= NULL
;
933 static void qemu_gluster_reopen_abort(BDRVReopenState
*state
)
935 BDRVGlusterReopenState
*reop_s
= state
->opaque
;
937 if (reop_s
== NULL
) {
942 glfs_close(reop_s
->fd
);
945 glfs_clear_preopened(reop_s
->glfs
);
947 g_free(state
->opaque
);
948 state
->opaque
= NULL
;
953 #ifdef CONFIG_GLUSTERFS_ZEROFILL
954 static coroutine_fn
int qemu_gluster_co_pwrite_zeroes(BlockDriverState
*bs
,
957 BdrvRequestFlags flags
)
961 BDRVGlusterState
*s
= bs
->opaque
;
965 acb
.coroutine
= qemu_coroutine_self();
966 acb
.aio_context
= bdrv_get_aio_context(bs
);
968 ret
= glfs_zerofill_async(s
->fd
, offset
, size
, gluster_finish_aiocb
, &acb
);
973 qemu_coroutine_yield();
977 static inline bool gluster_supports_zerofill(void)
982 static inline int qemu_gluster_zerofill(struct glfs_fd
*fd
, int64_t offset
,
985 return glfs_zerofill(fd
, offset
, size
);
989 static inline bool gluster_supports_zerofill(void)
994 static inline int qemu_gluster_zerofill(struct glfs_fd
*fd
, int64_t offset
,
1001 static int qemu_gluster_create(const char *filename
,
1002 QemuOpts
*opts
, Error
**errp
)
1004 BlockdevOptionsGluster
*gconf
;
1009 int64_t total_size
= 0;
1012 gconf
= g_new0(BlockdevOptionsGluster
, 1);
1013 gconf
->debug_level
= qemu_opt_get_number_del(opts
, GLUSTER_OPT_DEBUG
,
1014 GLUSTER_DEBUG_DEFAULT
);
1015 if (gconf
->debug_level
< 0) {
1016 gconf
->debug_level
= 0;
1017 } else if (gconf
->debug_level
> GLUSTER_DEBUG_MAX
) {
1018 gconf
->debug_level
= GLUSTER_DEBUG_MAX
;
1020 gconf
->has_debug_level
= true;
1022 gconf
->logfile
= qemu_opt_get_del(opts
, GLUSTER_OPT_LOGFILE
);
1023 if (!gconf
->logfile
) {
1024 gconf
->logfile
= g_strdup(GLUSTER_LOGFILE_DEFAULT
);
1026 gconf
->has_logfile
= true;
1028 glfs
= qemu_gluster_init(gconf
, filename
, NULL
, errp
);
1034 total_size
= ROUND_UP(qemu_opt_get_size_del(opts
, BLOCK_OPT_SIZE
, 0),
1037 tmp
= qemu_opt_get_del(opts
, BLOCK_OPT_PREALLOC
);
1038 if (!tmp
|| !strcmp(tmp
, "off")) {
1040 } else if (!strcmp(tmp
, "full") && gluster_supports_zerofill()) {
1043 error_setg(errp
, "Invalid preallocation mode: '%s'"
1044 " or GlusterFS doesn't support zerofill API", tmp
);
1049 fd
= glfs_creat(glfs
, gconf
->path
,
1050 O_WRONLY
| O_CREAT
| O_TRUNC
| O_BINARY
, S_IRUSR
| S_IWUSR
);
1054 if (!glfs_ftruncate(fd
, total_size
)) {
1055 if (prealloc
&& qemu_gluster_zerofill(fd
, 0, total_size
)) {
1062 if (glfs_close(fd
) != 0) {
1068 qapi_free_BlockdevOptionsGluster(gconf
);
1069 glfs_clear_preopened(glfs
);
1073 static coroutine_fn
int qemu_gluster_co_rw(BlockDriverState
*bs
,
1074 int64_t sector_num
, int nb_sectors
,
1075 QEMUIOVector
*qiov
, int write
)
1079 BDRVGlusterState
*s
= bs
->opaque
;
1080 size_t size
= nb_sectors
* BDRV_SECTOR_SIZE
;
1081 off_t offset
= sector_num
* BDRV_SECTOR_SIZE
;
1085 acb
.coroutine
= qemu_coroutine_self();
1086 acb
.aio_context
= bdrv_get_aio_context(bs
);
1089 ret
= glfs_pwritev_async(s
->fd
, qiov
->iov
, qiov
->niov
, offset
, 0,
1090 gluster_finish_aiocb
, &acb
);
1092 ret
= glfs_preadv_async(s
->fd
, qiov
->iov
, qiov
->niov
, offset
, 0,
1093 gluster_finish_aiocb
, &acb
);
1100 qemu_coroutine_yield();
1104 static int qemu_gluster_truncate(BlockDriverState
*bs
, int64_t offset
)
1107 BDRVGlusterState
*s
= bs
->opaque
;
1109 ret
= glfs_ftruncate(s
->fd
, offset
);
1117 static coroutine_fn
int qemu_gluster_co_readv(BlockDriverState
*bs
,
1122 return qemu_gluster_co_rw(bs
, sector_num
, nb_sectors
, qiov
, 0);
1125 static coroutine_fn
int qemu_gluster_co_writev(BlockDriverState
*bs
,
1130 return qemu_gluster_co_rw(bs
, sector_num
, nb_sectors
, qiov
, 1);
1133 static void qemu_gluster_close(BlockDriverState
*bs
)
1135 BDRVGlusterState
*s
= bs
->opaque
;
1142 glfs_clear_preopened(s
->glfs
);
1145 static coroutine_fn
int qemu_gluster_co_flush_to_disk(BlockDriverState
*bs
)
1149 BDRVGlusterState
*s
= bs
->opaque
;
1153 acb
.coroutine
= qemu_coroutine_self();
1154 acb
.aio_context
= bdrv_get_aio_context(bs
);
1156 ret
= glfs_fsync_async(s
->fd
, gluster_finish_aiocb
, &acb
);
1162 qemu_coroutine_yield();
1171 /* Some versions of Gluster (3.5.6 -> 3.5.8?) will not retain its cache
1172 * after a fsync failure, so we have no way of allowing the guest to safely
1173 * continue. Gluster versions prior to 3.5.6 don't retain the cache
1174 * either, but will invalidate the fd on error, so this is again our only
1177 * The 'resync-failed-syncs-after-fsync' xlator option for the
1178 * write-behind cache will cause later gluster versions to retain its
1179 * cache after error, so long as the fd remains open. However, we
1180 * currently have no way of knowing if this option is supported.
1182 * TODO: Once gluster provides a way for us to determine if the option
1183 * is supported, bypass the closure and setting drv to NULL. */
1184 qemu_gluster_close(bs
);
1189 #ifdef CONFIG_GLUSTERFS_DISCARD
1190 static coroutine_fn
int qemu_gluster_co_pdiscard(BlockDriverState
*bs
,
1191 int64_t offset
, int size
)
1195 BDRVGlusterState
*s
= bs
->opaque
;
1199 acb
.coroutine
= qemu_coroutine_self();
1200 acb
.aio_context
= bdrv_get_aio_context(bs
);
1202 ret
= glfs_discard_async(s
->fd
, offset
, size
, gluster_finish_aiocb
, &acb
);
1207 qemu_coroutine_yield();
1212 static int64_t qemu_gluster_getlength(BlockDriverState
*bs
)
1214 BDRVGlusterState
*s
= bs
->opaque
;
1217 ret
= glfs_lseek(s
->fd
, 0, SEEK_END
);
1225 static int64_t qemu_gluster_allocated_file_size(BlockDriverState
*bs
)
1227 BDRVGlusterState
*s
= bs
->opaque
;
1231 ret
= glfs_fstat(s
->fd
, &st
);
1235 return st
.st_blocks
* 512;
1239 static int qemu_gluster_has_zero_init(BlockDriverState
*bs
)
1241 /* GlusterFS volume could be backed by a block device */
1246 * Find allocation range in @bs around offset @start.
1247 * May change underlying file descriptor's file offset.
1248 * If @start is not in a hole, store @start in @data, and the
1249 * beginning of the next hole in @hole, and return 0.
1250 * If @start is in a non-trailing hole, store @start in @hole and the
1251 * beginning of the next non-hole in @data, and return 0.
1252 * If @start is in a trailing hole or beyond EOF, return -ENXIO.
1253 * If we can't find out, return a negative errno other than -ENXIO.
1255 * (Shamefully copied from raw-posix.c, only miniscule adaptions.)
1257 static int find_allocation(BlockDriverState
*bs
, off_t start
,
1258 off_t
*data
, off_t
*hole
)
1260 BDRVGlusterState
*s
= bs
->opaque
;
1262 if (!s
->supports_seek_data
) {
1266 #if defined SEEK_HOLE && defined SEEK_DATA
1271 * D1. offs == start: start is in data
1272 * D2. offs > start: start is in a hole, next data at offs
1273 * D3. offs < 0, errno = ENXIO: either start is in a trailing hole
1274 * or start is beyond EOF
1275 * If the latter happens, the file has been truncated behind
1276 * our back since we opened it. All bets are off then.
1277 * Treating like a trailing hole is simplest.
1278 * D4. offs < 0, errno != ENXIO: we learned nothing
1280 offs
= glfs_lseek(s
->fd
, start
, SEEK_DATA
);
1282 return -errno
; /* D3 or D4 */
1284 assert(offs
>= start
);
1287 /* D2: in hole, next data at offs */
1293 /* D1: in data, end not yet known */
1297 * H1. offs == start: start is in a hole
1298 * If this happens here, a hole has been dug behind our back
1299 * since the previous lseek().
1300 * H2. offs > start: either start is in data, next hole at offs,
1301 * or start is in trailing hole, EOF at offs
1302 * Linux treats trailing holes like any other hole: offs ==
1303 * start. Solaris seeks to EOF instead: offs > start (blech).
1304 * If that happens here, a hole has been dug behind our back
1305 * since the previous lseek().
1306 * H3. offs < 0, errno = ENXIO: start is beyond EOF
1307 * If this happens, the file has been truncated behind our
1308 * back since we opened it. Treat it like a trailing hole.
1309 * H4. offs < 0, errno != ENXIO: we learned nothing
1310 * Pretend we know nothing at all, i.e. "forget" about D1.
1312 offs
= glfs_lseek(s
->fd
, start
, SEEK_HOLE
);
1314 return -errno
; /* D1 and (H3 or H4) */
1316 assert(offs
>= start
);
1320 * D1 and H2: either in data, next hole at offs, or it was in
1321 * data but is now in a trailing hole. In the latter case,
1322 * all bets are off. Treating it as if it there was data all
1323 * the way to EOF is safe, so simply do that.
1339 * Returns the allocation status of the specified sectors.
1341 * If 'sector_num' is beyond the end of the disk image the return value is 0
1342 * and 'pnum' is set to 0.
1344 * 'pnum' is set to the number of sectors (including and immediately following
1345 * the specified sector) that are known to be in the same
1346 * allocated/unallocated state.
1348 * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes
1349 * beyond the end of the disk image it will be clamped.
1351 * (Based on raw_co_get_block_status() from raw-posix.c.)
1353 static int64_t coroutine_fn
qemu_gluster_co_get_block_status(
1354 BlockDriverState
*bs
, int64_t sector_num
, int nb_sectors
, int *pnum
,
1355 BlockDriverState
**file
)
1357 BDRVGlusterState
*s
= bs
->opaque
;
1358 off_t start
, data
= 0, hole
= 0;
1366 start
= sector_num
* BDRV_SECTOR_SIZE
;
1367 total_size
= bdrv_getlength(bs
);
1368 if (total_size
< 0) {
1370 } else if (start
>= total_size
) {
1373 } else if (start
+ nb_sectors
* BDRV_SECTOR_SIZE
> total_size
) {
1374 nb_sectors
= DIV_ROUND_UP(total_size
- start
, BDRV_SECTOR_SIZE
);
1377 ret
= find_allocation(bs
, start
, &data
, &hole
);
1378 if (ret
== -ENXIO
) {
1381 ret
= BDRV_BLOCK_ZERO
;
1382 } else if (ret
< 0) {
1383 /* No info available, so pretend there are no holes */
1385 ret
= BDRV_BLOCK_DATA
;
1386 } else if (data
== start
) {
1387 /* On a data extent, compute sectors to the end of the extent,
1388 * possibly including a partial sector at EOF. */
1389 *pnum
= MIN(nb_sectors
, DIV_ROUND_UP(hole
- start
, BDRV_SECTOR_SIZE
));
1390 ret
= BDRV_BLOCK_DATA
;
1392 /* On a hole, compute sectors to the beginning of the next extent. */
1393 assert(hole
== start
);
1394 *pnum
= MIN(nb_sectors
, (data
- start
) / BDRV_SECTOR_SIZE
);
1395 ret
= BDRV_BLOCK_ZERO
;
1400 return ret
| BDRV_BLOCK_OFFSET_VALID
| start
;
1404 static BlockDriver bdrv_gluster
= {
1405 .format_name
= "gluster",
1406 .protocol_name
= "gluster",
1407 .instance_size
= sizeof(BDRVGlusterState
),
1408 .bdrv_needs_filename
= false,
1409 .bdrv_file_open
= qemu_gluster_open
,
1410 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
1411 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
1412 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
1413 .bdrv_close
= qemu_gluster_close
,
1414 .bdrv_create
= qemu_gluster_create
,
1415 .bdrv_getlength
= qemu_gluster_getlength
,
1416 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
1417 .bdrv_truncate
= qemu_gluster_truncate
,
1418 .bdrv_co_readv
= qemu_gluster_co_readv
,
1419 .bdrv_co_writev
= qemu_gluster_co_writev
,
1420 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
1421 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
1422 #ifdef CONFIG_GLUSTERFS_DISCARD
1423 .bdrv_co_pdiscard
= qemu_gluster_co_pdiscard
,
1425 #ifdef CONFIG_GLUSTERFS_ZEROFILL
1426 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
1428 .bdrv_co_get_block_status
= qemu_gluster_co_get_block_status
,
1429 .create_opts
= &qemu_gluster_create_opts
,
1432 static BlockDriver bdrv_gluster_tcp
= {
1433 .format_name
= "gluster",
1434 .protocol_name
= "gluster+tcp",
1435 .instance_size
= sizeof(BDRVGlusterState
),
1436 .bdrv_needs_filename
= false,
1437 .bdrv_file_open
= qemu_gluster_open
,
1438 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
1439 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
1440 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
1441 .bdrv_close
= qemu_gluster_close
,
1442 .bdrv_create
= qemu_gluster_create
,
1443 .bdrv_getlength
= qemu_gluster_getlength
,
1444 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
1445 .bdrv_truncate
= qemu_gluster_truncate
,
1446 .bdrv_co_readv
= qemu_gluster_co_readv
,
1447 .bdrv_co_writev
= qemu_gluster_co_writev
,
1448 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
1449 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
1450 #ifdef CONFIG_GLUSTERFS_DISCARD
1451 .bdrv_co_pdiscard
= qemu_gluster_co_pdiscard
,
1453 #ifdef CONFIG_GLUSTERFS_ZEROFILL
1454 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
1456 .bdrv_co_get_block_status
= qemu_gluster_co_get_block_status
,
1457 .create_opts
= &qemu_gluster_create_opts
,
1460 static BlockDriver bdrv_gluster_unix
= {
1461 .format_name
= "gluster",
1462 .protocol_name
= "gluster+unix",
1463 .instance_size
= sizeof(BDRVGlusterState
),
1464 .bdrv_needs_filename
= true,
1465 .bdrv_file_open
= qemu_gluster_open
,
1466 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
1467 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
1468 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
1469 .bdrv_close
= qemu_gluster_close
,
1470 .bdrv_create
= qemu_gluster_create
,
1471 .bdrv_getlength
= qemu_gluster_getlength
,
1472 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
1473 .bdrv_truncate
= qemu_gluster_truncate
,
1474 .bdrv_co_readv
= qemu_gluster_co_readv
,
1475 .bdrv_co_writev
= qemu_gluster_co_writev
,
1476 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
1477 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
1478 #ifdef CONFIG_GLUSTERFS_DISCARD
1479 .bdrv_co_pdiscard
= qemu_gluster_co_pdiscard
,
1481 #ifdef CONFIG_GLUSTERFS_ZEROFILL
1482 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
1484 .bdrv_co_get_block_status
= qemu_gluster_co_get_block_status
,
1485 .create_opts
= &qemu_gluster_create_opts
,
1488 /* rdma is deprecated (actually never supported for volfile fetch).
1489 * Let's maintain it for the protocol compatibility, to make sure things
1490 * won't break immediately. For now, gluster+rdma will fall back to gluster+tcp
1491 * protocol with a warning.
1492 * TODO: remove gluster+rdma interface support
1494 static BlockDriver bdrv_gluster_rdma
= {
1495 .format_name
= "gluster",
1496 .protocol_name
= "gluster+rdma",
1497 .instance_size
= sizeof(BDRVGlusterState
),
1498 .bdrv_needs_filename
= true,
1499 .bdrv_file_open
= qemu_gluster_open
,
1500 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
1501 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
1502 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
1503 .bdrv_close
= qemu_gluster_close
,
1504 .bdrv_create
= qemu_gluster_create
,
1505 .bdrv_getlength
= qemu_gluster_getlength
,
1506 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
1507 .bdrv_truncate
= qemu_gluster_truncate
,
1508 .bdrv_co_readv
= qemu_gluster_co_readv
,
1509 .bdrv_co_writev
= qemu_gluster_co_writev
,
1510 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
1511 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
1512 #ifdef CONFIG_GLUSTERFS_DISCARD
1513 .bdrv_co_pdiscard
= qemu_gluster_co_pdiscard
,
1515 #ifdef CONFIG_GLUSTERFS_ZEROFILL
1516 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
1518 .bdrv_co_get_block_status
= qemu_gluster_co_get_block_status
,
1519 .create_opts
= &qemu_gluster_create_opts
,
1522 static void bdrv_gluster_init(void)
1524 bdrv_register(&bdrv_gluster_rdma
);
1525 bdrv_register(&bdrv_gluster_unix
);
1526 bdrv_register(&bdrv_gluster_tcp
);
1527 bdrv_register(&bdrv_gluster
);
1530 block_init(bdrv_gluster_init
);