2 * GlusterFS backend for QEMU
4 * Copyright (C) 2012 Bharata B Rao <bharata@linux.vnet.ibm.com>
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
10 #include "qemu/osdep.h"
11 #include <glusterfs/api/glfs.h>
12 #include "block/block_int.h"
13 #include "qapi/error.h"
16 typedef struct GlusterAIOCB
{
21 AioContext
*aio_context
;
24 typedef struct BDRVGlusterState
{
29 typedef struct GlusterConf
{
37 static void qemu_gluster_gconf_free(GlusterConf
*gconf
)
40 g_free(gconf
->server
);
41 g_free(gconf
->volname
);
43 g_free(gconf
->transport
);
48 static int parse_volume_options(GlusterConf
*gconf
, char *path
)
57 p
= q
= path
+ strspn(path
, "/");
62 gconf
->volname
= g_strndup(q
, p
- q
);
69 gconf
->image
= g_strdup(p
);
74 * file=gluster[+transport]://[server[:port]]/volname/image[?socket=...]
76 * 'gluster' is the protocol.
78 * 'transport' specifies the transport type used to connect to gluster
79 * management daemon (glusterd). Valid transport types are
80 * tcp, unix and rdma. If a transport type isn't specified, then tcp
83 * 'server' specifies the server where the volume file specification for
84 * the given volume resides. This can be either hostname, ipv4 address
85 * or ipv6 address. ipv6 address needs to be within square brackets [ ].
86 * If transport type is 'unix', then 'server' field should not be specified.
87 * The 'socket' field needs to be populated with the path to unix domain
90 * 'port' is the port number on which glusterd is listening. This is optional
91 * and if not specified, QEMU will send 0 which will make gluster to use the
92 * default port. If the transport type is unix, then 'port' should not be
95 * 'volname' is the name of the gluster volume which contains the VM image.
97 * 'image' is the path to the actual VM image that resides on gluster volume.
101 * file=gluster://1.2.3.4/testvol/a.img
102 * file=gluster+tcp://1.2.3.4/testvol/a.img
103 * file=gluster+tcp://1.2.3.4:24007/testvol/dir/a.img
104 * file=gluster+tcp://[1:2:3:4:5:6:7:8]/testvol/dir/a.img
105 * file=gluster+tcp://[1:2:3:4:5:6:7:8]:24007/testvol/dir/a.img
106 * file=gluster+tcp://server.domain.com:24007/testvol/dir/a.img
107 * file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket
108 * file=gluster+rdma://1.2.3.4:24007/testvol/a.img
110 static int qemu_gluster_parseuri(GlusterConf
*gconf
, const char *filename
)
113 QueryParams
*qp
= NULL
;
114 bool is_unix
= false;
117 uri
= uri_parse(filename
);
123 if (!uri
->scheme
|| !strcmp(uri
->scheme
, "gluster")) {
124 gconf
->transport
= g_strdup("tcp");
125 } else if (!strcmp(uri
->scheme
, "gluster+tcp")) {
126 gconf
->transport
= g_strdup("tcp");
127 } else if (!strcmp(uri
->scheme
, "gluster+unix")) {
128 gconf
->transport
= g_strdup("unix");
130 } else if (!strcmp(uri
->scheme
, "gluster+rdma")) {
131 gconf
->transport
= g_strdup("rdma");
137 ret
= parse_volume_options(gconf
, uri
->path
);
142 qp
= query_params_parse(uri
->query
);
143 if (qp
->n
> 1 || (is_unix
&& !qp
->n
) || (!is_unix
&& qp
->n
)) {
149 if (uri
->server
|| uri
->port
) {
153 if (strcmp(qp
->p
[0].name
, "socket")) {
157 gconf
->server
= g_strdup(qp
->p
[0].value
);
159 gconf
->server
= g_strdup(uri
->server
? uri
->server
: "localhost");
160 gconf
->port
= uri
->port
;
165 query_params_free(qp
);
171 static struct glfs
*qemu_gluster_init(GlusterConf
*gconf
, const char *filename
,
174 struct glfs
*glfs
= NULL
;
178 ret
= qemu_gluster_parseuri(gconf
, filename
);
180 error_setg(errp
, "Usage: file=gluster[+transport]://[server[:port]]/"
181 "volname/image[?socket=...]");
186 glfs
= glfs_new(gconf
->volname
);
191 ret
= glfs_set_volfile_server(glfs
, gconf
->transport
, gconf
->server
,
198 * TODO: Use GF_LOG_ERROR instead of hard code value of 4 here when
199 * GlusterFS makes GF_LOG_* macros available to libgfapi users.
201 ret
= glfs_set_logging(glfs
, "-", 4);
206 ret
= glfs_init(glfs
);
208 error_setg_errno(errp
, errno
,
209 "Gluster connection failed for server=%s port=%d "
210 "volume=%s image=%s transport=%s", gconf
->server
,
211 gconf
->port
, gconf
->volname
, gconf
->image
,
214 /* glfs_init sometimes doesn't set errno although docs suggest that */
231 static void qemu_gluster_complete_aio(void *opaque
)
233 GlusterAIOCB
*acb
= (GlusterAIOCB
*)opaque
;
235 qemu_bh_delete(acb
->bh
);
237 qemu_coroutine_enter(acb
->coroutine
, NULL
);
241 * AIO callback routine called from GlusterFS thread.
243 static void gluster_finish_aiocb(struct glfs_fd
*fd
, ssize_t ret
, void *arg
)
245 GlusterAIOCB
*acb
= (GlusterAIOCB
*)arg
;
247 if (!ret
|| ret
== acb
->size
) {
248 acb
->ret
= 0; /* Success */
249 } else if (ret
< 0) {
250 acb
->ret
= -errno
; /* Read/Write failed */
252 acb
->ret
= -EIO
; /* Partial read/write - fail it */
255 acb
->bh
= aio_bh_new(acb
->aio_context
, qemu_gluster_complete_aio
, acb
);
256 qemu_bh_schedule(acb
->bh
);
259 /* TODO Convert to fine grained options */
260 static QemuOptsList runtime_opts
= {
262 .head
= QTAILQ_HEAD_INITIALIZER(runtime_opts
.head
),
266 .type
= QEMU_OPT_STRING
,
267 .help
= "URL to the gluster image",
269 { /* end of list */ }
273 static void qemu_gluster_parse_flags(int bdrv_flags
, int *open_flags
)
275 assert(open_flags
!= NULL
);
277 *open_flags
|= O_BINARY
;
279 if (bdrv_flags
& BDRV_O_RDWR
) {
280 *open_flags
|= O_RDWR
;
282 *open_flags
|= O_RDONLY
;
285 if ((bdrv_flags
& BDRV_O_NOCACHE
)) {
286 *open_flags
|= O_DIRECT
;
290 static int qemu_gluster_open(BlockDriverState
*bs
, QDict
*options
,
291 int bdrv_flags
, Error
**errp
)
293 BDRVGlusterState
*s
= bs
->opaque
;
296 GlusterConf
*gconf
= g_new0(GlusterConf
, 1);
298 Error
*local_err
= NULL
;
299 const char *filename
;
301 opts
= qemu_opts_create(&runtime_opts
, NULL
, 0, &error_abort
);
302 qemu_opts_absorb_qdict(opts
, options
, &local_err
);
304 error_propagate(errp
, local_err
);
309 filename
= qemu_opt_get(opts
, "filename");
311 s
->glfs
= qemu_gluster_init(gconf
, filename
, errp
);
317 #ifdef CONFIG_GLUSTERFS_XLATOR_OPT
318 /* Without this, if fsync fails for a recoverable reason (for instance,
319 * ENOSPC), gluster will dump its cache, preventing retries. This means
320 * almost certain data loss. Not all gluster versions support the
321 * 'resync-failed-syncs-after-fsync' key value, but there is no way to
322 * discover during runtime if it is supported (this api returns success for
323 * unknown key/value pairs) */
324 ret
= glfs_set_xlator_option(s
->glfs
, "*-write-behind",
325 "resync-failed-syncs-after-fsync",
328 error_setg_errno(errp
, errno
, "Unable to set xlator key/value pair");
334 qemu_gluster_parse_flags(bdrv_flags
, &open_flags
);
336 s
->fd
= glfs_open(s
->glfs
, gconf
->image
, open_flags
);
343 qemu_gluster_gconf_free(gconf
);
356 typedef struct BDRVGlusterReopenState
{
359 } BDRVGlusterReopenState
;
362 static int qemu_gluster_reopen_prepare(BDRVReopenState
*state
,
363 BlockReopenQueue
*queue
, Error
**errp
)
366 BDRVGlusterReopenState
*reop_s
;
367 GlusterConf
*gconf
= NULL
;
370 assert(state
!= NULL
);
371 assert(state
->bs
!= NULL
);
373 state
->opaque
= g_new0(BDRVGlusterReopenState
, 1);
374 reop_s
= state
->opaque
;
376 qemu_gluster_parse_flags(state
->flags
, &open_flags
);
378 gconf
= g_new0(GlusterConf
, 1);
380 reop_s
->glfs
= qemu_gluster_init(gconf
, state
->bs
->filename
, errp
);
381 if (reop_s
->glfs
== NULL
) {
386 #ifdef CONFIG_GLUSTERFS_XLATOR_OPT
387 ret
= glfs_set_xlator_option(reop_s
->glfs
, "*-write-behind",
388 "resync-failed-syncs-after-fsync", "on");
390 error_setg_errno(errp
, errno
, "Unable to set xlator key/value pair");
396 reop_s
->fd
= glfs_open(reop_s
->glfs
, gconf
->image
, open_flags
);
397 if (reop_s
->fd
== NULL
) {
398 /* reops->glfs will be cleaned up in _abort */
404 /* state->opaque will be freed in either the _abort or _commit */
405 qemu_gluster_gconf_free(gconf
);
409 static void qemu_gluster_reopen_commit(BDRVReopenState
*state
)
411 BDRVGlusterReopenState
*reop_s
= state
->opaque
;
412 BDRVGlusterState
*s
= state
->bs
->opaque
;
423 /* use the newly opened image / connection */
425 s
->glfs
= reop_s
->glfs
;
427 g_free(state
->opaque
);
428 state
->opaque
= NULL
;
434 static void qemu_gluster_reopen_abort(BDRVReopenState
*state
)
436 BDRVGlusterReopenState
*reop_s
= state
->opaque
;
438 if (reop_s
== NULL
) {
443 glfs_close(reop_s
->fd
);
447 glfs_fini(reop_s
->glfs
);
450 g_free(state
->opaque
);
451 state
->opaque
= NULL
;
456 #ifdef CONFIG_GLUSTERFS_ZEROFILL
457 static coroutine_fn
int qemu_gluster_co_pwrite_zeroes(BlockDriverState
*bs
,
458 int64_t offset
, int size
, BdrvRequestFlags flags
)
462 BDRVGlusterState
*s
= bs
->opaque
;
466 acb
.coroutine
= qemu_coroutine_self();
467 acb
.aio_context
= bdrv_get_aio_context(bs
);
469 ret
= glfs_zerofill_async(s
->fd
, offset
, size
, gluster_finish_aiocb
, &acb
);
474 qemu_coroutine_yield();
478 static inline bool gluster_supports_zerofill(void)
483 static inline int qemu_gluster_zerofill(struct glfs_fd
*fd
, int64_t offset
,
486 return glfs_zerofill(fd
, offset
, size
);
490 static inline bool gluster_supports_zerofill(void)
495 static inline int qemu_gluster_zerofill(struct glfs_fd
*fd
, int64_t offset
,
502 static int qemu_gluster_create(const char *filename
,
503 QemuOpts
*opts
, Error
**errp
)
509 int64_t total_size
= 0;
511 GlusterConf
*gconf
= g_new0(GlusterConf
, 1);
513 glfs
= qemu_gluster_init(gconf
, filename
, errp
);
519 total_size
= ROUND_UP(qemu_opt_get_size_del(opts
, BLOCK_OPT_SIZE
, 0),
522 tmp
= qemu_opt_get_del(opts
, BLOCK_OPT_PREALLOC
);
523 if (!tmp
|| !strcmp(tmp
, "off")) {
525 } else if (!strcmp(tmp
, "full") &&
526 gluster_supports_zerofill()) {
529 error_setg(errp
, "Invalid preallocation mode: '%s'"
530 " or GlusterFS doesn't support zerofill API",
536 fd
= glfs_creat(glfs
, gconf
->image
,
537 O_WRONLY
| O_CREAT
| O_TRUNC
| O_BINARY
, S_IRUSR
| S_IWUSR
);
541 if (!glfs_ftruncate(fd
, total_size
)) {
542 if (prealloc
&& qemu_gluster_zerofill(fd
, 0, total_size
)) {
549 if (glfs_close(fd
) != 0) {
555 qemu_gluster_gconf_free(gconf
);
562 static coroutine_fn
int qemu_gluster_co_rw(BlockDriverState
*bs
,
563 int64_t sector_num
, int nb_sectors
, QEMUIOVector
*qiov
, int write
)
567 BDRVGlusterState
*s
= bs
->opaque
;
568 size_t size
= nb_sectors
* BDRV_SECTOR_SIZE
;
569 off_t offset
= sector_num
* BDRV_SECTOR_SIZE
;
573 acb
.coroutine
= qemu_coroutine_self();
574 acb
.aio_context
= bdrv_get_aio_context(bs
);
577 ret
= glfs_pwritev_async(s
->fd
, qiov
->iov
, qiov
->niov
, offset
, 0,
578 gluster_finish_aiocb
, &acb
);
580 ret
= glfs_preadv_async(s
->fd
, qiov
->iov
, qiov
->niov
, offset
, 0,
581 gluster_finish_aiocb
, &acb
);
588 qemu_coroutine_yield();
592 static int qemu_gluster_truncate(BlockDriverState
*bs
, int64_t offset
)
595 BDRVGlusterState
*s
= bs
->opaque
;
597 ret
= glfs_ftruncate(s
->fd
, offset
);
605 static coroutine_fn
int qemu_gluster_co_readv(BlockDriverState
*bs
,
606 int64_t sector_num
, int nb_sectors
, QEMUIOVector
*qiov
)
608 return qemu_gluster_co_rw(bs
, sector_num
, nb_sectors
, qiov
, 0);
611 static coroutine_fn
int qemu_gluster_co_writev(BlockDriverState
*bs
,
612 int64_t sector_num
, int nb_sectors
, QEMUIOVector
*qiov
)
614 return qemu_gluster_co_rw(bs
, sector_num
, nb_sectors
, qiov
, 1);
617 static void qemu_gluster_close(BlockDriverState
*bs
)
619 BDRVGlusterState
*s
= bs
->opaque
;
628 static coroutine_fn
int qemu_gluster_co_flush_to_disk(BlockDriverState
*bs
)
632 BDRVGlusterState
*s
= bs
->opaque
;
636 acb
.coroutine
= qemu_coroutine_self();
637 acb
.aio_context
= bdrv_get_aio_context(bs
);
639 ret
= glfs_fsync_async(s
->fd
, gluster_finish_aiocb
, &acb
);
645 qemu_coroutine_yield();
654 /* Some versions of Gluster (3.5.6 -> 3.5.8?) will not retain its cache
655 * after a fsync failure, so we have no way of allowing the guest to safely
656 * continue. Gluster versions prior to 3.5.6 don't retain the cache
657 * either, but will invalidate the fd on error, so this is again our only
660 * The 'resync-failed-syncs-after-fsync' xlator option for the
661 * write-behind cache will cause later gluster versions to retain its
662 * cache after error, so long as the fd remains open. However, we
663 * currently have no way of knowing if this option is supported.
665 * TODO: Once gluster provides a way for us to determine if the option
666 * is supported, bypass the closure and setting drv to NULL. */
667 qemu_gluster_close(bs
);
672 #ifdef CONFIG_GLUSTERFS_DISCARD
673 static coroutine_fn
int qemu_gluster_co_discard(BlockDriverState
*bs
,
674 int64_t sector_num
, int nb_sectors
)
678 BDRVGlusterState
*s
= bs
->opaque
;
679 size_t size
= nb_sectors
* BDRV_SECTOR_SIZE
;
680 off_t offset
= sector_num
* BDRV_SECTOR_SIZE
;
684 acb
.coroutine
= qemu_coroutine_self();
685 acb
.aio_context
= bdrv_get_aio_context(bs
);
687 ret
= glfs_discard_async(s
->fd
, offset
, size
, gluster_finish_aiocb
, &acb
);
692 qemu_coroutine_yield();
697 static int64_t qemu_gluster_getlength(BlockDriverState
*bs
)
699 BDRVGlusterState
*s
= bs
->opaque
;
702 ret
= glfs_lseek(s
->fd
, 0, SEEK_END
);
710 static int64_t qemu_gluster_allocated_file_size(BlockDriverState
*bs
)
712 BDRVGlusterState
*s
= bs
->opaque
;
716 ret
= glfs_fstat(s
->fd
, &st
);
720 return st
.st_blocks
* 512;
724 static int qemu_gluster_has_zero_init(BlockDriverState
*bs
)
726 /* GlusterFS volume could be backed by a block device */
730 static QemuOptsList qemu_gluster_create_opts
= {
731 .name
= "qemu-gluster-create-opts",
732 .head
= QTAILQ_HEAD_INITIALIZER(qemu_gluster_create_opts
.head
),
735 .name
= BLOCK_OPT_SIZE
,
736 .type
= QEMU_OPT_SIZE
,
737 .help
= "Virtual disk size"
740 .name
= BLOCK_OPT_PREALLOC
,
741 .type
= QEMU_OPT_STRING
,
742 .help
= "Preallocation mode (allowed values: off, full)"
744 { /* end of list */ }
748 static BlockDriver bdrv_gluster
= {
749 .format_name
= "gluster",
750 .protocol_name
= "gluster",
751 .instance_size
= sizeof(BDRVGlusterState
),
752 .bdrv_needs_filename
= true,
753 .bdrv_file_open
= qemu_gluster_open
,
754 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
755 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
756 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
757 .bdrv_close
= qemu_gluster_close
,
758 .bdrv_create
= qemu_gluster_create
,
759 .bdrv_getlength
= qemu_gluster_getlength
,
760 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
761 .bdrv_truncate
= qemu_gluster_truncate
,
762 .bdrv_co_readv
= qemu_gluster_co_readv
,
763 .bdrv_co_writev
= qemu_gluster_co_writev
,
764 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
765 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
766 #ifdef CONFIG_GLUSTERFS_DISCARD
767 .bdrv_co_discard
= qemu_gluster_co_discard
,
769 #ifdef CONFIG_GLUSTERFS_ZEROFILL
770 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
772 .create_opts
= &qemu_gluster_create_opts
,
775 static BlockDriver bdrv_gluster_tcp
= {
776 .format_name
= "gluster",
777 .protocol_name
= "gluster+tcp",
778 .instance_size
= sizeof(BDRVGlusterState
),
779 .bdrv_needs_filename
= true,
780 .bdrv_file_open
= qemu_gluster_open
,
781 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
782 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
783 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
784 .bdrv_close
= qemu_gluster_close
,
785 .bdrv_create
= qemu_gluster_create
,
786 .bdrv_getlength
= qemu_gluster_getlength
,
787 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
788 .bdrv_truncate
= qemu_gluster_truncate
,
789 .bdrv_co_readv
= qemu_gluster_co_readv
,
790 .bdrv_co_writev
= qemu_gluster_co_writev
,
791 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
792 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
793 #ifdef CONFIG_GLUSTERFS_DISCARD
794 .bdrv_co_discard
= qemu_gluster_co_discard
,
796 #ifdef CONFIG_GLUSTERFS_ZEROFILL
797 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
799 .create_opts
= &qemu_gluster_create_opts
,
802 static BlockDriver bdrv_gluster_unix
= {
803 .format_name
= "gluster",
804 .protocol_name
= "gluster+unix",
805 .instance_size
= sizeof(BDRVGlusterState
),
806 .bdrv_needs_filename
= true,
807 .bdrv_file_open
= qemu_gluster_open
,
808 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
809 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
810 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
811 .bdrv_close
= qemu_gluster_close
,
812 .bdrv_create
= qemu_gluster_create
,
813 .bdrv_getlength
= qemu_gluster_getlength
,
814 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
815 .bdrv_truncate
= qemu_gluster_truncate
,
816 .bdrv_co_readv
= qemu_gluster_co_readv
,
817 .bdrv_co_writev
= qemu_gluster_co_writev
,
818 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
819 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
820 #ifdef CONFIG_GLUSTERFS_DISCARD
821 .bdrv_co_discard
= qemu_gluster_co_discard
,
823 #ifdef CONFIG_GLUSTERFS_ZEROFILL
824 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
826 .create_opts
= &qemu_gluster_create_opts
,
829 static BlockDriver bdrv_gluster_rdma
= {
830 .format_name
= "gluster",
831 .protocol_name
= "gluster+rdma",
832 .instance_size
= sizeof(BDRVGlusterState
),
833 .bdrv_needs_filename
= true,
834 .bdrv_file_open
= qemu_gluster_open
,
835 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
836 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
837 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
838 .bdrv_close
= qemu_gluster_close
,
839 .bdrv_create
= qemu_gluster_create
,
840 .bdrv_getlength
= qemu_gluster_getlength
,
841 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
842 .bdrv_truncate
= qemu_gluster_truncate
,
843 .bdrv_co_readv
= qemu_gluster_co_readv
,
844 .bdrv_co_writev
= qemu_gluster_co_writev
,
845 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
846 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
847 #ifdef CONFIG_GLUSTERFS_DISCARD
848 .bdrv_co_discard
= qemu_gluster_co_discard
,
850 #ifdef CONFIG_GLUSTERFS_ZEROFILL
851 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
853 .create_opts
= &qemu_gluster_create_opts
,
856 static void bdrv_gluster_init(void)
858 bdrv_register(&bdrv_gluster_rdma
);
859 bdrv_register(&bdrv_gluster_unix
);
860 bdrv_register(&bdrv_gluster_tcp
);
861 bdrv_register(&bdrv_gluster
);
864 block_init(bdrv_gluster_init
);