2 * vhost-user-scsi sample application
4 * Copyright (c) 2016 Nutanix Inc. All rights reserved.
7 * Felipe Franciosi <felipe@nutanix.com>
9 * This work is licensed under the terms of the GNU GPL, version 2 only.
10 * See the COPYING file in the top-level directory.
13 #include "qemu/osdep.h"
14 #include "contrib/libvhost-user/libvhost-user.h"
15 #include "hw/virtio/virtio-scsi.h"
16 #include "iscsi/iscsi.h"
20 /* Small compat shim from glib 2.32 */
21 #ifndef G_SOURCE_CONTINUE
22 #define G_SOURCE_CONTINUE TRUE
24 #ifndef G_SOURCE_REMOVE
25 #define G_SOURCE_REMOVE FALSE
28 /* #define VUS_DEBUG 1 */
36 (void)clock_gettime(CLOCK_REALTIME, &ts); \
37 (void)strftime(timebuf, 64, "%Y%m%d %T", gmtime_r(&ts.tv_sec, &tm))
39 #define PEXT(lvl, msg, ...) do { \
41 fprintf(stderr, "%s.%06ld " lvl ": %s:%s():%d: " msg "\n", \
42 timebuf, ts.tv_nsec / 1000, \
43 __FILE__, __func__, __LINE__, ## __VA_ARGS__); \
46 #define PNOR(lvl, msg, ...) do { \
48 fprintf(stderr, "%s.%06ld " lvl ": " msg "\n", \
49 timebuf, ts.tv_nsec / 1000, ## __VA_ARGS__); \
53 #define PDBG(msg, ...) PEXT("DBG", msg, ## __VA_ARGS__)
54 #define PERR(msg, ...) PEXT("ERR", msg, ## __VA_ARGS__)
55 #define PLOG(msg, ...) PEXT("LOG", msg, ## __VA_ARGS__)
57 #define PDBG(msg, ...) { }
58 #define PERR(msg, ...) PNOR("ERR", msg, ## __VA_ARGS__)
59 #define PLOG(msg, ...) PNOR("LOG", msg, ## __VA_ARGS__)
62 /** vhost-user-scsi specific definitions **/
64 /* Only 1 LUN and device supported today */
65 #define VUS_MAX_LUNS 1
66 #define VUS_MAX_DEVS 1
68 #define VUS_ISCSI_INITIATOR "iqn.2016-11.com.nutanix:vhost-user-scsi"
70 typedef struct iscsi_lun
{
71 struct iscsi_context
*iscsi_ctx
;
75 typedef struct vhost_scsi_dev
{
79 GTree
*fdmap
; /* fd -> gsource context id */
80 iscsi_lun_t luns
[VUS_MAX_LUNS
];
83 static vhost_scsi_dev_t
*vhost_scsi_devs
[VUS_MAX_DEVS
];
85 /** glib event loop integration for libvhost-user and misc callbacks **/
87 QEMU_BUILD_BUG_ON((int)G_IO_IN
!= (int)VU_WATCH_IN
);
88 QEMU_BUILD_BUG_ON((int)G_IO_OUT
!= (int)VU_WATCH_OUT
);
89 QEMU_BUILD_BUG_ON((int)G_IO_PRI
!= (int)VU_WATCH_PRI
);
90 QEMU_BUILD_BUG_ON((int)G_IO_ERR
!= (int)VU_WATCH_ERR
);
91 QEMU_BUILD_BUG_ON((int)G_IO_HUP
!= (int)VU_WATCH_HUP
);
93 typedef struct vus_gsrc
{
95 vhost_scsi_dev_t
*vdev_scsi
;
100 static gint
vus_fdmap_compare(gconstpointer a
, gconstpointer b
)
102 return (b
> a
) - (b
< a
);
105 static gboolean
vus_gsrc_prepare(GSource
*src
, gint
*timeout
)
113 static gboolean
vus_gsrc_check(GSource
*src
)
115 vus_gsrc_t
*vus_src
= (vus_gsrc_t
*)src
;
119 return vus_src
->gfd
.revents
& vus_src
->gfd
.events
;
122 static gboolean
vus_gsrc_dispatch(GSource
*src
, GSourceFunc cb
, gpointer data
)
124 vhost_scsi_dev_t
*vdev_scsi
;
125 vus_gsrc_t
*vus_src
= (vus_gsrc_t
*)src
;
128 assert(!(vus_src
->vu_cb
&& cb
));
130 vdev_scsi
= vus_src
->vdev_scsi
;
137 if (vus_src
->vu_cb
) {
138 vus_src
->vu_cb(&vdev_scsi
->vu_dev
, vus_src
->gfd
.revents
, data
);
140 return G_SOURCE_CONTINUE
;
143 static GSourceFuncs vus_gsrc_funcs
= {
150 static int vus_gsrc_new(vhost_scsi_dev_t
*vdev_scsi
, int fd
, GIOCondition cond
,
151 vu_watch_cb vu_cb
, GSourceFunc gsrc_cb
, gpointer data
)
159 assert(vu_cb
|| gsrc_cb
);
160 assert(!(vu_cb
&& gsrc_cb
));
162 vus_gsrc
= g_source_new(&vus_gsrc_funcs
, sizeof(vus_gsrc_t
));
164 PERR("Error creating GSource for new watch");
167 vus_src
= (vus_gsrc_t
*)vus_gsrc
;
169 vus_src
->vdev_scsi
= vdev_scsi
;
170 vus_src
->gfd
.fd
= fd
;
171 vus_src
->gfd
.events
= cond
;
172 vus_src
->vu_cb
= vu_cb
;
174 g_source_add_poll(vus_gsrc
, &vus_src
->gfd
);
175 g_source_set_callback(vus_gsrc
, gsrc_cb
, data
, NULL
);
176 id
= g_source_attach(vus_gsrc
, NULL
);
178 g_source_unref(vus_gsrc
);
180 g_tree_insert(vdev_scsi
->fdmap
, (gpointer
)(uintptr_t)fd
,
181 (gpointer
)(uintptr_t)id
);
186 /* from libiscsi's scsi-lowlevel.h **
188 * nb. We can't directly include scsi-lowlevel.h due to a namespace conflict:
189 * QEMU's scsi.h also defines "SCSI_XFER_NONE".
192 #define SCSI_CDB_MAX_SIZE 16
194 struct scsi_iovector
{
195 struct scsi_iovec
*iov
;
202 struct scsi_allocated_memory
{
203 struct scsi_allocated_memory
*next
;
212 enum scsi_sense_key
{
213 SCSI_SENSE_NO_SENSE
= 0x00,
214 SCSI_SENSE_RECOVERED_ERROR
= 0x01,
215 SCSI_SENSE_NOT_READY
= 0x02,
216 SCSI_SENSE_MEDIUM_ERROR
= 0x03,
217 SCSI_SENSE_HARDWARE_ERROR
= 0x04,
218 SCSI_SENSE_ILLEGAL_REQUEST
= 0x05,
219 SCSI_SENSE_UNIT_ATTENTION
= 0x06,
220 SCSI_SENSE_DATA_PROTECTION
= 0x07,
221 SCSI_SENSE_BLANK_CHECK
= 0x08,
222 SCSI_SENSE_VENDOR_SPECIFIC
= 0x09,
223 SCSI_SENSE_COPY_ABORTED
= 0x0a,
224 SCSI_SENSE_COMMAND_ABORTED
= 0x0b,
225 SCSI_SENSE_OBSOLETE_ERROR_CODE
= 0x0c,
226 SCSI_SENSE_OVERFLOW_COMMAND
= 0x0d,
227 SCSI_SENSE_MISCOMPARE
= 0x0e
231 unsigned char error_type
;
232 enum scsi_sense_key key
;
234 unsigned sense_specific
:1;
235 unsigned ill_param_in_cdb
:1;
236 unsigned bit_pointer_valid
:1;
237 unsigned char bit_pointer
;
238 uint16_t field_pointer
;
242 SCSI_RESIDUAL_NO_RESIDUAL
= 0,
243 SCSI_RESIDUAL_UNDERFLOW
,
244 SCSI_RESIDUAL_OVERFLOW
252 unsigned char cdb
[SCSI_CDB_MAX_SIZE
];
253 enum scsi_residual residual_status
;
255 struct scsi_sense sense
;
256 struct scsi_data datain
;
257 struct scsi_allocated_memory
*mem
;
264 struct scsi_iovector iovector_in
;
265 struct scsi_iovector iovector_out
;
268 /** libiscsi integration **/
270 static int iscsi_add_lun(iscsi_lun_t
*lun
, char *iscsi_uri
)
272 struct iscsi_url
*iscsi_url
;
273 struct iscsi_context
*iscsi_ctx
;
279 iscsi_ctx
= iscsi_create_context(VUS_ISCSI_INITIATOR
);
281 PERR("Unable to create iSCSI context");
285 iscsi_url
= iscsi_parse_full_url(iscsi_ctx
, iscsi_uri
);
287 PERR("Unable to parse iSCSI URL: %s", iscsi_get_error(iscsi_ctx
));
291 iscsi_set_session_type(iscsi_ctx
, ISCSI_SESSION_NORMAL
);
292 iscsi_set_header_digest(iscsi_ctx
, ISCSI_HEADER_DIGEST_NONE_CRC32C
);
293 if (iscsi_full_connect_sync(iscsi_ctx
, iscsi_url
->portal
, iscsi_url
->lun
)) {
294 PERR("Unable to login to iSCSI portal: %s", iscsi_get_error(iscsi_ctx
));
298 lun
->iscsi_ctx
= iscsi_ctx
;
299 lun
->iscsi_lun
= iscsi_url
->lun
;
301 PDBG("Context %p created for lun 0: %s", iscsi_ctx
, iscsi_uri
);
305 iscsi_destroy_url(iscsi_url
);
310 (void)iscsi_destroy_context(iscsi_ctx
);
315 static struct scsi_task
*scsi_task_new(int cdb_len
, uint8_t *cdb
, int dir
,
317 struct scsi_task
*task
;
322 task
= calloc(1, sizeof(struct scsi_task
));
324 PERR("Error allocating task: %s", strerror(errno
));
328 memcpy(task
->cdb
, cdb
, cdb_len
);
329 task
->cdb_size
= cdb_len
;
330 task
->xfer_dir
= dir
;
331 task
->expxferlen
= xfer_len
;
336 static int get_cdb_len(uint8_t *cdb
)
340 switch (cdb
[0] >> 5) {
342 case 1: /* fall through */
347 PERR("Unable to determine cdb len (0x%02hhX)", cdb
[0] >> 5);
351 static int handle_cmd_sync(struct iscsi_context
*ctx
,
352 VirtIOSCSICmdReq
*req
,
353 struct iovec
*out
, unsigned int out_len
,
354 VirtIOSCSICmdResp
*rsp
,
355 struct iovec
*in
, unsigned int in_len
) {
356 struct scsi_task
*task
;
366 if (!(!req
->lun
[1] && req
->lun
[2] == 0x40 && !req
->lun
[3])) {
367 /* Ignore anything different than target=0, lun=0 */
368 PDBG("Ignoring unconnected lun (0x%hhX, 0x%hhX)",
369 req
->lun
[1], req
->lun
[3]);
370 rsp
->status
= SCSI_STATUS_CHECK_CONDITION
;
371 memset(rsp
->sense
, 0, sizeof(rsp
->sense
));
373 rsp
->sense
[0] = 0x70;
374 rsp
->sense
[2] = SCSI_SENSE_ILLEGAL_REQUEST
;
376 rsp
->sense
[12] = 0x24;
381 cdb_len
= get_cdb_len(req
->cdb
);
387 if (!out_len
&& !in_len
) {
388 dir
= SCSI_XFER_NONE
;
389 } else if (out_len
) {
390 dir
= SCSI_XFER_TO_DEV
;
391 for (i
= 0; i
< out_len
; i
++) {
392 len
+= out
[i
].iov_len
;
395 dir
= SCSI_XFER_FROM_DEV
;
396 for (i
= 0; i
< in_len
; i
++) {
397 len
+= in
[i
].iov_len
;
401 task
= scsi_task_new(cdb_len
, req
->cdb
, dir
, len
);
403 PERR("Unable to create iscsi task");
407 if (dir
== SCSI_XFER_TO_DEV
) {
408 task
->iovector_out
.iov
= (struct scsi_iovec
*)out
;
409 task
->iovector_out
.niov
= out_len
;
410 } else if (dir
== SCSI_XFER_FROM_DEV
) {
411 task
->iovector_in
.iov
= (struct scsi_iovec
*)in
;
412 task
->iovector_in
.niov
= in_len
;
415 PDBG("Sending iscsi cmd (cdb_len=%d, dir=%d, task=%p)",
417 if (!iscsi_scsi_command_sync(ctx
, 0, task
, NULL
)) {
418 PERR("Error serving SCSI command");
423 memset(rsp
, 0, sizeof(*rsp
));
425 rsp
->status
= task
->status
;
426 rsp
->resid
= task
->residual
;
428 if (task
->status
== SCSI_STATUS_CHECK_CONDITION
) {
429 rsp
->response
= VIRTIO_SCSI_S_FAILURE
;
430 rsp
->sense_len
= task
->datain
.size
- 2;
431 memcpy(rsp
->sense
, &task
->datain
.data
[2], rsp
->sense_len
);
436 PDBG("Filled in rsp: status=%hhX, resid=%u, response=%hhX, sense_len=%u",
437 rsp
->status
, rsp
->resid
, rsp
->response
, rsp
->sense_len
);
442 /** libvhost-user callbacks **/
444 static vhost_scsi_dev_t
*vdev_scsi_find_by_vu(VuDev
*vu_dev
);
446 static void vus_panic_cb(VuDev
*vu_dev
, const char *buf
)
448 vhost_scsi_dev_t
*vdev_scsi
;
452 vdev_scsi
= vdev_scsi_find_by_vu(vu_dev
);
455 PERR("vu_panic: %s", buf
);
459 assert(vdev_scsi
->loop
);
460 g_main_loop_quit(vdev_scsi
->loop
);
464 static void vus_add_watch_cb(VuDev
*vu_dev
, int fd
, int vu_evt
, vu_watch_cb cb
,
466 vhost_scsi_dev_t
*vdev_scsi
;
473 vdev_scsi
= vdev_scsi_find_by_vu(vu_dev
);
475 vus_panic_cb(vu_dev
, NULL
);
479 id
= (guint
)(uintptr_t)g_tree_lookup(vdev_scsi
->fdmap
,
480 (gpointer
)(uintptr_t)fd
);
482 GSource
*vus_src
= g_main_context_find_source_by_id(NULL
, id
);
484 g_source_destroy(vus_src
);
485 (void)g_tree_remove(vdev_scsi
->fdmap
, (gpointer
)(uintptr_t)fd
);
488 if (vus_gsrc_new(vdev_scsi
, fd
, vu_evt
, cb
, NULL
, pvt
)) {
489 vus_panic_cb(vu_dev
, NULL
);
493 static void vus_del_watch_cb(VuDev
*vu_dev
, int fd
)
495 vhost_scsi_dev_t
*vdev_scsi
;
501 vdev_scsi
= vdev_scsi_find_by_vu(vu_dev
);
503 vus_panic_cb(vu_dev
, NULL
);
507 id
= (guint
)(uintptr_t)g_tree_lookup(vdev_scsi
->fdmap
,
508 (gpointer
)(uintptr_t)fd
);
510 GSource
*vus_src
= g_main_context_find_source_by_id(NULL
, id
);
512 g_source_destroy(vus_src
);
513 (void)g_tree_remove(vdev_scsi
->fdmap
, (gpointer
)(uintptr_t)fd
);
517 static void vus_proc_ctl(VuDev
*vu_dev
, int idx
)
519 /* Control VQ not implemented */
522 static void vus_proc_evt(VuDev
*vu_dev
, int idx
)
524 /* Event VQ not implemented */
527 static void vus_proc_req(VuDev
*vu_dev
, int idx
)
529 vhost_scsi_dev_t
*vdev_scsi
;
534 vdev_scsi
= vdev_scsi_find_by_vu(vu_dev
);
536 vus_panic_cb(vu_dev
, NULL
);
540 if ((idx
< 0) || (idx
>= VHOST_MAX_NR_VIRTQUEUE
)) {
541 PERR("VQ Index out of range: %d", idx
);
542 vus_panic_cb(vu_dev
, NULL
);
546 vq
= vu_get_queue(vu_dev
, idx
);
548 PERR("Error fetching VQ (dev=%p, idx=%d)", vu_dev
, idx
);
549 vus_panic_cb(vu_dev
, NULL
);
553 PDBG("Got kicked on vq[%d]@%p", idx
, vq
);
556 VuVirtqElement
*elem
;
557 VirtIOSCSICmdReq
*req
;
558 VirtIOSCSICmdResp
*rsp
;
560 elem
= vu_queue_pop(vu_dev
, vq
, sizeof(VuVirtqElement
));
562 PDBG("No more elements pending on vq[%d]@%p", idx
, vq
);
565 PDBG("Popped elem@%p", elem
);
567 assert(!((elem
->out_num
> 1) && (elem
->in_num
> 1)));
568 assert((elem
->out_num
> 0) && (elem
->in_num
> 0));
570 if (elem
->out_sg
[0].iov_len
< sizeof(VirtIOSCSICmdReq
)) {
571 PERR("Invalid virtio-scsi req header");
572 vus_panic_cb(vu_dev
, NULL
);
575 req
= (VirtIOSCSICmdReq
*)elem
->out_sg
[0].iov_base
;
577 if (elem
->in_sg
[0].iov_len
< sizeof(VirtIOSCSICmdResp
)) {
578 PERR("Invalid virtio-scsi rsp header");
579 vus_panic_cb(vu_dev
, NULL
);
582 rsp
= (VirtIOSCSICmdResp
*)elem
->in_sg
[0].iov_base
;
584 if (handle_cmd_sync(vdev_scsi
->luns
[0].iscsi_ctx
,
585 req
, &elem
->out_sg
[1], elem
->out_num
- 1,
586 rsp
, &elem
->in_sg
[1], elem
->in_num
- 1) != 0) {
587 vus_panic_cb(vu_dev
, NULL
);
591 vu_queue_push(vu_dev
, vq
, elem
, 0);
592 vu_queue_notify(vu_dev
, vq
);
598 static void vus_queue_set_started(VuDev
*vu_dev
, int idx
, bool started
)
604 if ((idx
< 0) || (idx
>= VHOST_MAX_NR_VIRTQUEUE
)) {
605 PERR("VQ Index out of range: %d", idx
);
606 vus_panic_cb(vu_dev
, NULL
);
610 vq
= vu_get_queue(vu_dev
, idx
);
614 vu_set_queue_handler(vu_dev
, vq
, started
? vus_proc_ctl
: NULL
);
617 vu_set_queue_handler(vu_dev
, vq
, started
? vus_proc_evt
: NULL
);
620 vu_set_queue_handler(vu_dev
, vq
, started
? vus_proc_req
: NULL
);
624 static const VuDevIface vus_iface
= {
625 .queue_set_started
= vus_queue_set_started
,
628 static gboolean
vus_vhost_cb(gpointer data
)
630 VuDev
*vu_dev
= (VuDev
*)data
;
634 if (!vu_dispatch(vu_dev
) != 0) {
635 PERR("Error processing vhost message");
636 vus_panic_cb(vu_dev
, NULL
);
637 return G_SOURCE_REMOVE
;
640 return G_SOURCE_CONTINUE
;
645 static int unix_sock_new(char *unix_fn
)
648 struct sockaddr_un un
;
653 sock
= socket(AF_UNIX
, SOCK_STREAM
, 0);
659 un
.sun_family
= AF_UNIX
;
660 (void)snprintf(un
.sun_path
, sizeof(un
.sun_path
), "%s", unix_fn
);
661 len
= sizeof(un
.sun_family
) + strlen(un
.sun_path
);
663 (void)unlink(unix_fn
);
664 if (bind(sock
, (struct sockaddr
*)&un
, len
) < 0) {
669 if (listen(sock
, 1) < 0) {
682 /** vhost-user-scsi **/
684 static vhost_scsi_dev_t
*vdev_scsi_find_by_vu(VuDev
*vu_dev
)
690 for (i
= 0; i
< VUS_MAX_DEVS
; i
++) {
691 if (&vhost_scsi_devs
[i
]->vu_dev
== vu_dev
) {
692 return vhost_scsi_devs
[i
];
696 PERR("Unknown VuDev %p", vu_dev
);
700 static void vdev_scsi_deinit(vhost_scsi_dev_t
*vdev_scsi
)
706 if (vdev_scsi
->server_sock
>= 0) {
707 struct sockaddr_storage ss
;
708 socklen_t sslen
= sizeof(ss
);
710 if (getsockname(vdev_scsi
->server_sock
, (struct sockaddr
*)&ss
,
712 struct sockaddr_un
*su
= (struct sockaddr_un
*)&ss
;
713 (void)unlink(su
->sun_path
);
716 (void)close(vdev_scsi
->server_sock
);
717 vdev_scsi
->server_sock
= -1;
720 if (vdev_scsi
->loop
) {
721 g_main_loop_unref(vdev_scsi
->loop
);
722 vdev_scsi
->loop
= NULL
;
726 static vhost_scsi_dev_t
*vdev_scsi_new(char *unix_fn
)
728 vhost_scsi_dev_t
*vdev_scsi
= NULL
;
732 vdev_scsi
= calloc(1, sizeof(vhost_scsi_dev_t
));
734 PERR("calloc: %s", strerror(errno
));
738 vdev_scsi
->server_sock
= unix_sock_new(unix_fn
);
739 if (vdev_scsi
->server_sock
< 0) {
743 vdev_scsi
->loop
= g_main_loop_new(NULL
, FALSE
);
744 if (!vdev_scsi
->loop
) {
745 PERR("Error creating glib event loop");
749 vdev_scsi
->fdmap
= g_tree_new(vus_fdmap_compare
);
750 if (!vdev_scsi
->fdmap
) {
751 PERR("Error creating glib tree for fdmap");
758 vdev_scsi_deinit(vdev_scsi
);
764 static int vdev_scsi_add_iscsi_lun(vhost_scsi_dev_t
*vdev_scsi
,
765 char *iscsi_uri
, uint32_t lun
) {
768 assert(lun
< VUS_MAX_LUNS
);
770 if (vdev_scsi
->luns
[lun
].iscsi_ctx
) {
771 PERR("Lun %d already configured", lun
);
775 if (iscsi_add_lun(&vdev_scsi
->luns
[lun
], iscsi_uri
) != 0) {
782 static int vdev_scsi_run(vhost_scsi_dev_t
*vdev_scsi
)
788 assert(vdev_scsi
->server_sock
>= 0);
789 assert(vdev_scsi
->loop
);
791 cli_sock
= accept(vdev_scsi
->server_sock
, (void *)0, (void *)0);
797 vu_init(&vdev_scsi
->vu_dev
,
804 if (vus_gsrc_new(vdev_scsi
, cli_sock
, G_IO_IN
, NULL
, vus_vhost_cb
,
805 &vdev_scsi
->vu_dev
)) {
809 g_main_loop_run(vdev_scsi
->loop
);
812 vu_deinit(&vdev_scsi
->vu_dev
);
821 int main(int argc
, char **argv
)
823 vhost_scsi_dev_t
*vdev_scsi
= NULL
;
824 char *unix_fn
= NULL
;
825 char *iscsi_uri
= NULL
;
826 int opt
, err
= EXIT_SUCCESS
;
828 while ((opt
= getopt(argc
, argv
, "u:i:")) != -1) {
833 unix_fn
= strdup(optarg
);
836 iscsi_uri
= strdup(optarg
);
842 if (!unix_fn
|| !iscsi_uri
) {
846 vdev_scsi
= vdev_scsi_new(unix_fn
);
850 vhost_scsi_devs
[0] = vdev_scsi
;
852 if (vdev_scsi_add_iscsi_lun(vdev_scsi
, iscsi_uri
, 0) != 0) {
856 if (vdev_scsi_run(vdev_scsi
) != 0) {
862 vdev_scsi_deinit(vdev_scsi
);
879 fprintf(stderr
, "Usage: %s [ -u unix_sock_path -i iscsi_uri ] | [ -h ]\n",
881 fprintf(stderr
, " -u path to unix socket\n");
882 fprintf(stderr
, " -i iscsi uri for lun 0\n");
883 fprintf(stderr
, " -h print help and quit\n");