2 * vhost-user-scsi sample application
4 * Copyright (c) 2016 Nutanix Inc. All rights reserved.
7 * Felipe Franciosi <felipe@nutanix.com>
9 * This work is licensed under the terms of the GNU GPL, version 2 only.
10 * See the COPYING file in the top-level directory.
13 #include "qemu/osdep.h"
14 #include "contrib/libvhost-user/libvhost-user.h"
15 #include "hw/virtio/virtio-scsi.h"
16 #include "iscsi/iscsi.h"
20 /* #define VUS_DEBUG 1 */
28 (void)clock_gettime(CLOCK_REALTIME, &ts); \
29 (void)strftime(timebuf, 64, "%Y%m%d %T", gmtime_r(&ts.tv_sec, &tm))
31 #define PEXT(lvl, msg, ...) do { \
33 fprintf(stderr, "%s.%06ld " lvl ": %s:%s():%d: " msg "\n", \
34 timebuf, ts.tv_nsec / 1000, \
35 __FILE__, __func__, __LINE__, ## __VA_ARGS__); \
38 #define PNOR(lvl, msg, ...) do { \
40 fprintf(stderr, "%s.%06ld " lvl ": " msg "\n", \
41 timebuf, ts.tv_nsec / 1000, ## __VA_ARGS__); \
45 #define PDBG(msg, ...) PEXT("DBG", msg, ## __VA_ARGS__)
46 #define PERR(msg, ...) PEXT("ERR", msg, ## __VA_ARGS__)
47 #define PLOG(msg, ...) PEXT("LOG", msg, ## __VA_ARGS__)
49 #define PDBG(msg, ...) { }
50 #define PERR(msg, ...) PNOR("ERR", msg, ## __VA_ARGS__)
51 #define PLOG(msg, ...) PNOR("LOG", msg, ## __VA_ARGS__)
54 /** vhost-user-scsi specific definitions **/
56 /* Only 1 LUN and device supported today */
57 #define VUS_MAX_LUNS 1
58 #define VUS_ISCSI_INITIATOR "iqn.2016-11.com.nutanix:vhost-user-scsi"
60 typedef struct iscsi_lun
{
61 struct iscsi_context
*iscsi_ctx
;
65 typedef struct vhost_scsi_dev
{
69 GTree
*fdmap
; /* fd -> gsource context id */
70 iscsi_lun_t luns
[VUS_MAX_LUNS
];
73 /** glib event loop integration for libvhost-user and misc callbacks **/
75 QEMU_BUILD_BUG_ON((int)G_IO_IN
!= (int)VU_WATCH_IN
);
76 QEMU_BUILD_BUG_ON((int)G_IO_OUT
!= (int)VU_WATCH_OUT
);
77 QEMU_BUILD_BUG_ON((int)G_IO_PRI
!= (int)VU_WATCH_PRI
);
78 QEMU_BUILD_BUG_ON((int)G_IO_ERR
!= (int)VU_WATCH_ERR
);
79 QEMU_BUILD_BUG_ON((int)G_IO_HUP
!= (int)VU_WATCH_HUP
);
81 typedef struct vus_gsrc
{
83 vhost_scsi_dev_t
*vdev_scsi
;
88 static gint
vus_fdmap_compare(gconstpointer a
, gconstpointer b
)
90 return (b
> a
) - (b
< a
);
93 static gboolean
vus_gsrc_prepare(GSource
*src
, gint
*timeout
)
101 static gboolean
vus_gsrc_check(GSource
*src
)
103 vus_gsrc_t
*vus_src
= (vus_gsrc_t
*)src
;
107 return vus_src
->gfd
.revents
& vus_src
->gfd
.events
;
110 static gboolean
vus_gsrc_dispatch(GSource
*src
, GSourceFunc cb
, gpointer data
)
112 vhost_scsi_dev_t
*vdev_scsi
;
113 vus_gsrc_t
*vus_src
= (vus_gsrc_t
*)src
;
116 assert(!(vus_src
->vu_cb
&& cb
));
118 vdev_scsi
= vus_src
->vdev_scsi
;
125 if (vus_src
->vu_cb
) {
126 vus_src
->vu_cb(&vdev_scsi
->vu_dev
, vus_src
->gfd
.revents
, data
);
128 return G_SOURCE_CONTINUE
;
131 static GSourceFuncs vus_gsrc_funcs
= {
138 static void vus_gsrc_new(vhost_scsi_dev_t
*vdev_scsi
, int fd
, GIOCondition cond
,
139 vu_watch_cb vu_cb
, GSourceFunc gsrc_cb
, gpointer data
)
147 assert(vu_cb
|| gsrc_cb
);
148 assert(!(vu_cb
&& gsrc_cb
));
150 vus_gsrc
= g_source_new(&vus_gsrc_funcs
, sizeof(vus_gsrc_t
));
151 vus_src
= (vus_gsrc_t
*)vus_gsrc
;
153 vus_src
->vdev_scsi
= vdev_scsi
;
154 vus_src
->gfd
.fd
= fd
;
155 vus_src
->gfd
.events
= cond
;
156 vus_src
->vu_cb
= vu_cb
;
158 g_source_add_poll(vus_gsrc
, &vus_src
->gfd
);
159 g_source_set_callback(vus_gsrc
, gsrc_cb
, data
, NULL
);
160 id
= g_source_attach(vus_gsrc
, NULL
);
162 g_source_unref(vus_gsrc
);
164 g_tree_insert(vdev_scsi
->fdmap
, (gpointer
)(uintptr_t)fd
,
165 (gpointer
)(uintptr_t)id
);
168 /* from libiscsi's scsi-lowlevel.h **
170 * nb. We can't directly include scsi-lowlevel.h due to a namespace conflict:
171 * QEMU's scsi.h also defines "SCSI_XFER_NONE".
174 #define SCSI_CDB_MAX_SIZE 16
176 struct scsi_iovector
{
177 struct scsi_iovec
*iov
;
184 struct scsi_allocated_memory
{
185 struct scsi_allocated_memory
*next
;
194 enum scsi_sense_key
{
195 SCSI_SENSE_NO_SENSE
= 0x00,
196 SCSI_SENSE_RECOVERED_ERROR
= 0x01,
197 SCSI_SENSE_NOT_READY
= 0x02,
198 SCSI_SENSE_MEDIUM_ERROR
= 0x03,
199 SCSI_SENSE_HARDWARE_ERROR
= 0x04,
200 SCSI_SENSE_ILLEGAL_REQUEST
= 0x05,
201 SCSI_SENSE_UNIT_ATTENTION
= 0x06,
202 SCSI_SENSE_DATA_PROTECTION
= 0x07,
203 SCSI_SENSE_BLANK_CHECK
= 0x08,
204 SCSI_SENSE_VENDOR_SPECIFIC
= 0x09,
205 SCSI_SENSE_COPY_ABORTED
= 0x0a,
206 SCSI_SENSE_COMMAND_ABORTED
= 0x0b,
207 SCSI_SENSE_OBSOLETE_ERROR_CODE
= 0x0c,
208 SCSI_SENSE_OVERFLOW_COMMAND
= 0x0d,
209 SCSI_SENSE_MISCOMPARE
= 0x0e
213 unsigned char error_type
;
214 enum scsi_sense_key key
;
216 unsigned sense_specific
:1;
217 unsigned ill_param_in_cdb
:1;
218 unsigned bit_pointer_valid
:1;
219 unsigned char bit_pointer
;
220 uint16_t field_pointer
;
224 SCSI_RESIDUAL_NO_RESIDUAL
= 0,
225 SCSI_RESIDUAL_UNDERFLOW
,
226 SCSI_RESIDUAL_OVERFLOW
234 unsigned char cdb
[SCSI_CDB_MAX_SIZE
];
235 enum scsi_residual residual_status
;
237 struct scsi_sense sense
;
238 struct scsi_data datain
;
239 struct scsi_allocated_memory
*mem
;
246 struct scsi_iovector iovector_in
;
247 struct scsi_iovector iovector_out
;
250 /** libiscsi integration **/
252 static int iscsi_add_lun(iscsi_lun_t
*lun
, char *iscsi_uri
)
254 struct iscsi_url
*iscsi_url
;
255 struct iscsi_context
*iscsi_ctx
;
261 iscsi_ctx
= iscsi_create_context(VUS_ISCSI_INITIATOR
);
263 PERR("Unable to create iSCSI context");
267 iscsi_url
= iscsi_parse_full_url(iscsi_ctx
, iscsi_uri
);
269 PERR("Unable to parse iSCSI URL: %s", iscsi_get_error(iscsi_ctx
));
273 iscsi_set_session_type(iscsi_ctx
, ISCSI_SESSION_NORMAL
);
274 iscsi_set_header_digest(iscsi_ctx
, ISCSI_HEADER_DIGEST_NONE_CRC32C
);
275 if (iscsi_full_connect_sync(iscsi_ctx
, iscsi_url
->portal
, iscsi_url
->lun
)) {
276 PERR("Unable to login to iSCSI portal: %s", iscsi_get_error(iscsi_ctx
));
280 lun
->iscsi_ctx
= iscsi_ctx
;
281 lun
->iscsi_lun
= iscsi_url
->lun
;
283 PDBG("Context %p created for lun 0: %s", iscsi_ctx
, iscsi_uri
);
287 iscsi_destroy_url(iscsi_url
);
292 (void)iscsi_destroy_context(iscsi_ctx
);
297 static struct scsi_task
*scsi_task_new(int cdb_len
, uint8_t *cdb
, int dir
,
300 struct scsi_task
*task
;
305 task
= g_new0(struct scsi_task
, 1);
306 memcpy(task
->cdb
, cdb
, cdb_len
);
307 task
->cdb_size
= cdb_len
;
308 task
->xfer_dir
= dir
;
309 task
->expxferlen
= xfer_len
;
314 static int get_cdb_len(uint8_t *cdb
)
318 switch (cdb
[0] >> 5) {
320 case 1: /* fall through */
325 PERR("Unable to determine cdb len (0x%02hhX)", cdb
[0] >> 5);
329 static int handle_cmd_sync(struct iscsi_context
*ctx
,
330 VirtIOSCSICmdReq
*req
,
331 struct iovec
*out
, unsigned int out_len
,
332 VirtIOSCSICmdResp
*rsp
,
333 struct iovec
*in
, unsigned int in_len
)
335 struct scsi_task
*task
;
345 if (!(!req
->lun
[1] && req
->lun
[2] == 0x40 && !req
->lun
[3])) {
346 /* Ignore anything different than target=0, lun=0 */
347 PDBG("Ignoring unconnected lun (0x%hhX, 0x%hhX)",
348 req
->lun
[1], req
->lun
[3]);
349 rsp
->status
= SCSI_STATUS_CHECK_CONDITION
;
350 memset(rsp
->sense
, 0, sizeof(rsp
->sense
));
352 rsp
->sense
[0] = 0x70;
353 rsp
->sense
[2] = SCSI_SENSE_ILLEGAL_REQUEST
;
355 rsp
->sense
[12] = 0x24;
360 cdb_len
= get_cdb_len(req
->cdb
);
366 if (!out_len
&& !in_len
) {
367 dir
= SCSI_XFER_NONE
;
368 } else if (out_len
) {
369 dir
= SCSI_XFER_TO_DEV
;
370 for (i
= 0; i
< out_len
; i
++) {
371 len
+= out
[i
].iov_len
;
374 dir
= SCSI_XFER_FROM_DEV
;
375 for (i
= 0; i
< in_len
; i
++) {
376 len
+= in
[i
].iov_len
;
380 task
= scsi_task_new(cdb_len
, req
->cdb
, dir
, len
);
382 if (dir
== SCSI_XFER_TO_DEV
) {
383 task
->iovector_out
.iov
= (struct scsi_iovec
*)out
;
384 task
->iovector_out
.niov
= out_len
;
385 } else if (dir
== SCSI_XFER_FROM_DEV
) {
386 task
->iovector_in
.iov
= (struct scsi_iovec
*)in
;
387 task
->iovector_in
.niov
= in_len
;
390 PDBG("Sending iscsi cmd (cdb_len=%d, dir=%d, task=%p)",
392 if (!iscsi_scsi_command_sync(ctx
, 0, task
, NULL
)) {
393 PERR("Error serving SCSI command");
398 memset(rsp
, 0, sizeof(*rsp
));
400 rsp
->status
= task
->status
;
401 rsp
->resid
= task
->residual
;
403 if (task
->status
== SCSI_STATUS_CHECK_CONDITION
) {
404 rsp
->response
= VIRTIO_SCSI_S_FAILURE
;
405 rsp
->sense_len
= task
->datain
.size
- 2;
406 memcpy(rsp
->sense
, &task
->datain
.data
[2], rsp
->sense_len
);
411 PDBG("Filled in rsp: status=%hhX, resid=%u, response=%hhX, sense_len=%u",
412 rsp
->status
, rsp
->resid
, rsp
->response
, rsp
->sense_len
);
417 /** libvhost-user callbacks **/
419 static void vus_panic_cb(VuDev
*vu_dev
, const char *buf
)
421 vhost_scsi_dev_t
*vdev_scsi
;
425 vdev_scsi
= container_of(vu_dev
, vhost_scsi_dev_t
, vu_dev
);
427 PERR("vu_panic: %s", buf
);
430 g_main_loop_quit(vdev_scsi
->loop
);
433 static void vus_add_watch_cb(VuDev
*vu_dev
, int fd
, int vu_evt
, vu_watch_cb cb
,
436 vhost_scsi_dev_t
*vdev_scsi
;
443 vdev_scsi
= container_of(vu_dev
, vhost_scsi_dev_t
, vu_dev
);
444 id
= (guint
)(uintptr_t)g_tree_lookup(vdev_scsi
->fdmap
,
445 (gpointer
)(uintptr_t)fd
);
447 GSource
*vus_src
= g_main_context_find_source_by_id(NULL
, id
);
449 g_source_destroy(vus_src
);
450 (void)g_tree_remove(vdev_scsi
->fdmap
, (gpointer
)(uintptr_t)fd
);
453 vus_gsrc_new(vdev_scsi
, fd
, vu_evt
, cb
, NULL
, pvt
);
456 static void vus_del_watch_cb(VuDev
*vu_dev
, int fd
)
458 vhost_scsi_dev_t
*vdev_scsi
;
464 vdev_scsi
= container_of(vu_dev
, vhost_scsi_dev_t
, vu_dev
);
465 id
= (guint
)(uintptr_t)g_tree_lookup(vdev_scsi
->fdmap
,
466 (gpointer
)(uintptr_t)fd
);
468 GSource
*vus_src
= g_main_context_find_source_by_id(NULL
, id
);
470 g_source_destroy(vus_src
);
471 (void)g_tree_remove(vdev_scsi
->fdmap
, (gpointer
)(uintptr_t)fd
);
475 static void vus_proc_ctl(VuDev
*vu_dev
, int idx
)
477 /* Control VQ not implemented */
480 static void vus_proc_evt(VuDev
*vu_dev
, int idx
)
482 /* Event VQ not implemented */
485 static void vus_proc_req(VuDev
*vu_dev
, int idx
)
487 vhost_scsi_dev_t
*vdev_scsi
;
492 vdev_scsi
= container_of(vu_dev
, vhost_scsi_dev_t
, vu_dev
);
493 if (idx
< 0 || idx
>= VHOST_MAX_NR_VIRTQUEUE
) {
494 PERR("VQ Index out of range: %d", idx
);
495 vus_panic_cb(vu_dev
, NULL
);
499 vq
= vu_get_queue(vu_dev
, idx
);
501 PERR("Error fetching VQ (dev=%p, idx=%d)", vu_dev
, idx
);
502 vus_panic_cb(vu_dev
, NULL
);
506 PDBG("Got kicked on vq[%d]@%p", idx
, vq
);
509 VuVirtqElement
*elem
;
510 VirtIOSCSICmdReq
*req
;
511 VirtIOSCSICmdResp
*rsp
;
513 elem
= vu_queue_pop(vu_dev
, vq
, sizeof(VuVirtqElement
));
515 PDBG("No more elements pending on vq[%d]@%p", idx
, vq
);
518 PDBG("Popped elem@%p", elem
);
520 assert(!(elem
->out_num
> 1 && elem
->in_num
> 1));
521 assert(elem
->out_num
> 0 && elem
->in_num
> 0);
523 if (elem
->out_sg
[0].iov_len
< sizeof(VirtIOSCSICmdReq
)) {
524 PERR("Invalid virtio-scsi req header");
525 vus_panic_cb(vu_dev
, NULL
);
528 req
= (VirtIOSCSICmdReq
*)elem
->out_sg
[0].iov_base
;
530 if (elem
->in_sg
[0].iov_len
< sizeof(VirtIOSCSICmdResp
)) {
531 PERR("Invalid virtio-scsi rsp header");
532 vus_panic_cb(vu_dev
, NULL
);
535 rsp
= (VirtIOSCSICmdResp
*)elem
->in_sg
[0].iov_base
;
537 if (handle_cmd_sync(vdev_scsi
->luns
[0].iscsi_ctx
,
538 req
, &elem
->out_sg
[1], elem
->out_num
- 1,
539 rsp
, &elem
->in_sg
[1], elem
->in_num
- 1) != 0) {
540 vus_panic_cb(vu_dev
, NULL
);
544 vu_queue_push(vu_dev
, vq
, elem
, 0);
545 vu_queue_notify(vu_dev
, vq
);
551 static void vus_queue_set_started(VuDev
*vu_dev
, int idx
, bool started
)
557 if (idx
< 0 || idx
>= VHOST_MAX_NR_VIRTQUEUE
) {
558 PERR("VQ Index out of range: %d", idx
);
559 vus_panic_cb(vu_dev
, NULL
);
563 vq
= vu_get_queue(vu_dev
, idx
);
567 vu_set_queue_handler(vu_dev
, vq
, started
? vus_proc_ctl
: NULL
);
570 vu_set_queue_handler(vu_dev
, vq
, started
? vus_proc_evt
: NULL
);
573 vu_set_queue_handler(vu_dev
, vq
, started
? vus_proc_req
: NULL
);
577 static const VuDevIface vus_iface
= {
578 .queue_set_started
= vus_queue_set_started
,
581 static gboolean
vus_vhost_cb(gpointer data
)
583 VuDev
*vu_dev
= (VuDev
*)data
;
587 if (!vu_dispatch(vu_dev
) != 0) {
588 PERR("Error processing vhost message");
589 vus_panic_cb(vu_dev
, NULL
);
590 return G_SOURCE_REMOVE
;
593 return G_SOURCE_CONTINUE
;
598 static int unix_sock_new(char *unix_fn
)
601 struct sockaddr_un un
;
606 sock
= socket(AF_UNIX
, SOCK_STREAM
, 0);
612 un
.sun_family
= AF_UNIX
;
613 (void)snprintf(un
.sun_path
, sizeof(un
.sun_path
), "%s", unix_fn
);
614 len
= sizeof(un
.sun_family
) + strlen(un
.sun_path
);
616 (void)unlink(unix_fn
);
617 if (bind(sock
, (struct sockaddr
*)&un
, len
) < 0) {
622 if (listen(sock
, 1) < 0) {
635 /** vhost-user-scsi **/
637 static void vdev_scsi_free(vhost_scsi_dev_t
*vdev_scsi
)
639 if (vdev_scsi
->server_sock
>= 0) {
640 close(vdev_scsi
->server_sock
);
642 g_main_loop_unref(vdev_scsi
->loop
);
643 g_tree_destroy(vdev_scsi
->fdmap
);
647 static vhost_scsi_dev_t
*vdev_scsi_new(int server_sock
)
649 vhost_scsi_dev_t
*vdev_scsi
;
651 vdev_scsi
= g_new0(vhost_scsi_dev_t
, 1);
652 vdev_scsi
->server_sock
= server_sock
;
653 vdev_scsi
->loop
= g_main_loop_new(NULL
, FALSE
);
654 vdev_scsi
->fdmap
= g_tree_new(vus_fdmap_compare
);
659 static int vdev_scsi_add_iscsi_lun(vhost_scsi_dev_t
*vdev_scsi
,
660 char *iscsi_uri
, uint32_t lun
)
664 assert(lun
< VUS_MAX_LUNS
);
666 if (vdev_scsi
->luns
[lun
].iscsi_ctx
) {
667 PERR("Lun %d already configured", lun
);
671 if (iscsi_add_lun(&vdev_scsi
->luns
[lun
], iscsi_uri
) != 0) {
678 static int vdev_scsi_run(vhost_scsi_dev_t
*vdev_scsi
)
684 assert(vdev_scsi
->server_sock
>= 0);
685 assert(vdev_scsi
->loop
);
687 cli_sock
= accept(vdev_scsi
->server_sock
, (void *)0, (void *)0);
693 vu_init(&vdev_scsi
->vu_dev
,
700 vus_gsrc_new(vdev_scsi
, cli_sock
, G_IO_IN
, NULL
, vus_vhost_cb
,
703 g_main_loop_run(vdev_scsi
->loop
);
705 vu_deinit(&vdev_scsi
->vu_dev
);
710 int main(int argc
, char **argv
)
712 vhost_scsi_dev_t
*vdev_scsi
= NULL
;
713 char *unix_fn
= NULL
;
714 char *iscsi_uri
= NULL
;
715 int sock
, opt
, err
= EXIT_SUCCESS
;
717 while ((opt
= getopt(argc
, argv
, "u:i:")) != -1) {
722 unix_fn
= g_strdup(optarg
);
725 iscsi_uri
= g_strdup(optarg
);
731 if (!unix_fn
|| !iscsi_uri
) {
735 sock
= unix_sock_new(unix_fn
);
739 vdev_scsi
= vdev_scsi_new(sock
);
741 if (vdev_scsi_add_iscsi_lun(vdev_scsi
, iscsi_uri
, 0) != 0) {
745 if (vdev_scsi_run(vdev_scsi
) != 0) {
751 vdev_scsi_free(vdev_scsi
);
764 fprintf(stderr
, "Usage: %s [ -u unix_sock_path -i iscsi_uri ] | [ -h ]\n",
766 fprintf(stderr
, " -u path to unix socket\n");
767 fprintf(stderr
, " -i iscsi uri for lun 0\n");
768 fprintf(stderr
, " -h print help and quit\n");