2 * virtio transport for vsock
4 * Copyright (C) 2013-2015 Red Hat, Inc.
5 * Author: Asias He <asias@redhat.com>
6 * Stefan Hajnoczi <stefanha@redhat.com>
8 * Some of the code is take from Gerd Hoffmann <kraxel@redhat.com>'s
9 * early virtio-vsock proof-of-concept bits.
11 * This work is licensed under the terms of the GNU GPL, version 2.
13 #include <linux/spinlock.h>
14 #include <linux/module.h>
15 #include <linux/list.h>
16 #include <linux/atomic.h>
17 #include <linux/virtio.h>
18 #include <linux/virtio_ids.h>
19 #include <linux/virtio_config.h>
20 #include <linux/virtio_vsock.h>
22 #include <linux/mutex.h>
23 #include <net/af_vsock.h>
25 static struct workqueue_struct
*virtio_vsock_workqueue
;
26 static struct virtio_vsock
*the_virtio_vsock
;
27 static DEFINE_MUTEX(the_virtio_vsock_mutex
); /* protects the_virtio_vsock */
30 struct virtio_device
*vdev
;
31 struct virtqueue
*vqs
[VSOCK_VQ_MAX
];
33 /* Virtqueue processing is deferred to a workqueue */
34 struct work_struct tx_work
;
35 struct work_struct rx_work
;
36 struct work_struct event_work
;
38 /* The following fields are protected by tx_lock. vqs[VSOCK_VQ_TX]
39 * must be accessed with tx_lock held.
43 struct work_struct send_pkt_work
;
44 spinlock_t send_pkt_list_lock
;
45 struct list_head send_pkt_list
;
47 atomic_t queued_replies
;
49 /* The following fields are protected by rx_lock. vqs[VSOCK_VQ_RX]
50 * must be accessed with rx_lock held.
56 /* The following fields are protected by event_lock.
57 * vqs[VSOCK_VQ_EVENT] must be accessed with event_lock held.
59 struct mutex event_lock
;
60 struct virtio_vsock_event event_list
[8];
65 static struct virtio_vsock
*virtio_vsock_get(void)
67 return the_virtio_vsock
;
70 static u32
virtio_transport_get_local_cid(void)
72 struct virtio_vsock
*vsock
= virtio_vsock_get();
74 return vsock
->guest_cid
;
78 virtio_transport_send_pkt_work(struct work_struct
*work
)
80 struct virtio_vsock
*vsock
=
81 container_of(work
, struct virtio_vsock
, send_pkt_work
);
84 bool restart_rx
= false;
86 mutex_lock(&vsock
->tx_lock
);
88 vq
= vsock
->vqs
[VSOCK_VQ_TX
];
91 struct virtio_vsock_pkt
*pkt
;
92 struct scatterlist hdr
, buf
, *sgs
[2];
93 int ret
, in_sg
= 0, out_sg
= 0;
96 spin_lock_bh(&vsock
->send_pkt_list_lock
);
97 if (list_empty(&vsock
->send_pkt_list
)) {
98 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
102 pkt
= list_first_entry(&vsock
->send_pkt_list
,
103 struct virtio_vsock_pkt
, list
);
104 list_del_init(&pkt
->list
);
105 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
109 sg_init_one(&hdr
, &pkt
->hdr
, sizeof(pkt
->hdr
));
110 sgs
[out_sg
++] = &hdr
;
112 sg_init_one(&buf
, pkt
->buf
, pkt
->len
);
113 sgs
[out_sg
++] = &buf
;
116 ret
= virtqueue_add_sgs(vq
, sgs
, out_sg
, in_sg
, pkt
, GFP_KERNEL
);
117 /* Usually this means that there is no more space available in
121 spin_lock_bh(&vsock
->send_pkt_list_lock
);
122 list_add(&pkt
->list
, &vsock
->send_pkt_list
);
123 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
128 struct virtqueue
*rx_vq
= vsock
->vqs
[VSOCK_VQ_RX
];
131 val
= atomic_dec_return(&vsock
->queued_replies
);
133 /* Do we now have resources to resume rx processing? */
134 if (val
+ 1 == virtqueue_get_vring_size(rx_vq
))
144 mutex_unlock(&vsock
->tx_lock
);
147 queue_work(virtio_vsock_workqueue
, &vsock
->rx_work
);
151 virtio_transport_send_pkt(struct virtio_vsock_pkt
*pkt
)
153 struct virtio_vsock
*vsock
;
156 vsock
= virtio_vsock_get();
158 virtio_transport_free_pkt(pkt
);
163 atomic_inc(&vsock
->queued_replies
);
165 spin_lock_bh(&vsock
->send_pkt_list_lock
);
166 list_add_tail(&pkt
->list
, &vsock
->send_pkt_list
);
167 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
169 queue_work(virtio_vsock_workqueue
, &vsock
->send_pkt_work
);
173 static void virtio_vsock_rx_fill(struct virtio_vsock
*vsock
)
175 int buf_len
= VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE
;
176 struct virtio_vsock_pkt
*pkt
;
177 struct scatterlist hdr
, buf
, *sgs
[2];
178 struct virtqueue
*vq
;
181 vq
= vsock
->vqs
[VSOCK_VQ_RX
];
184 pkt
= kzalloc(sizeof(*pkt
), GFP_KERNEL
);
188 pkt
->buf
= kmalloc(buf_len
, GFP_KERNEL
);
190 virtio_transport_free_pkt(pkt
);
196 sg_init_one(&hdr
, &pkt
->hdr
, sizeof(pkt
->hdr
));
199 sg_init_one(&buf
, pkt
->buf
, buf_len
);
201 ret
= virtqueue_add_sgs(vq
, sgs
, 0, 2, pkt
, GFP_KERNEL
);
203 virtio_transport_free_pkt(pkt
);
207 } while (vq
->num_free
);
208 if (vsock
->rx_buf_nr
> vsock
->rx_buf_max_nr
)
209 vsock
->rx_buf_max_nr
= vsock
->rx_buf_nr
;
213 static void virtio_transport_tx_work(struct work_struct
*work
)
215 struct virtio_vsock
*vsock
=
216 container_of(work
, struct virtio_vsock
, tx_work
);
217 struct virtqueue
*vq
;
220 vq
= vsock
->vqs
[VSOCK_VQ_TX
];
221 mutex_lock(&vsock
->tx_lock
);
223 struct virtio_vsock_pkt
*pkt
;
226 virtqueue_disable_cb(vq
);
227 while ((pkt
= virtqueue_get_buf(vq
, &len
)) != NULL
) {
228 virtio_transport_free_pkt(pkt
);
231 } while (!virtqueue_enable_cb(vq
));
232 mutex_unlock(&vsock
->tx_lock
);
235 queue_work(virtio_vsock_workqueue
, &vsock
->send_pkt_work
);
238 /* Is there space left for replies to rx packets? */
239 static bool virtio_transport_more_replies(struct virtio_vsock
*vsock
)
241 struct virtqueue
*vq
= vsock
->vqs
[VSOCK_VQ_RX
];
244 smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */
245 val
= atomic_read(&vsock
->queued_replies
);
247 return val
< virtqueue_get_vring_size(vq
);
250 static void virtio_transport_rx_work(struct work_struct
*work
)
252 struct virtio_vsock
*vsock
=
253 container_of(work
, struct virtio_vsock
, rx_work
);
254 struct virtqueue
*vq
;
256 vq
= vsock
->vqs
[VSOCK_VQ_RX
];
258 mutex_lock(&vsock
->rx_lock
);
261 virtqueue_disable_cb(vq
);
263 struct virtio_vsock_pkt
*pkt
;
266 if (!virtio_transport_more_replies(vsock
)) {
267 /* Stop rx until the device processes already
268 * pending replies. Leave rx virtqueue
269 * callbacks disabled.
274 pkt
= virtqueue_get_buf(vq
, &len
);
281 /* Drop short/long packets */
282 if (unlikely(len
< sizeof(pkt
->hdr
) ||
283 len
> sizeof(pkt
->hdr
) + pkt
->len
)) {
284 virtio_transport_free_pkt(pkt
);
288 pkt
->len
= len
- sizeof(pkt
->hdr
);
289 virtio_transport_recv_pkt(pkt
);
291 } while (!virtqueue_enable_cb(vq
));
294 if (vsock
->rx_buf_nr
< vsock
->rx_buf_max_nr
/ 2)
295 virtio_vsock_rx_fill(vsock
);
296 mutex_unlock(&vsock
->rx_lock
);
299 /* event_lock must be held */
300 static int virtio_vsock_event_fill_one(struct virtio_vsock
*vsock
,
301 struct virtio_vsock_event
*event
)
303 struct scatterlist sg
;
304 struct virtqueue
*vq
;
306 vq
= vsock
->vqs
[VSOCK_VQ_EVENT
];
308 sg_init_one(&sg
, event
, sizeof(*event
));
310 return virtqueue_add_inbuf(vq
, &sg
, 1, event
, GFP_KERNEL
);
313 /* event_lock must be held */
314 static void virtio_vsock_event_fill(struct virtio_vsock
*vsock
)
318 for (i
= 0; i
< ARRAY_SIZE(vsock
->event_list
); i
++) {
319 struct virtio_vsock_event
*event
= &vsock
->event_list
[i
];
321 virtio_vsock_event_fill_one(vsock
, event
);
324 virtqueue_kick(vsock
->vqs
[VSOCK_VQ_EVENT
]);
327 static void virtio_vsock_reset_sock(struct sock
*sk
)
330 sk
->sk_state
= SS_UNCONNECTED
;
331 sk
->sk_err
= ECONNRESET
;
332 sk
->sk_error_report(sk
);
336 static void virtio_vsock_update_guest_cid(struct virtio_vsock
*vsock
)
338 struct virtio_device
*vdev
= vsock
->vdev
;
341 vdev
->config
->get(vdev
, offsetof(struct virtio_vsock_config
, guest_cid
),
342 &guest_cid
, sizeof(guest_cid
));
343 vsock
->guest_cid
= le64_to_cpu(guest_cid
);
346 /* event_lock must be held */
347 static void virtio_vsock_event_handle(struct virtio_vsock
*vsock
,
348 struct virtio_vsock_event
*event
)
350 switch (le32_to_cpu(event
->id
)) {
351 case VIRTIO_VSOCK_EVENT_TRANSPORT_RESET
:
352 virtio_vsock_update_guest_cid(vsock
);
353 vsock_for_each_connected_socket(virtio_vsock_reset_sock
);
358 static void virtio_transport_event_work(struct work_struct
*work
)
360 struct virtio_vsock
*vsock
=
361 container_of(work
, struct virtio_vsock
, event_work
);
362 struct virtqueue
*vq
;
364 vq
= vsock
->vqs
[VSOCK_VQ_EVENT
];
366 mutex_lock(&vsock
->event_lock
);
369 struct virtio_vsock_event
*event
;
372 virtqueue_disable_cb(vq
);
373 while ((event
= virtqueue_get_buf(vq
, &len
)) != NULL
) {
374 if (len
== sizeof(*event
))
375 virtio_vsock_event_handle(vsock
, event
);
377 virtio_vsock_event_fill_one(vsock
, event
);
379 } while (!virtqueue_enable_cb(vq
));
381 virtqueue_kick(vsock
->vqs
[VSOCK_VQ_EVENT
]);
383 mutex_unlock(&vsock
->event_lock
);
386 static void virtio_vsock_event_done(struct virtqueue
*vq
)
388 struct virtio_vsock
*vsock
= vq
->vdev
->priv
;
392 queue_work(virtio_vsock_workqueue
, &vsock
->event_work
);
395 static void virtio_vsock_tx_done(struct virtqueue
*vq
)
397 struct virtio_vsock
*vsock
= vq
->vdev
->priv
;
401 queue_work(virtio_vsock_workqueue
, &vsock
->tx_work
);
404 static void virtio_vsock_rx_done(struct virtqueue
*vq
)
406 struct virtio_vsock
*vsock
= vq
->vdev
->priv
;
410 queue_work(virtio_vsock_workqueue
, &vsock
->rx_work
);
413 static struct virtio_transport virtio_transport
= {
415 .get_local_cid
= virtio_transport_get_local_cid
,
417 .init
= virtio_transport_do_socket_init
,
418 .destruct
= virtio_transport_destruct
,
419 .release
= virtio_transport_release
,
420 .connect
= virtio_transport_connect
,
421 .shutdown
= virtio_transport_shutdown
,
423 .dgram_bind
= virtio_transport_dgram_bind
,
424 .dgram_dequeue
= virtio_transport_dgram_dequeue
,
425 .dgram_enqueue
= virtio_transport_dgram_enqueue
,
426 .dgram_allow
= virtio_transport_dgram_allow
,
428 .stream_dequeue
= virtio_transport_stream_dequeue
,
429 .stream_enqueue
= virtio_transport_stream_enqueue
,
430 .stream_has_data
= virtio_transport_stream_has_data
,
431 .stream_has_space
= virtio_transport_stream_has_space
,
432 .stream_rcvhiwat
= virtio_transport_stream_rcvhiwat
,
433 .stream_is_active
= virtio_transport_stream_is_active
,
434 .stream_allow
= virtio_transport_stream_allow
,
436 .notify_poll_in
= virtio_transport_notify_poll_in
,
437 .notify_poll_out
= virtio_transport_notify_poll_out
,
438 .notify_recv_init
= virtio_transport_notify_recv_init
,
439 .notify_recv_pre_block
= virtio_transport_notify_recv_pre_block
,
440 .notify_recv_pre_dequeue
= virtio_transport_notify_recv_pre_dequeue
,
441 .notify_recv_post_dequeue
= virtio_transport_notify_recv_post_dequeue
,
442 .notify_send_init
= virtio_transport_notify_send_init
,
443 .notify_send_pre_block
= virtio_transport_notify_send_pre_block
,
444 .notify_send_pre_enqueue
= virtio_transport_notify_send_pre_enqueue
,
445 .notify_send_post_enqueue
= virtio_transport_notify_send_post_enqueue
,
447 .set_buffer_size
= virtio_transport_set_buffer_size
,
448 .set_min_buffer_size
= virtio_transport_set_min_buffer_size
,
449 .set_max_buffer_size
= virtio_transport_set_max_buffer_size
,
450 .get_buffer_size
= virtio_transport_get_buffer_size
,
451 .get_min_buffer_size
= virtio_transport_get_min_buffer_size
,
452 .get_max_buffer_size
= virtio_transport_get_max_buffer_size
,
455 .send_pkt
= virtio_transport_send_pkt
,
458 static int virtio_vsock_probe(struct virtio_device
*vdev
)
460 vq_callback_t
*callbacks
[] = {
461 virtio_vsock_rx_done
,
462 virtio_vsock_tx_done
,
463 virtio_vsock_event_done
,
465 static const char * const names
[] = {
470 struct virtio_vsock
*vsock
= NULL
;
473 ret
= mutex_lock_interruptible(&the_virtio_vsock_mutex
);
477 /* Only one virtio-vsock device per guest is supported */
478 if (the_virtio_vsock
) {
483 vsock
= kzalloc(sizeof(*vsock
), GFP_KERNEL
);
491 ret
= vsock
->vdev
->config
->find_vqs(vsock
->vdev
, VSOCK_VQ_MAX
,
492 vsock
->vqs
, callbacks
, names
);
496 virtio_vsock_update_guest_cid(vsock
);
498 ret
= vsock_core_init(&virtio_transport
.transport
);
502 vsock
->rx_buf_nr
= 0;
503 vsock
->rx_buf_max_nr
= 0;
504 atomic_set(&vsock
->queued_replies
, 0);
507 the_virtio_vsock
= vsock
;
508 mutex_init(&vsock
->tx_lock
);
509 mutex_init(&vsock
->rx_lock
);
510 mutex_init(&vsock
->event_lock
);
511 spin_lock_init(&vsock
->send_pkt_list_lock
);
512 INIT_LIST_HEAD(&vsock
->send_pkt_list
);
513 INIT_WORK(&vsock
->rx_work
, virtio_transport_rx_work
);
514 INIT_WORK(&vsock
->tx_work
, virtio_transport_tx_work
);
515 INIT_WORK(&vsock
->event_work
, virtio_transport_event_work
);
516 INIT_WORK(&vsock
->send_pkt_work
, virtio_transport_send_pkt_work
);
518 mutex_lock(&vsock
->rx_lock
);
519 virtio_vsock_rx_fill(vsock
);
520 mutex_unlock(&vsock
->rx_lock
);
522 mutex_lock(&vsock
->event_lock
);
523 virtio_vsock_event_fill(vsock
);
524 mutex_unlock(&vsock
->event_lock
);
526 mutex_unlock(&the_virtio_vsock_mutex
);
530 vsock
->vdev
->config
->del_vqs(vsock
->vdev
);
533 mutex_unlock(&the_virtio_vsock_mutex
);
537 static void virtio_vsock_remove(struct virtio_device
*vdev
)
539 struct virtio_vsock
*vsock
= vdev
->priv
;
540 struct virtio_vsock_pkt
*pkt
;
542 flush_work(&vsock
->rx_work
);
543 flush_work(&vsock
->tx_work
);
544 flush_work(&vsock
->event_work
);
545 flush_work(&vsock
->send_pkt_work
);
547 vdev
->config
->reset(vdev
);
549 mutex_lock(&vsock
->rx_lock
);
550 while ((pkt
= virtqueue_detach_unused_buf(vsock
->vqs
[VSOCK_VQ_RX
])))
551 virtio_transport_free_pkt(pkt
);
552 mutex_unlock(&vsock
->rx_lock
);
554 mutex_lock(&vsock
->tx_lock
);
555 while ((pkt
= virtqueue_detach_unused_buf(vsock
->vqs
[VSOCK_VQ_TX
])))
556 virtio_transport_free_pkt(pkt
);
557 mutex_unlock(&vsock
->tx_lock
);
559 spin_lock_bh(&vsock
->send_pkt_list_lock
);
560 while (!list_empty(&vsock
->send_pkt_list
)) {
561 pkt
= list_first_entry(&vsock
->send_pkt_list
,
562 struct virtio_vsock_pkt
, list
);
563 list_del(&pkt
->list
);
564 virtio_transport_free_pkt(pkt
);
566 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
568 mutex_lock(&the_virtio_vsock_mutex
);
569 the_virtio_vsock
= NULL
;
571 mutex_unlock(&the_virtio_vsock_mutex
);
573 vdev
->config
->del_vqs(vdev
);
578 static struct virtio_device_id id_table
[] = {
579 { VIRTIO_ID_VSOCK
, VIRTIO_DEV_ANY_ID
},
583 static unsigned int features
[] = {
586 static struct virtio_driver virtio_vsock_driver
= {
587 .feature_table
= features
,
588 .feature_table_size
= ARRAY_SIZE(features
),
589 .driver
.name
= KBUILD_MODNAME
,
590 .driver
.owner
= THIS_MODULE
,
591 .id_table
= id_table
,
592 .probe
= virtio_vsock_probe
,
593 .remove
= virtio_vsock_remove
,
596 static int __init
virtio_vsock_init(void)
600 virtio_vsock_workqueue
= alloc_workqueue("virtio_vsock", 0, 0);
601 if (!virtio_vsock_workqueue
)
603 ret
= register_virtio_driver(&virtio_vsock_driver
);
605 destroy_workqueue(virtio_vsock_workqueue
);
609 static void __exit
virtio_vsock_exit(void)
611 unregister_virtio_driver(&virtio_vsock_driver
);
612 destroy_workqueue(virtio_vsock_workqueue
);
615 module_init(virtio_vsock_init
);
616 module_exit(virtio_vsock_exit
);
617 MODULE_LICENSE("GPL v2");
618 MODULE_AUTHOR("Asias He");
619 MODULE_DESCRIPTION("virtio transport for vsock");
620 MODULE_DEVICE_TABLE(virtio
, id_table
);