2 * Copyright (c) 2018 Jakub Jermar
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
9 * - Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * - Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * - The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 /** @file VIRTIO support
32 #include "virtio-pci.h"
41 /** Allocate DMA buffers
43 * @param buffers[in] Number of buffers to allocate.
44 * @param size[in] Size of each buffer.
45 * @param write[in] True if the buffers are writable by the driver, false
47 * @param buf[out] Output array holding virtual addresses of the allocated
49 * @param buf_p[out] Output array holding physical addresses of the allocated
52 * The buffers can be deallocated by virtio_teardown_dma_bufs().
54 * @return EOK on success or error code.
56 errno_t
virtio_setup_dma_bufs(unsigned int buffers
, size_t size
,
57 bool write
, void *buf
[], uintptr_t buf_p
[])
60 * Allocate all buffers at once in one large chunk.
62 void *virt
= AS_AREA_ANY
;
64 errno_t rc
= dmamem_map_anonymous(buffers
* size
, 0,
65 write
? AS_AREA_WRITE
: AS_AREA_READ
, 0, &phys
, &virt
);
69 ddf_msg(LVL_NOTE
, "DMA buffers: %p-%p", virt
, virt
+ buffers
* size
);
72 * Calculate addresses of the individual buffers for easy access.
74 for (unsigned i
= 0; i
< buffers
; i
++) {
75 buf
[i
] = virt
+ i
* size
;
76 buf_p
[i
] = phys
+ i
* size
;
82 /** Deallocate DMA buffers
84 * @param buf[in] Array holding the virtual addresses of the DMA buffers
85 * previously allocated by virtio_setup_dma_bufs().
87 extern void virtio_teardown_dma_bufs(void *buf
[])
90 dmamem_unmap_anonymous(buf
[0]);
95 void virtio_virtq_desc_set(virtio_dev_t
*vdev
, uint16_t num
, uint16_t descno
,
96 uint64_t addr
, uint32_t len
, uint16_t flags
, uint16_t next
)
98 virtq_desc_t
*d
= &vdev
->queues
[num
].desc
[descno
];
99 pio_write_le64(&d
->addr
, addr
);
100 pio_write_le32(&d
->len
, len
);
101 pio_write_le16(&d
->flags
, flags
);
102 pio_write_le16(&d
->next
, next
);
105 uint16_t virtio_virtq_desc_get_next(virtio_dev_t
*vdev
, uint16_t num
,
108 virtq_desc_t
*d
= &vdev
->queues
[num
].desc
[descno
];
109 if (!(pio_read_le16(&d
->flags
) & VIRTQ_DESC_F_NEXT
))
110 return (uint16_t) -1U;
111 return pio_read_le16(&d
->next
);
114 /** Create free descriptor list from the unused VIRTIO descriptors
116 * @param vdev[in] VIRTIO device for which the free list will be created.
117 * @param num[in] Index of the virtqueue for which the free list will be
119 * @param size[in] Number of descriptors on the free list. The free list will
120 * contain descriptors starting from 0 to \a size - 1.
121 * @param head[out] Variable that will hold the VIRTIO descriptor at the head
124 void virtio_create_desc_free_list(virtio_dev_t
*vdev
, uint16_t num
,
125 uint16_t size
, uint16_t *head
)
127 for (unsigned i
= 0; i
< size
; i
++) {
128 virtio_virtq_desc_set(vdev
, num
, i
, 0, 0,
129 VIRTQ_DESC_F_NEXT
, (i
+ 1 == size
) ? 0xffffu
: i
+ 1);
134 /** Allocate a descriptor from the free list
136 * @param vdev[in] VIRTIO device with the free list.
137 * @param num[in] Index of the virtqueue with free list.
138 * @param head[in,out] Head of the free list.
140 * @return Allocated descriptor or 0xFFFF if the list is empty.
142 uint16_t virtio_alloc_desc(virtio_dev_t
*vdev
, uint16_t num
, uint16_t *head
)
144 virtq_t
*q
= &vdev
->queues
[num
];
145 fibril_mutex_lock(&q
->lock
);
146 uint16_t descno
= *head
;
147 if (descno
!= (uint16_t) -1U)
148 *head
= virtio_virtq_desc_get_next(vdev
, num
, descno
);
149 fibril_mutex_unlock(&q
->lock
);
153 /** Free a descriptor into the free list
155 * @param vdev[in] VIRTIO device with the free list.
156 * @param num[in] Index of the virtqueue with free list.
157 * @param head[in,out] Head of the free list.
158 * @param descno[in] The freed descriptor.
160 void virtio_free_desc(virtio_dev_t
*vdev
, uint16_t num
, uint16_t *head
,
163 virtq_t
*q
= &vdev
->queues
[num
];
164 fibril_mutex_lock(&q
->lock
);
165 virtio_virtq_desc_set(vdev
, num
, descno
, 0, 0, VIRTQ_DESC_F_NEXT
,
168 fibril_mutex_unlock(&q
->lock
);
171 void virtio_virtq_produce_available(virtio_dev_t
*vdev
, uint16_t num
,
174 virtq_t
*q
= &vdev
->queues
[num
];
176 fibril_mutex_lock(&q
->lock
);
177 uint16_t idx
= pio_read_le16(&q
->avail
->idx
);
178 pio_write_le16(&q
->avail
->ring
[idx
% q
->queue_size
], descno
);
180 pio_write_le16(&q
->avail
->idx
, idx
+ 1);
182 pio_write_le16(q
->notify
, num
);
183 fibril_mutex_unlock(&q
->lock
);
186 bool virtio_virtq_consume_used(virtio_dev_t
*vdev
, uint16_t num
,
187 uint16_t *descno
, uint32_t *len
)
189 virtq_t
*q
= &vdev
->queues
[num
];
191 fibril_mutex_lock(&q
->lock
);
192 uint16_t last_idx
= q
->used_last_idx
% q
->queue_size
;
193 if (last_idx
== (pio_read_le16(&q
->used
->idx
) % q
->queue_size
)) {
194 fibril_mutex_unlock(&q
->lock
);
198 *descno
= (uint16_t) pio_read_le32(&q
->used
->ring
[last_idx
].id
);
199 *len
= pio_read_le32(&q
->used
->ring
[last_idx
].len
);
202 fibril_mutex_unlock(&q
->lock
);
207 errno_t
virtio_virtq_setup(virtio_dev_t
*vdev
, uint16_t num
, uint16_t size
)
209 virtq_t
*q
= &vdev
->queues
[num
];
210 virtio_pci_common_cfg_t
*cfg
= vdev
->common_cfg
;
212 /* Program the queue of our interest */
213 pio_write_le16(&cfg
->queue_select
, num
);
215 /* Trim the size of the queue as needed */
216 if (size
> pio_read_16(&cfg
->queue_size
)) {
217 ddf_msg(LVL_ERROR
, "Virtq %u: not enough descriptors", num
);
220 pio_write_le16(&cfg
->queue_size
, size
);
221 ddf_msg(LVL_NOTE
, "Virtq %u: %u descriptors", num
, (unsigned) size
);
223 size_t avail_offset
= 0;
224 size_t used_offset
= 0;
227 * Compute the size of the needed DMA memory and also the offsets of
228 * the individual components
230 size_t mem_size
= sizeof(virtq_desc_t
[size
]);
231 mem_size
= ALIGN_UP(mem_size
, _Alignof(virtq_avail_t
));
232 avail_offset
= mem_size
;
233 mem_size
+= sizeof(virtq_avail_t
) + sizeof(ioport16_t
[size
]) +
235 mem_size
= ALIGN_UP(mem_size
, _Alignof(virtq_used_t
));
236 used_offset
= mem_size
;
237 mem_size
+= sizeof(virtq_used_t
) + sizeof(virtq_used_elem_t
[size
]) +
241 * Allocate DMA memory for the virtqueues
243 q
->virt
= AS_AREA_ANY
;
244 errno_t rc
= dmamem_map_anonymous(mem_size
, 0,
245 AS_AREA_READ
| AS_AREA_WRITE
, 0, &q
->phys
, &q
->virt
);
251 fibril_mutex_initialize(&q
->lock
);
254 q
->queue_size
= size
;
256 q
->avail
= q
->virt
+ avail_offset
;
257 q
->used
= q
->virt
+ used_offset
;
258 q
->used_last_idx
= 0;
260 memset(q
->virt
, 0, q
->size
);
263 * Write the configured addresses to device's common config
265 pio_write_le64(&cfg
->queue_desc
, q
->phys
);
266 pio_write_le64(&cfg
->queue_avail
, q
->phys
+ avail_offset
);
267 pio_write_le64(&cfg
->queue_used
, q
->phys
+ used_offset
);
269 ddf_msg(LVL_NOTE
, "DMA memory for virtq %d: virt=%p, phys=%p, size=%zu",
270 num
, q
->virt
, (void *) q
->phys
, q
->size
);
272 /* Determine virtq's notification address */
273 q
->notify
= vdev
->notify_base
+
274 pio_read_le16(&cfg
->queue_notif_off
) * vdev
->notify_off_multiplier
;
276 ddf_msg(LVL_NOTE
, "notification register: %p", q
->notify
);
278 /* Enable the queue */
279 pio_write_le16(&cfg
->queue_enable
, 1);
280 ddf_msg(LVL_NOTE
, "virtq %d set", num
);
285 void virtio_virtq_teardown(virtio_dev_t
*vdev
, uint16_t num
)
287 virtio_pci_common_cfg_t
*cfg
= vdev
->common_cfg
;
289 /* Disable the queue */
290 pio_write_le16(&cfg
->queue_enable
, 0);
292 virtq_t
*q
= &vdev
->queues
[num
];
294 dmamem_unmap_anonymous(q
->virt
);
298 * Perform device initialization as described in section 3.1.1 of the
299 * specification, steps 1 - 6.
301 errno_t
virtio_device_setup_start(virtio_dev_t
*vdev
, uint32_t features
)
303 virtio_pci_common_cfg_t
*cfg
= vdev
->common_cfg
;
305 /* 1. Reset the device */
306 uint8_t status
= VIRTIO_DEV_STATUS_RESET
;
307 pio_write_8(&cfg
->device_status
, status
);
309 /* 2. Acknowledge we found the device */
310 status
|= VIRTIO_DEV_STATUS_ACKNOWLEDGE
;
311 pio_write_8(&cfg
->device_status
, status
);
313 /* 3. We know how to drive the device */
314 status
|= VIRTIO_DEV_STATUS_DRIVER
;
315 pio_write_8(&cfg
->device_status
, status
);
317 /* 4. Read the offered feature flags */
318 pio_write_le32(&cfg
->device_feature_select
, VIRTIO_FEATURES_0_31
);
319 uint32_t device_features
= pio_read_le32(&cfg
->device_feature
);
321 ddf_msg(LVL_NOTE
, "offered features %x", device_features
);
323 if (features
!= (features
& device_features
))
325 features
&= device_features
;
327 /* 4. Write the accepted feature flags */
328 pio_write_le32(&cfg
->driver_feature_select
, VIRTIO_FEATURES_0_31
);
329 pio_write_le32(&cfg
->driver_feature
, features
);
331 ddf_msg(LVL_NOTE
, "accepted features %x", features
);
333 /* 5. Set FEATURES_OK */
334 status
|= VIRTIO_DEV_STATUS_FEATURES_OK
;
335 pio_write_8(&cfg
->device_status
, status
);
337 /* 6. Test if the device supports our feature subset */
338 status
= pio_read_8(&cfg
->device_status
);
339 if (!(status
& VIRTIO_DEV_STATUS_FEATURES_OK
))
346 * Perform device initialization as described in section 3.1.1 of the
347 * specification, step 8 (go live).
349 void virtio_device_setup_finalize(virtio_dev_t
*vdev
)
351 virtio_pci_common_cfg_t
*cfg
= vdev
->common_cfg
;
354 uint8_t status
= pio_read_8(&cfg
->device_status
);
355 pio_write_8(&cfg
->device_status
, status
| VIRTIO_DEV_STATUS_DRIVER_OK
);
358 void virtio_device_setup_fail(virtio_dev_t
*vdev
)
360 virtio_pci_common_cfg_t
*cfg
= vdev
->common_cfg
;
362 uint8_t status
= pio_read_8(&cfg
->device_status
);
363 pio_write_8(&cfg
->device_status
, status
| VIRTIO_DEV_STATUS_FAILED
);