2 * Copyright (c) 2018 Citrix Systems Inc.
3 * (c) Gerd Hoffmann <kraxel@redhat.com>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; under version 2 of the License.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License along
15 * with this program; if not, see <http://www.gnu.org/licenses/>.
17 * Contributions after 2012-01-13 are licensed under the terms of the
18 * GNU GPL, version 2 or (at your option) any later version.
21 #include "qemu/osdep.h"
22 #include "qemu/error-report.h"
23 #include "qapi/error.h"
25 #include "hw/xen/xen_common.h"
26 #include "hw/block/xen_blkif.h"
27 #include "sysemu/block-backend.h"
28 #include "sysemu/iothread.h"
29 #include "xen-block.h"
31 typedef struct XenBlockRequest
{
41 XenBlockDataPlane
*dataplane
;
42 QLIST_ENTRY(XenBlockRequest
) list
;
46 struct XenBlockDataPlane
{
48 XenEventChannel
*event_channel
;
49 unsigned int *ring_ref
;
50 unsigned int nr_ring_ref
;
55 blkif_back_rings_t rings
;
57 QLIST_HEAD(inflight_head
, XenBlockRequest
) inflight
;
58 QLIST_HEAD(finished_head
, XenBlockRequest
) finished
;
59 QLIST_HEAD(freelist_head
, XenBlockRequest
) freelist
;
61 int requests_inflight
;
62 int requests_finished
;
63 unsigned int max_requests
;
70 static void reset_request(XenBlockRequest
*request
)
72 memset(&request
->req
, 0, sizeof(request
->req
));
79 request
->aio_inflight
= 0;
80 request
->aio_errors
= 0;
82 request
->dataplane
= NULL
;
83 memset(&request
->list
, 0, sizeof(request
->list
));
84 memset(&request
->acct
, 0, sizeof(request
->acct
));
86 qemu_iovec_reset(&request
->v
);
89 static XenBlockRequest
*xen_block_start_request(XenBlockDataPlane
*dataplane
)
91 XenBlockRequest
*request
= NULL
;
93 if (QLIST_EMPTY(&dataplane
->freelist
)) {
94 if (dataplane
->requests_total
>= dataplane
->max_requests
) {
97 /* allocate new struct */
98 request
= g_malloc0(sizeof(*request
));
99 request
->dataplane
= dataplane
;
100 dataplane
->requests_total
++;
101 qemu_iovec_init(&request
->v
, 1);
103 /* get one from freelist */
104 request
= QLIST_FIRST(&dataplane
->freelist
);
105 QLIST_REMOVE(request
, list
);
107 QLIST_INSERT_HEAD(&dataplane
->inflight
, request
, list
);
108 dataplane
->requests_inflight
++;
114 static void xen_block_finish_request(XenBlockRequest
*request
)
116 XenBlockDataPlane
*dataplane
= request
->dataplane
;
118 QLIST_REMOVE(request
, list
);
119 QLIST_INSERT_HEAD(&dataplane
->finished
, request
, list
);
120 dataplane
->requests_inflight
--;
121 dataplane
->requests_finished
++;
124 static void xen_block_release_request(XenBlockRequest
*request
, bool finish
)
126 XenBlockDataPlane
*dataplane
= request
->dataplane
;
128 QLIST_REMOVE(request
, list
);
129 reset_request(request
);
130 request
->dataplane
= dataplane
;
131 QLIST_INSERT_HEAD(&dataplane
->freelist
, request
, list
);
133 dataplane
->requests_finished
--;
135 dataplane
->requests_inflight
--;
140 * translate request into iovec + start offset
141 * do sanity checks along the way
143 static int xen_block_parse_request(XenBlockRequest
*request
)
145 XenBlockDataPlane
*dataplane
= request
->dataplane
;
149 switch (request
->req
.operation
) {
152 case BLKIF_OP_FLUSH_DISKCACHE
:
153 request
->presync
= 1;
154 if (!request
->req
.nr_segments
) {
160 case BLKIF_OP_DISCARD
:
163 error_report("error: unknown operation (%d)", request
->req
.operation
);
167 if (request
->req
.operation
!= BLKIF_OP_READ
&&
168 blk_is_read_only(dataplane
->blk
)) {
169 error_report("error: write req for ro device");
173 request
->start
= request
->req
.sector_number
* dataplane
->file_blk
;
174 for (i
= 0; i
< request
->req
.nr_segments
; i
++) {
175 if (i
== BLKIF_MAX_SEGMENTS_PER_REQUEST
) {
176 error_report("error: nr_segments too big");
179 if (request
->req
.seg
[i
].first_sect
> request
->req
.seg
[i
].last_sect
) {
180 error_report("error: first > last sector");
183 if (request
->req
.seg
[i
].last_sect
* dataplane
->file_blk
>=
185 error_report("error: page crossing");
189 len
= (request
->req
.seg
[i
].last_sect
-
190 request
->req
.seg
[i
].first_sect
+ 1) * dataplane
->file_blk
;
191 request
->size
+= len
;
193 if (request
->start
+ request
->size
> dataplane
->file_size
) {
194 error_report("error: access beyond end of file");
200 request
->status
= BLKIF_RSP_ERROR
;
204 static int xen_block_copy_request(XenBlockRequest
*request
)
206 XenBlockDataPlane
*dataplane
= request
->dataplane
;
207 XenDevice
*xendev
= dataplane
->xendev
;
208 XenDeviceGrantCopySegment segs
[BLKIF_MAX_SEGMENTS_PER_REQUEST
];
210 int64_t file_blk
= dataplane
->file_blk
;
211 bool to_domain
= (request
->req
.operation
== BLKIF_OP_READ
);
212 void *virt
= request
->buf
;
213 Error
*local_err
= NULL
;
215 if (request
->req
.nr_segments
== 0) {
219 count
= request
->req
.nr_segments
;
221 for (i
= 0; i
< count
; i
++) {
223 segs
[i
].dest
.foreign
.ref
= request
->req
.seg
[i
].gref
;
224 segs
[i
].dest
.foreign
.offset
= request
->req
.seg
[i
].first_sect
*
226 segs
[i
].source
.virt
= virt
;
228 segs
[i
].source
.foreign
.ref
= request
->req
.seg
[i
].gref
;
229 segs
[i
].source
.foreign
.offset
= request
->req
.seg
[i
].first_sect
*
231 segs
[i
].dest
.virt
= virt
;
233 segs
[i
].len
= (request
->req
.seg
[i
].last_sect
-
234 request
->req
.seg
[i
].first_sect
+ 1) * file_blk
;
238 xen_device_copy_grant_refs(xendev
, to_domain
, segs
, count
, &local_err
);
241 error_reportf_err(local_err
, "failed to copy data: ");
243 request
->aio_errors
++;
250 static int xen_block_do_aio(XenBlockRequest
*request
);
252 static void xen_block_complete_aio(void *opaque
, int ret
)
254 XenBlockRequest
*request
= opaque
;
255 XenBlockDataPlane
*dataplane
= request
->dataplane
;
257 aio_context_acquire(dataplane
->ctx
);
260 error_report("%s I/O error",
261 request
->req
.operation
== BLKIF_OP_READ
?
263 request
->aio_errors
++;
266 request
->aio_inflight
--;
267 if (request
->presync
) {
268 request
->presync
= 0;
269 xen_block_do_aio(request
);
272 if (request
->aio_inflight
> 0) {
276 switch (request
->req
.operation
) {
278 /* in case of failure request->aio_errors is increased */
280 xen_block_copy_request(request
);
282 qemu_vfree(request
->buf
);
285 case BLKIF_OP_FLUSH_DISKCACHE
:
286 if (!request
->req
.nr_segments
) {
289 qemu_vfree(request
->buf
);
295 request
->status
= request
->aio_errors
? BLKIF_RSP_ERROR
: BLKIF_RSP_OKAY
;
296 xen_block_finish_request(request
);
298 switch (request
->req
.operation
) {
300 case BLKIF_OP_FLUSH_DISKCACHE
:
301 if (!request
->req
.nr_segments
) {
305 if (request
->status
== BLKIF_RSP_OKAY
) {
306 block_acct_done(blk_get_stats(dataplane
->blk
), &request
->acct
);
308 block_acct_failed(blk_get_stats(dataplane
->blk
), &request
->acct
);
311 case BLKIF_OP_DISCARD
:
315 qemu_bh_schedule(dataplane
->bh
);
318 aio_context_release(dataplane
->ctx
);
321 static bool xen_block_split_discard(XenBlockRequest
*request
,
322 blkif_sector_t sector_number
,
325 XenBlockDataPlane
*dataplane
= request
->dataplane
;
328 uint64_t byte_remaining
, limit
;
329 uint64_t sec_start
= sector_number
;
330 uint64_t sec_count
= nr_sectors
;
332 /* Wrap around, or overflowing byte limit? */
333 if (sec_start
+ sec_count
< sec_count
||
334 sec_start
+ sec_count
> INT64_MAX
/ dataplane
->file_blk
) {
338 limit
= BDRV_REQUEST_MAX_SECTORS
* dataplane
->file_blk
;
339 byte_offset
= sec_start
* dataplane
->file_blk
;
340 byte_remaining
= sec_count
* dataplane
->file_blk
;
343 byte_chunk
= byte_remaining
> limit
? limit
: byte_remaining
;
344 request
->aio_inflight
++;
345 blk_aio_pdiscard(dataplane
->blk
, byte_offset
, byte_chunk
,
346 xen_block_complete_aio
, request
);
347 byte_remaining
-= byte_chunk
;
348 byte_offset
+= byte_chunk
;
349 } while (byte_remaining
> 0);
354 static int xen_block_do_aio(XenBlockRequest
*request
)
356 XenBlockDataPlane
*dataplane
= request
->dataplane
;
358 request
->buf
= qemu_memalign(XC_PAGE_SIZE
, request
->size
);
359 if (request
->req
.nr_segments
&&
360 (request
->req
.operation
== BLKIF_OP_WRITE
||
361 request
->req
.operation
== BLKIF_OP_FLUSH_DISKCACHE
) &&
362 xen_block_copy_request(request
)) {
363 qemu_vfree(request
->buf
);
367 request
->aio_inflight
++;
368 if (request
->presync
) {
369 blk_aio_flush(request
->dataplane
->blk
, xen_block_complete_aio
,
374 switch (request
->req
.operation
) {
376 qemu_iovec_add(&request
->v
, request
->buf
, request
->size
);
377 block_acct_start(blk_get_stats(dataplane
->blk
), &request
->acct
,
378 request
->v
.size
, BLOCK_ACCT_READ
);
379 request
->aio_inflight
++;
380 blk_aio_preadv(dataplane
->blk
, request
->start
, &request
->v
, 0,
381 xen_block_complete_aio
, request
);
384 case BLKIF_OP_FLUSH_DISKCACHE
:
385 if (!request
->req
.nr_segments
) {
389 qemu_iovec_add(&request
->v
, request
->buf
, request
->size
);
390 block_acct_start(blk_get_stats(dataplane
->blk
), &request
->acct
,
392 request
->req
.operation
== BLKIF_OP_WRITE
?
393 BLOCK_ACCT_WRITE
: BLOCK_ACCT_FLUSH
);
394 request
->aio_inflight
++;
395 blk_aio_pwritev(dataplane
->blk
, request
->start
, &request
->v
, 0,
396 xen_block_complete_aio
, request
);
398 case BLKIF_OP_DISCARD
:
400 struct blkif_request_discard
*req
= (void *)&request
->req
;
401 if (!xen_block_split_discard(request
, req
->sector_number
,
408 /* unknown operation (shouldn't happen -- parse catches this) */
412 xen_block_complete_aio(request
, 0);
417 xen_block_finish_request(request
);
418 request
->status
= BLKIF_RSP_ERROR
;
422 static int xen_block_send_response_one(XenBlockRequest
*request
)
424 XenBlockDataPlane
*dataplane
= request
->dataplane
;
426 int have_requests
= 0;
427 blkif_response_t
*resp
;
429 /* Place on the response ring for the relevant domain. */
430 switch (dataplane
->protocol
) {
431 case BLKIF_PROTOCOL_NATIVE
:
432 resp
= (blkif_response_t
*)RING_GET_RESPONSE(
433 &dataplane
->rings
.native
,
434 dataplane
->rings
.native
.rsp_prod_pvt
);
436 case BLKIF_PROTOCOL_X86_32
:
437 resp
= (blkif_response_t
*)RING_GET_RESPONSE(
438 &dataplane
->rings
.x86_32_part
,
439 dataplane
->rings
.x86_32_part
.rsp_prod_pvt
);
441 case BLKIF_PROTOCOL_X86_64
:
442 resp
= (blkif_response_t
*)RING_GET_RESPONSE(
443 &dataplane
->rings
.x86_64_part
,
444 dataplane
->rings
.x86_64_part
.rsp_prod_pvt
);
450 resp
->id
= request
->req
.id
;
451 resp
->operation
= request
->req
.operation
;
452 resp
->status
= request
->status
;
454 dataplane
->rings
.common
.rsp_prod_pvt
++;
456 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&dataplane
->rings
.common
,
458 if (dataplane
->rings
.common
.rsp_prod_pvt
==
459 dataplane
->rings
.common
.req_cons
) {
461 * Tail check for pending requests. Allows frontend to avoid
462 * notifications if requests are already in flight (lower
463 * overheads and promotes batching).
465 RING_FINAL_CHECK_FOR_REQUESTS(&dataplane
->rings
.common
,
467 } else if (RING_HAS_UNCONSUMED_REQUESTS(&dataplane
->rings
.common
)) {
472 dataplane
->more_work
++;
477 /* walk finished list, send outstanding responses, free requests */
478 static void xen_block_send_response_all(XenBlockDataPlane
*dataplane
)
480 XenBlockRequest
*request
;
483 while (!QLIST_EMPTY(&dataplane
->finished
)) {
484 request
= QLIST_FIRST(&dataplane
->finished
);
485 send_notify
+= xen_block_send_response_one(request
);
486 xen_block_release_request(request
, true);
489 Error
*local_err
= NULL
;
491 xen_device_notify_event_channel(dataplane
->xendev
,
492 dataplane
->event_channel
,
495 error_report_err(local_err
);
500 static int xen_block_get_request(XenBlockDataPlane
*dataplane
,
501 XenBlockRequest
*request
, RING_IDX rc
)
503 switch (dataplane
->protocol
) {
504 case BLKIF_PROTOCOL_NATIVE
: {
505 blkif_request_t
*req
=
506 RING_GET_REQUEST(&dataplane
->rings
.native
, rc
);
508 memcpy(&request
->req
, req
, sizeof(request
->req
));
511 case BLKIF_PROTOCOL_X86_32
: {
512 blkif_x86_32_request_t
*req
=
513 RING_GET_REQUEST(&dataplane
->rings
.x86_32_part
, rc
);
515 blkif_get_x86_32_req(&request
->req
, req
);
518 case BLKIF_PROTOCOL_X86_64
: {
519 blkif_x86_64_request_t
*req
=
520 RING_GET_REQUEST(&dataplane
->rings
.x86_64_part
, rc
);
522 blkif_get_x86_64_req(&request
->req
, req
);
526 /* Prevent the compiler from accessing the on-ring fields instead. */
531 static void xen_block_handle_requests(XenBlockDataPlane
*dataplane
)
534 XenBlockRequest
*request
;
536 dataplane
->more_work
= 0;
538 rc
= dataplane
->rings
.common
.req_cons
;
539 rp
= dataplane
->rings
.common
.sring
->req_prod
;
540 xen_rmb(); /* Ensure we see queued requests up to 'rp'. */
542 xen_block_send_response_all(dataplane
);
544 /* pull request from ring */
545 if (RING_REQUEST_CONS_OVERFLOW(&dataplane
->rings
.common
, rc
)) {
548 request
= xen_block_start_request(dataplane
);
549 if (request
== NULL
) {
550 dataplane
->more_work
++;
553 xen_block_get_request(dataplane
, request
, rc
);
554 dataplane
->rings
.common
.req_cons
= ++rc
;
557 if (xen_block_parse_request(request
) != 0) {
558 switch (request
->req
.operation
) {
560 block_acct_invalid(blk_get_stats(dataplane
->blk
),
564 block_acct_invalid(blk_get_stats(dataplane
->blk
),
567 case BLKIF_OP_FLUSH_DISKCACHE
:
568 block_acct_invalid(blk_get_stats(dataplane
->blk
),
574 if (xen_block_send_response_one(request
)) {
575 Error
*local_err
= NULL
;
577 xen_device_notify_event_channel(dataplane
->xendev
,
578 dataplane
->event_channel
,
581 error_report_err(local_err
);
584 xen_block_release_request(request
, false);
588 xen_block_do_aio(request
);
591 if (dataplane
->more_work
&&
592 dataplane
->requests_inflight
< dataplane
->max_requests
) {
593 qemu_bh_schedule(dataplane
->bh
);
597 static void xen_block_dataplane_bh(void *opaque
)
599 XenBlockDataPlane
*dataplane
= opaque
;
601 aio_context_acquire(dataplane
->ctx
);
602 xen_block_handle_requests(dataplane
);
603 aio_context_release(dataplane
->ctx
);
606 static void xen_block_dataplane_event(void *opaque
)
608 XenBlockDataPlane
*dataplane
= opaque
;
610 qemu_bh_schedule(dataplane
->bh
);
613 XenBlockDataPlane
*xen_block_dataplane_create(XenDevice
*xendev
,
617 XenBlockDataPlane
*dataplane
= g_new0(XenBlockDataPlane
, 1);
619 dataplane
->xendev
= xendev
;
620 dataplane
->file_blk
= conf
->logical_block_size
;
621 dataplane
->blk
= conf
->blk
;
622 dataplane
->file_size
= blk_getlength(dataplane
->blk
);
624 QLIST_INIT(&dataplane
->inflight
);
625 QLIST_INIT(&dataplane
->finished
);
626 QLIST_INIT(&dataplane
->freelist
);
629 dataplane
->iothread
= iothread
;
630 object_ref(OBJECT(dataplane
->iothread
));
631 dataplane
->ctx
= iothread_get_aio_context(dataplane
->iothread
);
633 dataplane
->ctx
= qemu_get_aio_context();
635 dataplane
->bh
= aio_bh_new(dataplane
->ctx
, xen_block_dataplane_bh
,
641 void xen_block_dataplane_destroy(XenBlockDataPlane
*dataplane
)
643 XenBlockRequest
*request
;
649 while (!QLIST_EMPTY(&dataplane
->freelist
)) {
650 request
= QLIST_FIRST(&dataplane
->freelist
);
651 QLIST_REMOVE(request
, list
);
652 qemu_iovec_destroy(&request
->v
);
656 qemu_bh_delete(dataplane
->bh
);
657 if (dataplane
->iothread
) {
658 object_unref(OBJECT(dataplane
->iothread
));
664 void xen_block_dataplane_stop(XenBlockDataPlane
*dataplane
)
672 aio_context_acquire(dataplane
->ctx
);
673 blk_set_aio_context(dataplane
->blk
, qemu_get_aio_context());
674 aio_context_release(dataplane
->ctx
);
676 xendev
= dataplane
->xendev
;
678 if (dataplane
->event_channel
) {
679 Error
*local_err
= NULL
;
681 xen_device_unbind_event_channel(xendev
, dataplane
->event_channel
,
683 dataplane
->event_channel
= NULL
;
686 error_report_err(local_err
);
690 if (dataplane
->sring
) {
691 Error
*local_err
= NULL
;
693 xen_device_unmap_grant_refs(xendev
, dataplane
->sring
,
694 dataplane
->nr_ring_ref
, &local_err
);
695 dataplane
->sring
= NULL
;
698 error_report_err(local_err
);
702 g_free(dataplane
->ring_ref
);
703 dataplane
->ring_ref
= NULL
;
706 void xen_block_dataplane_start(XenBlockDataPlane
*dataplane
,
707 const unsigned int ring_ref
[],
708 unsigned int nr_ring_ref
,
709 unsigned int event_channel
,
710 unsigned int protocol
,
713 XenDevice
*xendev
= dataplane
->xendev
;
714 Error
*local_err
= NULL
;
715 unsigned int ring_size
;
718 dataplane
->nr_ring_ref
= nr_ring_ref
;
719 dataplane
->ring_ref
= g_new(unsigned int, nr_ring_ref
);
721 for (i
= 0; i
< nr_ring_ref
; i
++) {
722 dataplane
->ring_ref
[i
] = ring_ref
[i
];
725 dataplane
->protocol
= protocol
;
727 ring_size
= XC_PAGE_SIZE
* dataplane
->nr_ring_ref
;
728 switch (dataplane
->protocol
) {
729 case BLKIF_PROTOCOL_NATIVE
:
731 dataplane
->max_requests
= __CONST_RING_SIZE(blkif
, ring_size
);
734 case BLKIF_PROTOCOL_X86_32
:
736 dataplane
->max_requests
= __CONST_RING_SIZE(blkif_x86_32
, ring_size
);
739 case BLKIF_PROTOCOL_X86_64
:
741 dataplane
->max_requests
= __CONST_RING_SIZE(blkif_x86_64
, ring_size
);
745 error_setg(errp
, "unknown protocol %u", dataplane
->protocol
);
749 xen_device_set_max_grant_refs(xendev
, dataplane
->nr_ring_ref
,
752 error_propagate(errp
, local_err
);
756 dataplane
->sring
= xen_device_map_grant_refs(xendev
,
758 dataplane
->nr_ring_ref
,
759 PROT_READ
| PROT_WRITE
,
762 error_propagate(errp
, local_err
);
766 switch (dataplane
->protocol
) {
767 case BLKIF_PROTOCOL_NATIVE
:
769 blkif_sring_t
*sring_native
= dataplane
->sring
;
771 BACK_RING_INIT(&dataplane
->rings
.native
, sring_native
, ring_size
);
774 case BLKIF_PROTOCOL_X86_32
:
776 blkif_x86_32_sring_t
*sring_x86_32
= dataplane
->sring
;
778 BACK_RING_INIT(&dataplane
->rings
.x86_32_part
, sring_x86_32
,
782 case BLKIF_PROTOCOL_X86_64
:
784 blkif_x86_64_sring_t
*sring_x86_64
= dataplane
->sring
;
786 BACK_RING_INIT(&dataplane
->rings
.x86_64_part
, sring_x86_64
,
792 dataplane
->event_channel
=
793 xen_device_bind_event_channel(xendev
, event_channel
,
794 xen_block_dataplane_event
, dataplane
,
797 error_propagate(errp
, local_err
);
801 aio_context_acquire(dataplane
->ctx
);
802 blk_set_aio_context(dataplane
->blk
, dataplane
->ctx
);
803 aio_context_release(dataplane
->ctx
);
807 xen_block_dataplane_stop(dataplane
);