2 * xen paravirt block device backend
4 * (c) Gerd Hoffmann <kraxel@redhat.com>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; under version 2 of the License.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, see <http://www.gnu.org/licenses/>.
18 * Contributions after 2012-01-13 are licensed under the terms of the
19 * GNU GPL, version 2 or (at your option) any later version.
22 #include "qemu/osdep.h"
23 #include <sys/ioctl.h>
27 #include "hw/xen/xen_backend.h"
28 #include "xen_blkif.h"
29 #include "sysemu/blockdev.h"
30 #include "sysemu/block-backend.h"
31 #include "qapi/error.h"
32 #include "qapi/qmp/qdict.h"
33 #include "qapi/qmp/qstring.h"
35 /* ------------------------------------------------------------- */
37 static int batch_maps
= 0;
39 static int max_requests
= 32;
41 /* ------------------------------------------------------------- */
43 #define BLOCK_SIZE 512
44 #define IOCB_COUNT (BLKIF_MAX_SEGMENTS_PER_REQUEST + 2)
46 struct PersistentGrant
{
48 struct XenBlkDev
*blkdev
;
51 typedef struct PersistentGrant PersistentGrant
;
53 struct PersistentRegion
{
58 typedef struct PersistentRegion PersistentRegion
;
71 uint32_t domids
[BLKIF_MAX_SEGMENTS_PER_REQUEST
];
72 uint32_t refs
[BLKIF_MAX_SEGMENTS_PER_REQUEST
];
74 void *page
[BLKIF_MAX_SEGMENTS_PER_REQUEST
];
82 struct XenBlkDev
*blkdev
;
83 QLIST_ENTRY(ioreq
) list
;
88 struct XenDevice xendev
; /* must be first */
95 const char *fileproto
;
102 blkif_back_rings_t rings
;
107 QLIST_HEAD(inflight_head
, ioreq
) inflight
;
108 QLIST_HEAD(finished_head
, ioreq
) finished
;
109 QLIST_HEAD(freelist_head
, ioreq
) freelist
;
111 int requests_inflight
;
112 int requests_finished
;
114 /* Persistent grants extension */
115 gboolean feature_discard
;
116 gboolean feature_persistent
;
117 GTree
*persistent_gnts
;
118 GSList
*persistent_regions
;
119 unsigned int persistent_gnt_count
;
120 unsigned int max_grants
;
123 gboolean feature_grant_copy
;
125 /* qemu block driver */
131 /* ------------------------------------------------------------- */
133 static void ioreq_reset(struct ioreq
*ioreq
)
135 memset(&ioreq
->req
, 0, sizeof(ioreq
->req
));
141 memset(ioreq
->domids
, 0, sizeof(ioreq
->domids
));
142 memset(ioreq
->refs
, 0, sizeof(ioreq
->refs
));
144 memset(ioreq
->page
, 0, sizeof(ioreq
->page
));
147 ioreq
->aio_inflight
= 0;
148 ioreq
->aio_errors
= 0;
150 ioreq
->blkdev
= NULL
;
151 memset(&ioreq
->list
, 0, sizeof(ioreq
->list
));
152 memset(&ioreq
->acct
, 0, sizeof(ioreq
->acct
));
154 qemu_iovec_reset(&ioreq
->v
);
157 static gint
int_cmp(gconstpointer a
, gconstpointer b
, gpointer user_data
)
159 uint ua
= GPOINTER_TO_UINT(a
);
160 uint ub
= GPOINTER_TO_UINT(b
);
161 return (ua
> ub
) - (ua
< ub
);
164 static void destroy_grant(gpointer pgnt
)
166 PersistentGrant
*grant
= pgnt
;
167 xengnttab_handle
*gnt
= grant
->blkdev
->xendev
.gnttabdev
;
169 if (xengnttab_unmap(gnt
, grant
->page
, 1) != 0) {
170 xen_pv_printf(&grant
->blkdev
->xendev
, 0,
171 "xengnttab_unmap failed: %s\n",
174 grant
->blkdev
->persistent_gnt_count
--;
175 xen_pv_printf(&grant
->blkdev
->xendev
, 3,
176 "unmapped grant %p\n", grant
->page
);
180 static void remove_persistent_region(gpointer data
, gpointer dev
)
182 PersistentRegion
*region
= data
;
183 struct XenBlkDev
*blkdev
= dev
;
184 xengnttab_handle
*gnt
= blkdev
->xendev
.gnttabdev
;
186 if (xengnttab_unmap(gnt
, region
->addr
, region
->num
) != 0) {
187 xen_pv_printf(&blkdev
->xendev
, 0,
188 "xengnttab_unmap region %p failed: %s\n",
189 region
->addr
, strerror(errno
));
191 xen_pv_printf(&blkdev
->xendev
, 3,
192 "unmapped grant region %p with %d pages\n",
193 region
->addr
, region
->num
);
197 static struct ioreq
*ioreq_start(struct XenBlkDev
*blkdev
)
199 struct ioreq
*ioreq
= NULL
;
201 if (QLIST_EMPTY(&blkdev
->freelist
)) {
202 if (blkdev
->requests_total
>= max_requests
) {
205 /* allocate new struct */
206 ioreq
= g_malloc0(sizeof(*ioreq
));
207 ioreq
->blkdev
= blkdev
;
208 blkdev
->requests_total
++;
209 qemu_iovec_init(&ioreq
->v
, BLKIF_MAX_SEGMENTS_PER_REQUEST
);
211 /* get one from freelist */
212 ioreq
= QLIST_FIRST(&blkdev
->freelist
);
213 QLIST_REMOVE(ioreq
, list
);
215 QLIST_INSERT_HEAD(&blkdev
->inflight
, ioreq
, list
);
216 blkdev
->requests_inflight
++;
222 static void ioreq_finish(struct ioreq
*ioreq
)
224 struct XenBlkDev
*blkdev
= ioreq
->blkdev
;
226 QLIST_REMOVE(ioreq
, list
);
227 QLIST_INSERT_HEAD(&blkdev
->finished
, ioreq
, list
);
228 blkdev
->requests_inflight
--;
229 blkdev
->requests_finished
++;
232 static void ioreq_release(struct ioreq
*ioreq
, bool finish
)
234 struct XenBlkDev
*blkdev
= ioreq
->blkdev
;
236 QLIST_REMOVE(ioreq
, list
);
238 ioreq
->blkdev
= blkdev
;
239 QLIST_INSERT_HEAD(&blkdev
->freelist
, ioreq
, list
);
241 blkdev
->requests_finished
--;
243 blkdev
->requests_inflight
--;
248 * translate request into iovec + start offset
249 * do sanity checks along the way
251 static int ioreq_parse(struct ioreq
*ioreq
)
253 struct XenBlkDev
*blkdev
= ioreq
->blkdev
;
258 xen_pv_printf(&blkdev
->xendev
, 3,
259 "op %d, nr %d, handle %d, id %" PRId64
", sector %" PRId64
"\n",
260 ioreq
->req
.operation
, ioreq
->req
.nr_segments
,
261 ioreq
->req
.handle
, ioreq
->req
.id
, ioreq
->req
.sector_number
);
262 switch (ioreq
->req
.operation
) {
264 ioreq
->prot
= PROT_WRITE
; /* to memory */
266 case BLKIF_OP_FLUSH_DISKCACHE
:
268 if (!ioreq
->req
.nr_segments
) {
273 ioreq
->prot
= PROT_READ
; /* from memory */
275 case BLKIF_OP_DISCARD
:
278 xen_pv_printf(&blkdev
->xendev
, 0, "error: unknown operation (%d)\n",
279 ioreq
->req
.operation
);
283 if (ioreq
->req
.operation
!= BLKIF_OP_READ
&& blkdev
->mode
[0] != 'w') {
284 xen_pv_printf(&blkdev
->xendev
, 0, "error: write req for ro device\n");
288 ioreq
->start
= ioreq
->req
.sector_number
* blkdev
->file_blk
;
289 for (i
= 0; i
< ioreq
->req
.nr_segments
; i
++) {
290 if (i
== BLKIF_MAX_SEGMENTS_PER_REQUEST
) {
291 xen_pv_printf(&blkdev
->xendev
, 0, "error: nr_segments too big\n");
294 if (ioreq
->req
.seg
[i
].first_sect
> ioreq
->req
.seg
[i
].last_sect
) {
295 xen_pv_printf(&blkdev
->xendev
, 0, "error: first > last sector\n");
298 if (ioreq
->req
.seg
[i
].last_sect
* BLOCK_SIZE
>= XC_PAGE_SIZE
) {
299 xen_pv_printf(&blkdev
->xendev
, 0, "error: page crossing\n");
303 ioreq
->domids
[i
] = blkdev
->xendev
.dom
;
304 ioreq
->refs
[i
] = ioreq
->req
.seg
[i
].gref
;
306 mem
= ioreq
->req
.seg
[i
].first_sect
* blkdev
->file_blk
;
307 len
= (ioreq
->req
.seg
[i
].last_sect
- ioreq
->req
.seg
[i
].first_sect
+ 1) * blkdev
->file_blk
;
308 qemu_iovec_add(&ioreq
->v
, (void*)mem
, len
);
310 if (ioreq
->start
+ ioreq
->v
.size
> blkdev
->file_size
) {
311 xen_pv_printf(&blkdev
->xendev
, 0, "error: access beyond end of file\n");
317 ioreq
->status
= BLKIF_RSP_ERROR
;
321 static void ioreq_unmap(struct ioreq
*ioreq
)
323 xengnttab_handle
*gnt
= ioreq
->blkdev
->xendev
.gnttabdev
;
326 if (ioreq
->num_unmap
== 0 || ioreq
->mapped
== 0) {
333 if (xengnttab_unmap(gnt
, ioreq
->pages
, ioreq
->num_unmap
) != 0) {
334 xen_pv_printf(&ioreq
->blkdev
->xendev
, 0,
335 "xengnttab_unmap failed: %s\n",
338 ioreq
->blkdev
->cnt_map
-= ioreq
->num_unmap
;
341 for (i
= 0; i
< ioreq
->num_unmap
; i
++) {
342 if (!ioreq
->page
[i
]) {
345 if (xengnttab_unmap(gnt
, ioreq
->page
[i
], 1) != 0) {
346 xen_pv_printf(&ioreq
->blkdev
->xendev
, 0,
347 "xengnttab_unmap failed: %s\n",
350 ioreq
->blkdev
->cnt_map
--;
351 ioreq
->page
[i
] = NULL
;
357 static int ioreq_map(struct ioreq
*ioreq
)
359 xengnttab_handle
*gnt
= ioreq
->blkdev
->xendev
.gnttabdev
;
360 uint32_t domids
[BLKIF_MAX_SEGMENTS_PER_REQUEST
];
361 uint32_t refs
[BLKIF_MAX_SEGMENTS_PER_REQUEST
];
362 void *page
[BLKIF_MAX_SEGMENTS_PER_REQUEST
];
363 int i
, j
, new_maps
= 0;
364 PersistentGrant
*grant
;
365 PersistentRegion
*region
;
366 /* domids and refs variables will contain the information necessary
367 * to map the grants that are needed to fulfill this request.
369 * After mapping the needed grants, the page array will contain the
370 * memory address of each granted page in the order specified in ioreq
371 * (disregarding if it's a persistent grant or not).
374 if (ioreq
->v
.niov
== 0 || ioreq
->mapped
== 1) {
377 if (ioreq
->blkdev
->feature_persistent
) {
378 for (i
= 0; i
< ioreq
->v
.niov
; i
++) {
379 grant
= g_tree_lookup(ioreq
->blkdev
->persistent_gnts
,
380 GUINT_TO_POINTER(ioreq
->refs
[i
]));
383 page
[i
] = grant
->page
;
384 xen_pv_printf(&ioreq
->blkdev
->xendev
, 3,
385 "using persistent-grant %" PRIu32
"\n",
388 /* Add the grant to the list of grants that
391 domids
[new_maps
] = ioreq
->domids
[i
];
392 refs
[new_maps
] = ioreq
->refs
[i
];
397 /* Set the protection to RW, since grants may be reused later
398 * with a different protection than the one needed for this request
400 ioreq
->prot
= PROT_WRITE
| PROT_READ
;
402 /* All grants in the request should be mapped */
403 memcpy(refs
, ioreq
->refs
, sizeof(refs
));
404 memcpy(domids
, ioreq
->domids
, sizeof(domids
));
405 memset(page
, 0, sizeof(page
));
406 new_maps
= ioreq
->v
.niov
;
409 if (batch_maps
&& new_maps
) {
410 ioreq
->pages
= xengnttab_map_grant_refs
411 (gnt
, new_maps
, domids
, refs
, ioreq
->prot
);
412 if (ioreq
->pages
== NULL
) {
413 xen_pv_printf(&ioreq
->blkdev
->xendev
, 0,
414 "can't map %d grant refs (%s, %d maps)\n",
415 new_maps
, strerror(errno
), ioreq
->blkdev
->cnt_map
);
418 for (i
= 0, j
= 0; i
< ioreq
->v
.niov
; i
++) {
419 if (page
[i
] == NULL
) {
420 page
[i
] = ioreq
->pages
+ (j
++) * XC_PAGE_SIZE
;
423 ioreq
->blkdev
->cnt_map
+= new_maps
;
424 } else if (new_maps
) {
425 for (i
= 0; i
< new_maps
; i
++) {
426 ioreq
->page
[i
] = xengnttab_map_grant_ref
427 (gnt
, domids
[i
], refs
[i
], ioreq
->prot
);
428 if (ioreq
->page
[i
] == NULL
) {
429 xen_pv_printf(&ioreq
->blkdev
->xendev
, 0,
430 "can't map grant ref %d (%s, %d maps)\n",
431 refs
[i
], strerror(errno
), ioreq
->blkdev
->cnt_map
);
436 ioreq
->blkdev
->cnt_map
++;
438 for (i
= 0, j
= 0; i
< ioreq
->v
.niov
; i
++) {
439 if (page
[i
] == NULL
) {
440 page
[i
] = ioreq
->page
[j
++];
444 if (ioreq
->blkdev
->feature_persistent
&& new_maps
!= 0 &&
445 (!batch_maps
|| (ioreq
->blkdev
->persistent_gnt_count
+ new_maps
<=
446 ioreq
->blkdev
->max_grants
))) {
448 * If we are using persistent grants and batch mappings only
449 * add the new maps to the list of persistent grants if the whole
450 * area can be persistently mapped.
453 region
= g_malloc0(sizeof(*region
));
454 region
->addr
= ioreq
->pages
;
455 region
->num
= new_maps
;
456 ioreq
->blkdev
->persistent_regions
= g_slist_append(
457 ioreq
->blkdev
->persistent_regions
,
460 while ((ioreq
->blkdev
->persistent_gnt_count
< ioreq
->blkdev
->max_grants
)
462 /* Go through the list of newly mapped grants and add as many
463 * as possible to the list of persistently mapped grants.
465 * Since we start at the end of ioreq->page(s), we only need
466 * to decrease new_maps to prevent this granted pages from
467 * being unmapped in ioreq_unmap.
469 grant
= g_malloc0(sizeof(*grant
));
472 grant
->page
= ioreq
->pages
+ (new_maps
) * XC_PAGE_SIZE
;
474 grant
->page
= ioreq
->page
[new_maps
];
476 grant
->blkdev
= ioreq
->blkdev
;
477 xen_pv_printf(&ioreq
->blkdev
->xendev
, 3,
478 "adding grant %" PRIu32
" page: %p\n",
479 refs
[new_maps
], grant
->page
);
480 g_tree_insert(ioreq
->blkdev
->persistent_gnts
,
481 GUINT_TO_POINTER(refs
[new_maps
]),
483 ioreq
->blkdev
->persistent_gnt_count
++;
485 assert(!batch_maps
|| new_maps
== 0);
487 for (i
= 0; i
< ioreq
->v
.niov
; i
++) {
488 ioreq
->v
.iov
[i
].iov_base
+= (uintptr_t)page
[i
];
491 ioreq
->num_unmap
= new_maps
;
495 #if CONFIG_XEN_CTRL_INTERFACE_VERSION >= 480
497 static void ioreq_free_copy_buffers(struct ioreq
*ioreq
)
501 for (i
= 0; i
< ioreq
->v
.niov
; i
++) {
502 ioreq
->page
[i
] = NULL
;
505 qemu_vfree(ioreq
->pages
);
508 static int ioreq_init_copy_buffers(struct ioreq
*ioreq
)
512 if (ioreq
->v
.niov
== 0) {
516 ioreq
->pages
= qemu_memalign(XC_PAGE_SIZE
, ioreq
->v
.niov
* XC_PAGE_SIZE
);
518 for (i
= 0; i
< ioreq
->v
.niov
; i
++) {
519 ioreq
->page
[i
] = ioreq
->pages
+ i
* XC_PAGE_SIZE
;
520 ioreq
->v
.iov
[i
].iov_base
= ioreq
->page
[i
];
526 static int ioreq_grant_copy(struct ioreq
*ioreq
)
528 xengnttab_handle
*gnt
= ioreq
->blkdev
->xendev
.gnttabdev
;
529 xengnttab_grant_copy_segment_t segs
[BLKIF_MAX_SEGMENTS_PER_REQUEST
];
531 int64_t file_blk
= ioreq
->blkdev
->file_blk
;
533 if (ioreq
->v
.niov
== 0) {
537 count
= ioreq
->v
.niov
;
539 for (i
= 0; i
< count
; i
++) {
540 if (ioreq
->req
.operation
== BLKIF_OP_READ
) {
541 segs
[i
].flags
= GNTCOPY_dest_gref
;
542 segs
[i
].dest
.foreign
.ref
= ioreq
->refs
[i
];
543 segs
[i
].dest
.foreign
.domid
= ioreq
->domids
[i
];
544 segs
[i
].dest
.foreign
.offset
= ioreq
->req
.seg
[i
].first_sect
* file_blk
;
545 segs
[i
].source
.virt
= ioreq
->v
.iov
[i
].iov_base
;
547 segs
[i
].flags
= GNTCOPY_source_gref
;
548 segs
[i
].source
.foreign
.ref
= ioreq
->refs
[i
];
549 segs
[i
].source
.foreign
.domid
= ioreq
->domids
[i
];
550 segs
[i
].source
.foreign
.offset
= ioreq
->req
.seg
[i
].first_sect
* file_blk
;
551 segs
[i
].dest
.virt
= ioreq
->v
.iov
[i
].iov_base
;
553 segs
[i
].len
= (ioreq
->req
.seg
[i
].last_sect
554 - ioreq
->req
.seg
[i
].first_sect
+ 1) * file_blk
;
557 rc
= xengnttab_grant_copy(gnt
, count
, segs
);
560 xen_pv_printf(&ioreq
->blkdev
->xendev
, 0,
561 "failed to copy data %d\n", rc
);
566 for (i
= 0; i
< count
; i
++) {
567 if (segs
[i
].status
!= GNTST_okay
) {
568 xen_pv_printf(&ioreq
->blkdev
->xendev
, 3,
569 "failed to copy data %d for gref %d, domid %d\n",
570 segs
[i
].status
, ioreq
->refs
[i
], ioreq
->domids
[i
]);
579 static void ioreq_free_copy_buffers(struct ioreq
*ioreq
)
584 static int ioreq_init_copy_buffers(struct ioreq
*ioreq
)
589 static int ioreq_grant_copy(struct ioreq
*ioreq
)
595 static int ioreq_runio_qemu_aio(struct ioreq
*ioreq
);
597 static void qemu_aio_complete(void *opaque
, int ret
)
599 struct ioreq
*ioreq
= opaque
;
602 xen_pv_printf(&ioreq
->blkdev
->xendev
, 0, "%s I/O error\n",
603 ioreq
->req
.operation
== BLKIF_OP_READ
? "read" : "write");
607 ioreq
->aio_inflight
--;
608 if (ioreq
->presync
) {
610 ioreq_runio_qemu_aio(ioreq
);
613 if (ioreq
->aio_inflight
> 0) {
617 if (ioreq
->blkdev
->feature_grant_copy
) {
618 switch (ioreq
->req
.operation
) {
620 /* in case of failure ioreq->aio_errors is increased */
622 ioreq_grant_copy(ioreq
);
624 ioreq_free_copy_buffers(ioreq
);
627 case BLKIF_OP_FLUSH_DISKCACHE
:
628 if (!ioreq
->req
.nr_segments
) {
631 ioreq_free_copy_buffers(ioreq
);
638 ioreq
->status
= ioreq
->aio_errors
? BLKIF_RSP_ERROR
: BLKIF_RSP_OKAY
;
639 if (!ioreq
->blkdev
->feature_grant_copy
) {
643 switch (ioreq
->req
.operation
) {
645 case BLKIF_OP_FLUSH_DISKCACHE
:
646 if (!ioreq
->req
.nr_segments
) {
650 if (ioreq
->status
== BLKIF_RSP_OKAY
) {
651 block_acct_done(blk_get_stats(ioreq
->blkdev
->blk
), &ioreq
->acct
);
653 block_acct_failed(blk_get_stats(ioreq
->blkdev
->blk
), &ioreq
->acct
);
656 case BLKIF_OP_DISCARD
:
660 qemu_bh_schedule(ioreq
->blkdev
->bh
);
663 static int ioreq_runio_qemu_aio(struct ioreq
*ioreq
)
665 struct XenBlkDev
*blkdev
= ioreq
->blkdev
;
667 if (ioreq
->blkdev
->feature_grant_copy
) {
668 ioreq_init_copy_buffers(ioreq
);
669 if (ioreq
->req
.nr_segments
&& (ioreq
->req
.operation
== BLKIF_OP_WRITE
||
670 ioreq
->req
.operation
== BLKIF_OP_FLUSH_DISKCACHE
) &&
671 ioreq_grant_copy(ioreq
)) {
672 ioreq_free_copy_buffers(ioreq
);
676 if (ioreq
->req
.nr_segments
&& ioreq_map(ioreq
)) {
681 ioreq
->aio_inflight
++;
682 if (ioreq
->presync
) {
683 blk_aio_flush(ioreq
->blkdev
->blk
, qemu_aio_complete
, ioreq
);
687 switch (ioreq
->req
.operation
) {
689 block_acct_start(blk_get_stats(blkdev
->blk
), &ioreq
->acct
,
690 ioreq
->v
.size
, BLOCK_ACCT_READ
);
691 ioreq
->aio_inflight
++;
692 blk_aio_preadv(blkdev
->blk
, ioreq
->start
, &ioreq
->v
, 0,
693 qemu_aio_complete
, ioreq
);
696 case BLKIF_OP_FLUSH_DISKCACHE
:
697 if (!ioreq
->req
.nr_segments
) {
701 block_acct_start(blk_get_stats(blkdev
->blk
), &ioreq
->acct
,
703 ioreq
->req
.operation
== BLKIF_OP_WRITE
?
704 BLOCK_ACCT_WRITE
: BLOCK_ACCT_FLUSH
);
705 ioreq
->aio_inflight
++;
706 blk_aio_pwritev(blkdev
->blk
, ioreq
->start
, &ioreq
->v
, 0,
707 qemu_aio_complete
, ioreq
);
709 case BLKIF_OP_DISCARD
:
711 struct blkif_request_discard
*discard_req
= (void *)&ioreq
->req
;
712 ioreq
->aio_inflight
++;
713 blk_aio_pdiscard(blkdev
->blk
,
714 discard_req
->sector_number
<< BDRV_SECTOR_BITS
,
715 discard_req
->nr_sectors
<< BDRV_SECTOR_BITS
,
716 qemu_aio_complete
, ioreq
);
720 /* unknown operation (shouldn't happen -- parse catches this) */
721 if (!ioreq
->blkdev
->feature_grant_copy
) {
727 qemu_aio_complete(ioreq
, 0);
733 ioreq
->status
= BLKIF_RSP_ERROR
;
737 static int blk_send_response_one(struct ioreq
*ioreq
)
739 struct XenBlkDev
*blkdev
= ioreq
->blkdev
;
741 int have_requests
= 0;
742 blkif_response_t resp
;
745 resp
.id
= ioreq
->req
.id
;
746 resp
.operation
= ioreq
->req
.operation
;
747 resp
.status
= ioreq
->status
;
749 /* Place on the response ring for the relevant domain. */
750 switch (blkdev
->protocol
) {
751 case BLKIF_PROTOCOL_NATIVE
:
752 dst
= RING_GET_RESPONSE(&blkdev
->rings
.native
, blkdev
->rings
.native
.rsp_prod_pvt
);
754 case BLKIF_PROTOCOL_X86_32
:
755 dst
= RING_GET_RESPONSE(&blkdev
->rings
.x86_32_part
,
756 blkdev
->rings
.x86_32_part
.rsp_prod_pvt
);
758 case BLKIF_PROTOCOL_X86_64
:
759 dst
= RING_GET_RESPONSE(&blkdev
->rings
.x86_64_part
,
760 blkdev
->rings
.x86_64_part
.rsp_prod_pvt
);
766 memcpy(dst
, &resp
, sizeof(resp
));
767 blkdev
->rings
.common
.rsp_prod_pvt
++;
769 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&blkdev
->rings
.common
, send_notify
);
770 if (blkdev
->rings
.common
.rsp_prod_pvt
== blkdev
->rings
.common
.req_cons
) {
772 * Tail check for pending requests. Allows frontend to avoid
773 * notifications if requests are already in flight (lower
774 * overheads and promotes batching).
776 RING_FINAL_CHECK_FOR_REQUESTS(&blkdev
->rings
.common
, have_requests
);
777 } else if (RING_HAS_UNCONSUMED_REQUESTS(&blkdev
->rings
.common
)) {
787 /* walk finished list, send outstanding responses, free requests */
788 static void blk_send_response_all(struct XenBlkDev
*blkdev
)
793 while (!QLIST_EMPTY(&blkdev
->finished
)) {
794 ioreq
= QLIST_FIRST(&blkdev
->finished
);
795 send_notify
+= blk_send_response_one(ioreq
);
796 ioreq_release(ioreq
, true);
799 xen_pv_send_notify(&blkdev
->xendev
);
803 static int blk_get_request(struct XenBlkDev
*blkdev
, struct ioreq
*ioreq
, RING_IDX rc
)
805 switch (blkdev
->protocol
) {
806 case BLKIF_PROTOCOL_NATIVE
:
807 memcpy(&ioreq
->req
, RING_GET_REQUEST(&blkdev
->rings
.native
, rc
),
810 case BLKIF_PROTOCOL_X86_32
:
811 blkif_get_x86_32_req(&ioreq
->req
,
812 RING_GET_REQUEST(&blkdev
->rings
.x86_32_part
, rc
));
814 case BLKIF_PROTOCOL_X86_64
:
815 blkif_get_x86_64_req(&ioreq
->req
,
816 RING_GET_REQUEST(&blkdev
->rings
.x86_64_part
, rc
));
819 /* Prevent the compiler from accessing the on-ring fields instead. */
824 static void blk_handle_requests(struct XenBlkDev
*blkdev
)
829 blkdev
->more_work
= 0;
831 rc
= blkdev
->rings
.common
.req_cons
;
832 rp
= blkdev
->rings
.common
.sring
->req_prod
;
833 xen_rmb(); /* Ensure we see queued requests up to 'rp'. */
835 blk_send_response_all(blkdev
);
837 /* pull request from ring */
838 if (RING_REQUEST_CONS_OVERFLOW(&blkdev
->rings
.common
, rc
)) {
841 ioreq
= ioreq_start(blkdev
);
846 blk_get_request(blkdev
, ioreq
, rc
);
847 blkdev
->rings
.common
.req_cons
= ++rc
;
850 if (ioreq_parse(ioreq
) != 0) {
852 switch (ioreq
->req
.operation
) {
854 block_acct_invalid(blk_get_stats(blkdev
->blk
),
858 block_acct_invalid(blk_get_stats(blkdev
->blk
),
861 case BLKIF_OP_FLUSH_DISKCACHE
:
862 block_acct_invalid(blk_get_stats(blkdev
->blk
),
868 if (blk_send_response_one(ioreq
)) {
869 xen_pv_send_notify(&blkdev
->xendev
);
871 ioreq_release(ioreq
, false);
875 ioreq_runio_qemu_aio(ioreq
);
878 if (blkdev
->more_work
&& blkdev
->requests_inflight
< max_requests
) {
879 qemu_bh_schedule(blkdev
->bh
);
883 /* ------------------------------------------------------------- */
885 static void blk_bh(void *opaque
)
887 struct XenBlkDev
*blkdev
= opaque
;
888 blk_handle_requests(blkdev
);
892 * We need to account for the grant allocations requiring contiguous
893 * chunks; the worst case number would be
894 * max_req * max_seg + (max_req - 1) * (max_seg - 1) + 1,
895 * but in order to keep things simple just use
896 * 2 * max_req * max_seg.
898 #define MAX_GRANTS(max_req, max_seg) (2 * (max_req) * (max_seg))
900 static void blk_alloc(struct XenDevice
*xendev
)
902 struct XenBlkDev
*blkdev
= container_of(xendev
, struct XenBlkDev
, xendev
);
904 QLIST_INIT(&blkdev
->inflight
);
905 QLIST_INIT(&blkdev
->finished
);
906 QLIST_INIT(&blkdev
->freelist
);
907 blkdev
->bh
= qemu_bh_new(blk_bh
, blkdev
);
908 if (xen_mode
!= XEN_EMULATE
) {
911 if (xengnttab_set_max_grants(xendev
->gnttabdev
,
912 MAX_GRANTS(max_requests
, BLKIF_MAX_SEGMENTS_PER_REQUEST
)) < 0) {
913 xen_pv_printf(xendev
, 0, "xengnttab_set_max_grants failed: %s\n",
918 static void blk_parse_discard(struct XenBlkDev
*blkdev
)
922 blkdev
->feature_discard
= true;
924 if (xenstore_read_be_int(&blkdev
->xendev
, "discard-enable", &enable
) == 0) {
925 blkdev
->feature_discard
= !!enable
;
928 if (blkdev
->feature_discard
) {
929 xenstore_write_be_int(&blkdev
->xendev
, "feature-discard", 1);
933 static int blk_init(struct XenDevice
*xendev
)
935 struct XenBlkDev
*blkdev
= container_of(xendev
, struct XenBlkDev
, xendev
);
937 char *directiosafe
= NULL
;
939 /* read xenstore entries */
940 if (blkdev
->params
== NULL
) {
942 blkdev
->params
= xenstore_read_be_str(&blkdev
->xendev
, "params");
943 if (blkdev
->params
!= NULL
) {
944 h
= strchr(blkdev
->params
, ':');
947 blkdev
->fileproto
= blkdev
->params
;
948 blkdev
->filename
= h
+1;
951 blkdev
->fileproto
= "<unset>";
952 blkdev
->filename
= blkdev
->params
;
955 if (!strcmp("aio", blkdev
->fileproto
)) {
956 blkdev
->fileproto
= "raw";
958 if (!strcmp("vhd", blkdev
->fileproto
)) {
959 blkdev
->fileproto
= "vpc";
961 if (blkdev
->mode
== NULL
) {
962 blkdev
->mode
= xenstore_read_be_str(&blkdev
->xendev
, "mode");
964 if (blkdev
->type
== NULL
) {
965 blkdev
->type
= xenstore_read_be_str(&blkdev
->xendev
, "type");
967 if (blkdev
->dev
== NULL
) {
968 blkdev
->dev
= xenstore_read_be_str(&blkdev
->xendev
, "dev");
970 if (blkdev
->devtype
== NULL
) {
971 blkdev
->devtype
= xenstore_read_be_str(&blkdev
->xendev
, "device-type");
973 directiosafe
= xenstore_read_be_str(&blkdev
->xendev
, "direct-io-safe");
974 blkdev
->directiosafe
= (directiosafe
&& atoi(directiosafe
));
976 /* do we have all we need? */
977 if (blkdev
->params
== NULL
||
978 blkdev
->mode
== NULL
||
979 blkdev
->type
== NULL
||
980 blkdev
->dev
== NULL
) {
985 if (strcmp(blkdev
->mode
, "w")) {
986 info
|= VDISK_READONLY
;
990 if (blkdev
->devtype
&& !strcmp(blkdev
->devtype
, "cdrom")) {
994 blkdev
->file_blk
= BLOCK_SIZE
;
997 * blk_connect supplies sector-size and sectors
999 xenstore_write_be_int(&blkdev
->xendev
, "feature-flush-cache", 1);
1000 xenstore_write_be_int(&blkdev
->xendev
, "feature-persistent", 1);
1001 xenstore_write_be_int(&blkdev
->xendev
, "info", info
);
1003 blk_parse_discard(blkdev
);
1005 g_free(directiosafe
);
1009 g_free(blkdev
->params
);
1010 blkdev
->params
= NULL
;
1011 g_free(blkdev
->mode
);
1012 blkdev
->mode
= NULL
;
1013 g_free(blkdev
->type
);
1014 blkdev
->type
= NULL
;
1015 g_free(blkdev
->dev
);
1017 g_free(blkdev
->devtype
);
1018 blkdev
->devtype
= NULL
;
1019 g_free(directiosafe
);
1020 blkdev
->directiosafe
= false;
1024 static int blk_connect(struct XenDevice
*xendev
)
1026 struct XenBlkDev
*blkdev
= container_of(xendev
, struct XenBlkDev
, xendev
);
1027 int pers
, index
, qflags
;
1028 bool readonly
= true;
1029 bool writethrough
= true;
1032 if (blkdev
->directiosafe
) {
1033 qflags
= BDRV_O_NOCACHE
| BDRV_O_NATIVE_AIO
;
1036 writethrough
= false;
1038 if (strcmp(blkdev
->mode
, "w") == 0) {
1039 qflags
|= BDRV_O_RDWR
;
1042 if (blkdev
->feature_discard
) {
1043 qflags
|= BDRV_O_UNMAP
;
1046 /* init qemu block driver */
1047 index
= (blkdev
->xendev
.dev
- 202 * 256) / 16;
1048 blkdev
->dinfo
= drive_get(IF_XEN
, 0, index
);
1049 if (!blkdev
->dinfo
) {
1050 Error
*local_err
= NULL
;
1051 QDict
*options
= NULL
;
1053 if (strcmp(blkdev
->fileproto
, "<unset>")) {
1054 options
= qdict_new();
1055 qdict_put(options
, "driver", qstring_from_str(blkdev
->fileproto
));
1058 /* setup via xenbus -> create new block driver instance */
1059 xen_pv_printf(&blkdev
->xendev
, 2, "create new bdrv (xenbus setup)\n");
1060 blkdev
->blk
= blk_new_open(blkdev
->filename
, NULL
, options
,
1061 qflags
, &local_err
);
1063 xen_pv_printf(&blkdev
->xendev
, 0, "error: %s\n",
1064 error_get_pretty(local_err
));
1065 error_free(local_err
);
1068 blk_set_enable_write_cache(blkdev
->blk
, !writethrough
);
1070 /* setup via qemu cmdline -> already setup for us */
1071 xen_pv_printf(&blkdev
->xendev
, 2,
1072 "get configured bdrv (cmdline setup)\n");
1073 blkdev
->blk
= blk_by_legacy_dinfo(blkdev
->dinfo
);
1074 if (blk_is_read_only(blkdev
->blk
) && !readonly
) {
1075 xen_pv_printf(&blkdev
->xendev
, 0, "Unexpected read-only drive");
1079 /* blkdev->blk is not create by us, we get a reference
1080 * so we can blk_unref() unconditionally */
1081 blk_ref(blkdev
->blk
);
1083 blk_attach_dev_legacy(blkdev
->blk
, blkdev
);
1084 blkdev
->file_size
= blk_getlength(blkdev
->blk
);
1085 if (blkdev
->file_size
< 0) {
1086 BlockDriverState
*bs
= blk_bs(blkdev
->blk
);
1087 const char *drv_name
= bs
? bdrv_get_format_name(bs
) : NULL
;
1088 xen_pv_printf(&blkdev
->xendev
, 1, "blk_getlength: %d (%s) | drv %s\n",
1089 (int)blkdev
->file_size
, strerror(-blkdev
->file_size
),
1091 blkdev
->file_size
= 0;
1094 xen_pv_printf(xendev
, 1, "type \"%s\", fileproto \"%s\", filename \"%s\","
1095 " size %" PRId64
" (%" PRId64
" MB)\n",
1096 blkdev
->type
, blkdev
->fileproto
, blkdev
->filename
,
1097 blkdev
->file_size
, blkdev
->file_size
>> 20);
1099 /* Fill in number of sector size and number of sectors */
1100 xenstore_write_be_int(&blkdev
->xendev
, "sector-size", blkdev
->file_blk
);
1101 xenstore_write_be_int64(&blkdev
->xendev
, "sectors",
1102 blkdev
->file_size
/ blkdev
->file_blk
);
1104 if (xenstore_read_fe_int(&blkdev
->xendev
, "ring-ref", &blkdev
->ring_ref
) == -1) {
1107 if (xenstore_read_fe_int(&blkdev
->xendev
, "event-channel",
1108 &blkdev
->xendev
.remote_port
) == -1) {
1111 if (xenstore_read_fe_int(&blkdev
->xendev
, "feature-persistent", &pers
)) {
1112 blkdev
->feature_persistent
= FALSE
;
1114 blkdev
->feature_persistent
= !!pers
;
1117 if (!blkdev
->xendev
.protocol
) {
1118 blkdev
->protocol
= BLKIF_PROTOCOL_NATIVE
;
1119 } else if (strcmp(blkdev
->xendev
.protocol
, XEN_IO_PROTO_ABI_NATIVE
) == 0) {
1120 blkdev
->protocol
= BLKIF_PROTOCOL_NATIVE
;
1121 } else if (strcmp(blkdev
->xendev
.protocol
, XEN_IO_PROTO_ABI_X86_32
) == 0) {
1122 blkdev
->protocol
= BLKIF_PROTOCOL_X86_32
;
1123 } else if (strcmp(blkdev
->xendev
.protocol
, XEN_IO_PROTO_ABI_X86_64
) == 0) {
1124 blkdev
->protocol
= BLKIF_PROTOCOL_X86_64
;
1126 blkdev
->protocol
= BLKIF_PROTOCOL_NATIVE
;
1129 blkdev
->sring
= xengnttab_map_grant_ref(blkdev
->xendev
.gnttabdev
,
1132 PROT_READ
| PROT_WRITE
);
1133 if (!blkdev
->sring
) {
1138 switch (blkdev
->protocol
) {
1139 case BLKIF_PROTOCOL_NATIVE
:
1141 blkif_sring_t
*sring_native
= blkdev
->sring
;
1142 BACK_RING_INIT(&blkdev
->rings
.native
, sring_native
, XC_PAGE_SIZE
);
1145 case BLKIF_PROTOCOL_X86_32
:
1147 blkif_x86_32_sring_t
*sring_x86_32
= blkdev
->sring
;
1149 BACK_RING_INIT(&blkdev
->rings
.x86_32_part
, sring_x86_32
, XC_PAGE_SIZE
);
1152 case BLKIF_PROTOCOL_X86_64
:
1154 blkif_x86_64_sring_t
*sring_x86_64
= blkdev
->sring
;
1156 BACK_RING_INIT(&blkdev
->rings
.x86_64_part
, sring_x86_64
, XC_PAGE_SIZE
);
1161 if (blkdev
->feature_persistent
) {
1162 /* Init persistent grants */
1163 blkdev
->max_grants
= max_requests
* BLKIF_MAX_SEGMENTS_PER_REQUEST
;
1164 blkdev
->persistent_gnts
= g_tree_new_full((GCompareDataFunc
)int_cmp
,
1167 (GDestroyNotify
)g_free
:
1168 (GDestroyNotify
)destroy_grant
);
1169 blkdev
->persistent_regions
= NULL
;
1170 blkdev
->persistent_gnt_count
= 0;
1173 xen_be_bind_evtchn(&blkdev
->xendev
);
1175 blkdev
->feature_grant_copy
=
1176 (xengnttab_grant_copy(blkdev
->xendev
.gnttabdev
, 0, NULL
) == 0);
1178 xen_pv_printf(&blkdev
->xendev
, 3, "grant copy operation %s\n",
1179 blkdev
->feature_grant_copy
? "enabled" : "disabled");
1181 xen_pv_printf(&blkdev
->xendev
, 1, "ok: proto %s, ring-ref %d, "
1182 "remote port %d, local port %d\n",
1183 blkdev
->xendev
.protocol
, blkdev
->ring_ref
,
1184 blkdev
->xendev
.remote_port
, blkdev
->xendev
.local_port
);
1188 static void blk_disconnect(struct XenDevice
*xendev
)
1190 struct XenBlkDev
*blkdev
= container_of(xendev
, struct XenBlkDev
, xendev
);
1193 blk_detach_dev(blkdev
->blk
, blkdev
);
1194 blk_unref(blkdev
->blk
);
1197 xen_pv_unbind_evtchn(&blkdev
->xendev
);
1199 if (blkdev
->sring
) {
1200 xengnttab_unmap(blkdev
->xendev
.gnttabdev
, blkdev
->sring
, 1);
1202 blkdev
->sring
= NULL
;
1206 * Unmap persistent grants before switching to the closed state
1207 * so the frontend can free them.
1209 * In the !batch_maps case g_tree_destroy will take care of unmapping
1210 * the grant, but in the batch_maps case we need to iterate over every
1211 * region in persistent_regions and unmap it.
1213 if (blkdev
->feature_persistent
) {
1214 g_tree_destroy(blkdev
->persistent_gnts
);
1215 assert(batch_maps
|| blkdev
->persistent_gnt_count
== 0);
1217 blkdev
->persistent_gnt_count
= 0;
1218 g_slist_foreach(blkdev
->persistent_regions
,
1219 (GFunc
)remove_persistent_region
, blkdev
);
1220 g_slist_free(blkdev
->persistent_regions
);
1222 blkdev
->feature_persistent
= false;
1226 static int blk_free(struct XenDevice
*xendev
)
1228 struct XenBlkDev
*blkdev
= container_of(xendev
, struct XenBlkDev
, xendev
);
1229 struct ioreq
*ioreq
;
1231 if (blkdev
->blk
|| blkdev
->sring
) {
1232 blk_disconnect(xendev
);
1235 while (!QLIST_EMPTY(&blkdev
->freelist
)) {
1236 ioreq
= QLIST_FIRST(&blkdev
->freelist
);
1237 QLIST_REMOVE(ioreq
, list
);
1238 qemu_iovec_destroy(&ioreq
->v
);
1242 g_free(blkdev
->params
);
1243 g_free(blkdev
->mode
);
1244 g_free(blkdev
->type
);
1245 g_free(blkdev
->dev
);
1246 g_free(blkdev
->devtype
);
1247 qemu_bh_delete(blkdev
->bh
);
1251 static void blk_event(struct XenDevice
*xendev
)
1253 struct XenBlkDev
*blkdev
= container_of(xendev
, struct XenBlkDev
, xendev
);
1255 qemu_bh_schedule(blkdev
->bh
);
1258 struct XenDevOps xen_blkdev_ops
= {
1259 .size
= sizeof(struct XenBlkDev
),
1260 .flags
= DEVOPS_FLAG_NEED_GNTDEV
,
1263 .initialise
= blk_connect
,
1264 .disconnect
= blk_disconnect
,