hw/block/dataplane/xen-block.c

   1 /*
   2  * Copyright (c) 2018  Citrix Systems Inc.
   3  * (c) Gerd Hoffmann <kraxel@redhat.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; under version 2 of the License.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along
  15  * with this program; if not, see <http://www.gnu.org/licenses/>.
  16  *
  17  * Contributions after 2012-01-13 are licensed under the terms of the
  18  * GNU GPL, version 2 or (at your option) any later version.
  19  */
  20
  21 #include "qemu/osdep.h"
  22 #include "qemu/error-report.h"
  23 #include "qemu/main-loop.h"
  24 #include "qemu/memalign.h"
  25 #include "qapi/error.h"
  26 #include "hw/xen/xen.h"
  27 #include "hw/block/xen_blkif.h"
  28 #include "hw/xen/interface/io/ring.h"
  29 #include "sysemu/block-backend.h"
  30 #include "sysemu/iothread.h"
  31 #include "xen-block.h"
  32
  33 typedef struct XenBlockRequest {
  34     blkif_request_t req;
  35     int16_t status;
  36     off_t start;
  37     QEMUIOVector v;
  38     void *buf;
  39     size_t size;
  40     int presync;
  41     int aio_inflight;
  42     int aio_errors;
  43     XenBlockDataPlane *dataplane;
  44     QLIST_ENTRY(XenBlockRequest) list;
  45     BlockAcctCookie acct;
  46 } XenBlockRequest;
  47
  48 struct XenBlockDataPlane {
  49     XenDevice *xendev;
  50     XenEventChannel *event_channel;
  51     unsigned int *ring_ref;
  52     unsigned int nr_ring_ref;
  53     void *sring;
  54     int protocol;
  55     blkif_back_rings_t rings;
  56     int more_work;
  57     QLIST_HEAD(inflight_head, XenBlockRequest) inflight;
  58     QLIST_HEAD(freelist_head, XenBlockRequest) freelist;
  59     int requests_total;
  60     int requests_inflight;
  61     unsigned int max_requests;
  62     BlockBackend *blk;
  63     unsigned int sector_size;
  64     QEMUBH *bh;
  65     IOThread *iothread;
  66     AioContext *ctx;
  67 };
  68
  69 static int xen_block_send_response(XenBlockRequest *request);
  70
  71 static void reset_request(XenBlockRequest *request)
  72 {
  73     memset(&request->req, 0, sizeof(request->req));
  74     request->status = 0;
  75     request->start = 0;
  76     request->size = 0;
  77     request->presync = 0;
  78
  79     request->aio_inflight = 0;
  80     request->aio_errors = 0;
  81
  82     request->dataplane = NULL;
  83     memset(&request->list, 0, sizeof(request->list));
  84     memset(&request->acct, 0, sizeof(request->acct));
  85
  86     qemu_iovec_reset(&request->v);
  87 }
  88
  89 static XenBlockRequest *xen_block_start_request(XenBlockDataPlane *dataplane)
  90 {
  91     XenBlockRequest *request = NULL;
  92
  93     if (QLIST_EMPTY(&dataplane->freelist)) {
  94         if (dataplane->requests_total >= dataplane->max_requests) {
  95             goto out;
  96         }
  97         /* allocate new struct */
  98         request = g_malloc0(sizeof(*request));
  99         request->dataplane = dataplane;
 100         /*
 101          * We cannot need more pages per requests than this, and since we
 102          * re-use requests, allocate the memory once here. It will be freed
 103          * xen_block_dataplane_destroy() when the request list is freed.
 104          */
 105         request->buf = qemu_memalign(XEN_PAGE_SIZE,
 106                                      BLKIF_MAX_SEGMENTS_PER_REQUEST *
 107                                      XEN_PAGE_SIZE);
 108         dataplane->requests_total++;
 109         qemu_iovec_init(&request->v, 1);
 110     } else {
 111         /* get one from freelist */
 112         request = QLIST_FIRST(&dataplane->freelist);
 113         QLIST_REMOVE(request, list);
 114     }
 115     QLIST_INSERT_HEAD(&dataplane->inflight, request, list);
 116     dataplane->requests_inflight++;
 117
 118 out:
 119     return request;
 120 }
 121
 122 static void xen_block_complete_request(XenBlockRequest *request)
 123 {
 124     XenBlockDataPlane *dataplane = request->dataplane;
 125
 126     if (xen_block_send_response(request)) {
 127         Error *local_err = NULL;
 128
 129         xen_device_notify_event_channel(dataplane->xendev,
 130                                         dataplane->event_channel,
 131                                         &local_err);
 132         if (local_err) {
 133             error_report_err(local_err);
 134         }
 135     }
 136
 137     QLIST_REMOVE(request, list);
 138     dataplane->requests_inflight--;
 139     reset_request(request);
 140     request->dataplane = dataplane;
 141     QLIST_INSERT_HEAD(&dataplane->freelist, request, list);
 142 }
 143
 144 /*
 145  * translate request into iovec + start offset
 146  * do sanity checks along the way
 147  */
 148 static int xen_block_parse_request(XenBlockRequest *request)
 149 {
 150     XenBlockDataPlane *dataplane = request->dataplane;
 151     size_t len;
 152     int i;
 153
 154     switch (request->req.operation) {
 155     case BLKIF_OP_READ:
 156         break;
 157     case BLKIF_OP_FLUSH_DISKCACHE:
 158         request->presync = 1;
 159         if (!request->req.nr_segments) {
 160             return 0;
 161         }
 162         /* fall through */
 163     case BLKIF_OP_WRITE:
 164         break;
 165     case BLKIF_OP_DISCARD:
 166         return 0;
 167     default:
 168         error_report("error: unknown operation (%d)", request->req.operation);
 169         goto err;
 170     };
 171
 172     if (request->req.operation != BLKIF_OP_READ &&
 173         !blk_is_writable(dataplane->blk)) {
 174         error_report("error: write req for ro device");
 175         goto err;
 176     }
 177
 178     request->start = request->req.sector_number * dataplane->sector_size;
 179     for (i = 0; i < request->req.nr_segments; i++) {
 180         if (i == BLKIF_MAX_SEGMENTS_PER_REQUEST) {
 181             error_report("error: nr_segments too big");
 182             goto err;
 183         }
 184         if (request->req.seg[i].first_sect > request->req.seg[i].last_sect) {
 185             error_report("error: first > last sector");
 186             goto err;
 187         }
 188         if (request->req.seg[i].last_sect * dataplane->sector_size >=
 189             XEN_PAGE_SIZE) {
 190             error_report("error: page crossing");
 191             goto err;
 192         }
 193
 194         len = (request->req.seg[i].last_sect -
 195                request->req.seg[i].first_sect + 1) * dataplane->sector_size;
 196         request->size += len;
 197     }
 198     if (request->start + request->size > blk_getlength(dataplane->blk)) {
 199         error_report("error: access beyond end of file");
 200         goto err;
 201     }
 202     return 0;
 203
 204 err:
 205     request->status = BLKIF_RSP_ERROR;
 206     return -1;
 207 }
 208
 209 static int xen_block_copy_request(XenBlockRequest *request)
 210 {
 211     XenBlockDataPlane *dataplane = request->dataplane;
 212     XenDevice *xendev = dataplane->xendev;
 213     XenDeviceGrantCopySegment segs[BLKIF_MAX_SEGMENTS_PER_REQUEST];
 214     int i, count;
 215     bool to_domain = (request->req.operation == BLKIF_OP_READ);
 216     void *virt = request->buf;
 217     Error *local_err = NULL;
 218
 219     if (request->req.nr_segments == 0) {
 220         return 0;
 221     }
 222
 223     count = request->req.nr_segments;
 224
 225     for (i = 0; i < count; i++) {
 226         if (to_domain) {
 227             segs[i].dest.foreign.ref = request->req.seg[i].gref;
 228             segs[i].dest.foreign.offset = request->req.seg[i].first_sect *
 229                 dataplane->sector_size;
 230             segs[i].source.virt = virt;
 231         } else {
 232             segs[i].source.foreign.ref = request->req.seg[i].gref;
 233             segs[i].source.foreign.offset = request->req.seg[i].first_sect *
 234                 dataplane->sector_size;
 235             segs[i].dest.virt = virt;
 236         }
 237         segs[i].len = (request->req.seg[i].last_sect -
 238                        request->req.seg[i].first_sect + 1) *
 239                       dataplane->sector_size;
 240         virt += segs[i].len;
 241     }
 242
 243     xen_device_copy_grant_refs(xendev, to_domain, segs, count, &local_err);
 244
 245     if (local_err) {
 246         error_reportf_err(local_err, "failed to copy data: ");
 247
 248         request->aio_errors++;
 249         return -1;
 250     }
 251
 252     return 0;
 253 }
 254
 255 static int xen_block_do_aio(XenBlockRequest *request);
 256
 257 static void xen_block_complete_aio(void *opaque, int ret)
 258 {
 259     XenBlockRequest *request = opaque;
 260     XenBlockDataPlane *dataplane = request->dataplane;
 261
 262     aio_context_acquire(dataplane->ctx);
 263
 264     if (ret != 0) {
 265         error_report("%s I/O error",
 266                      request->req.operation == BLKIF_OP_READ ?
 267                      "read" : "write");
 268         request->aio_errors++;
 269     }
 270
 271     request->aio_inflight--;
 272     if (request->presync) {
 273         request->presync = 0;
 274         xen_block_do_aio(request);
 275         goto done;
 276     }
 277     if (request->aio_inflight > 0) {
 278         goto done;
 279     }
 280
 281     switch (request->req.operation) {
 282     case BLKIF_OP_READ:
 283         /* in case of failure request->aio_errors is increased */
 284         if (ret == 0) {
 285             xen_block_copy_request(request);
 286         }
 287         break;
 288     case BLKIF_OP_WRITE:
 289     case BLKIF_OP_FLUSH_DISKCACHE:
 290     default:
 291         break;
 292     }
 293
 294     request->status = request->aio_errors ? BLKIF_RSP_ERROR : BLKIF_RSP_OKAY;
 295
 296     switch (request->req.operation) {
 297     case BLKIF_OP_WRITE:
 298     case BLKIF_OP_FLUSH_DISKCACHE:
 299         if (!request->req.nr_segments) {
 300             break;
 301         }
 302         /* fall through */
 303     case BLKIF_OP_READ:
 304         if (request->status == BLKIF_RSP_OKAY) {
 305             block_acct_done(blk_get_stats(dataplane->blk), &request->acct);
 306         } else {
 307             block_acct_failed(blk_get_stats(dataplane->blk), &request->acct);
 308         }
 309         break;
 310     case BLKIF_OP_DISCARD:
 311     default:
 312         break;
 313     }
 314
 315     xen_block_complete_request(request);
 316
 317     if (dataplane->more_work) {
 318         qemu_bh_schedule(dataplane->bh);
 319     }
 320
 321 done:
 322     aio_context_release(dataplane->ctx);
 323 }
 324
 325 static bool xen_block_split_discard(XenBlockRequest *request,
 326                                     blkif_sector_t sector_number,
 327                                     uint64_t nr_sectors)
 328 {
 329     XenBlockDataPlane *dataplane = request->dataplane;
 330     int64_t byte_offset;
 331     int byte_chunk;
 332     uint64_t byte_remaining;
 333     uint64_t sec_start = sector_number;
 334     uint64_t sec_count = nr_sectors;
 335
 336     /* Wrap around, or overflowing byte limit? */
 337     if (sec_start + sec_count < sec_count ||
 338         sec_start + sec_count > INT64_MAX / dataplane->sector_size) {
 339         return false;
 340     }
 341
 342     byte_offset = sec_start * dataplane->sector_size;
 343     byte_remaining = sec_count * dataplane->sector_size;
 344
 345     do {
 346         byte_chunk = byte_remaining > BDRV_REQUEST_MAX_BYTES ?
 347             BDRV_REQUEST_MAX_BYTES : byte_remaining;
 348         request->aio_inflight++;
 349         blk_aio_pdiscard(dataplane->blk, byte_offset, byte_chunk,
 350                          xen_block_complete_aio, request);
 351         byte_remaining -= byte_chunk;
 352         byte_offset += byte_chunk;
 353     } while (byte_remaining > 0);
 354
 355     return true;
 356 }
 357
 358 static int xen_block_do_aio(XenBlockRequest *request)
 359 {
 360     XenBlockDataPlane *dataplane = request->dataplane;
 361
 362     if (request->req.nr_segments &&
 363         (request->req.operation == BLKIF_OP_WRITE ||
 364          request->req.operation == BLKIF_OP_FLUSH_DISKCACHE) &&
 365         xen_block_copy_request(request)) {
 366         goto err;
 367     }
 368
 369     request->aio_inflight++;
 370     if (request->presync) {
 371         blk_aio_flush(request->dataplane->blk, xen_block_complete_aio,
 372                       request);
 373         return 0;
 374     }
 375
 376     switch (request->req.operation) {
 377     case BLKIF_OP_READ:
 378         qemu_iovec_add(&request->v, request->buf, request->size);
 379         block_acct_start(blk_get_stats(dataplane->blk), &request->acct,
 380                          request->v.size, BLOCK_ACCT_READ);
 381         request->aio_inflight++;
 382         blk_aio_preadv(dataplane->blk, request->start, &request->v, 0,
 383                        xen_block_complete_aio, request);
 384         break;
 385     case BLKIF_OP_WRITE:
 386     case BLKIF_OP_FLUSH_DISKCACHE:
 387         if (!request->req.nr_segments) {
 388             break;
 389         }
 390
 391         qemu_iovec_add(&request->v, request->buf, request->size);
 392         block_acct_start(blk_get_stats(dataplane->blk), &request->acct,
 393                          request->v.size,
 394                          request->req.operation == BLKIF_OP_WRITE ?
 395                          BLOCK_ACCT_WRITE : BLOCK_ACCT_FLUSH);
 396         request->aio_inflight++;
 397         blk_aio_pwritev(dataplane->blk, request->start, &request->v, 0,
 398                         xen_block_complete_aio, request);
 399         break;
 400     case BLKIF_OP_DISCARD:
 401     {
 402         struct blkif_request_discard *req = (void *)&request->req;
 403         if (!xen_block_split_discard(request, req->sector_number,
 404                                      req->nr_sectors)) {
 405             goto err;
 406         }
 407         break;
 408     }
 409     default:
 410         /* unknown operation (shouldn't happen -- parse catches this) */
 411         goto err;
 412     }
 413
 414     xen_block_complete_aio(request, 0);
 415
 416     return 0;
 417
 418 err:
 419     request->status = BLKIF_RSP_ERROR;
 420     xen_block_complete_request(request);
 421     return -1;
 422 }
 423
 424 static int xen_block_send_response(XenBlockRequest *request)
 425 {
 426     XenBlockDataPlane *dataplane = request->dataplane;
 427     int send_notify = 0;
 428     int have_requests = 0;
 429     blkif_response_t *resp;
 430
 431     /* Place on the response ring for the relevant domain. */
 432     switch (dataplane->protocol) {
 433     case BLKIF_PROTOCOL_NATIVE:
 434         resp = (blkif_response_t *)RING_GET_RESPONSE(
 435             &dataplane->rings.native,
 436             dataplane->rings.native.rsp_prod_pvt);
 437         break;
 438     case BLKIF_PROTOCOL_X86_32:
 439         resp = (blkif_response_t *)RING_GET_RESPONSE(
 440             &dataplane->rings.x86_32_part,
 441             dataplane->rings.x86_32_part.rsp_prod_pvt);
 442         break;
 443     case BLKIF_PROTOCOL_X86_64:
 444         resp = (blkif_response_t *)RING_GET_RESPONSE(
 445             &dataplane->rings.x86_64_part,
 446             dataplane->rings.x86_64_part.rsp_prod_pvt);
 447         break;
 448     default:
 449         return 0;
 450     }
 451
 452     resp->id = request->req.id;
 453     resp->operation = request->req.operation;
 454     resp->status = request->status;
 455
 456     dataplane->rings.common.rsp_prod_pvt++;
 457
 458     RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&dataplane->rings.common,
 459                                          send_notify);
 460     if (dataplane->rings.common.rsp_prod_pvt ==
 461         dataplane->rings.common.req_cons) {
 462         /*
 463          * Tail check for pending requests. Allows frontend to avoid
 464          * notifications if requests are already in flight (lower
 465          * overheads and promotes batching).
 466          */
 467         RING_FINAL_CHECK_FOR_REQUESTS(&dataplane->rings.common,
 468                                       have_requests);
 469     } else if (RING_HAS_UNCONSUMED_REQUESTS(&dataplane->rings.common)) {
 470         have_requests = 1;
 471     }
 472
 473     if (have_requests) {
 474         dataplane->more_work++;
 475     }
 476     return send_notify;
 477 }
 478
 479 static int xen_block_get_request(XenBlockDataPlane *dataplane,
 480                                  XenBlockRequest *request, RING_IDX rc)
 481 {
 482     switch (dataplane->protocol) {
 483     case BLKIF_PROTOCOL_NATIVE: {
 484         blkif_request_t *req =
 485             RING_GET_REQUEST(&dataplane->rings.native, rc);
 486
 487         memcpy(&request->req, req, sizeof(request->req));
 488         break;
 489     }
 490     case BLKIF_PROTOCOL_X86_32: {
 491         blkif_x86_32_request_t *req =
 492             RING_GET_REQUEST(&dataplane->rings.x86_32_part, rc);
 493
 494         blkif_get_x86_32_req(&request->req, req);
 495         break;
 496     }
 497     case BLKIF_PROTOCOL_X86_64: {
 498         blkif_x86_64_request_t *req =
 499             RING_GET_REQUEST(&dataplane->rings.x86_64_part, rc);
 500
 501         blkif_get_x86_64_req(&request->req, req);
 502         break;
 503     }
 504     }
 505     /* Prevent the compiler from accessing the on-ring fields instead. */
 506     barrier();
 507     return 0;
 508 }
 509
 510 /*
 511  * Threshold of in-flight requests above which we will start using
 512  * defer_call_begin()/defer_call_end() to batch requests.
 513  */
 514 #define IO_PLUG_THRESHOLD 1
 515
 516 static bool xen_block_handle_requests(XenBlockDataPlane *dataplane)
 517 {
 518     RING_IDX rc, rp;
 519     XenBlockRequest *request;
 520     int inflight_atstart = dataplane->requests_inflight;
 521     int batched = 0;
 522     bool done_something = false;
 523
 524     dataplane->more_work = 0;
 525
 526     rc = dataplane->rings.common.req_cons;
 527     rp = dataplane->rings.common.sring->req_prod;
 528     xen_rmb(); /* Ensure we see queued requests up to 'rp'. */
 529
 530     /*
 531      * If there was more than IO_PLUG_THRESHOLD requests in flight
 532      * when we got here, this is an indication that there the bottleneck
 533      * is below us, so it's worth beginning to batch up I/O requests
 534      * rather than submitting them immediately. The maximum number
 535      * of requests we're willing to batch is the number already in
 536      * flight, so it can grow up to max_requests when the bottleneck
 537      * is below us.
 538      */
 539     if (inflight_atstart > IO_PLUG_THRESHOLD) {
 540         defer_call_begin();
 541     }
 542     while (rc != rp) {
 543         /* pull request from ring */
 544         if (RING_REQUEST_CONS_OVERFLOW(&dataplane->rings.common, rc)) {
 545             break;
 546         }
 547         request = xen_block_start_request(dataplane);
 548         if (request == NULL) {
 549             dataplane->more_work++;
 550             break;
 551         }
 552         xen_block_get_request(dataplane, request, rc);
 553         dataplane->rings.common.req_cons = ++rc;
 554         done_something = true;
 555
 556         /* parse them */
 557         if (xen_block_parse_request(request) != 0) {
 558             switch (request->req.operation) {
 559             case BLKIF_OP_READ:
 560                 block_acct_invalid(blk_get_stats(dataplane->blk),
 561                                    BLOCK_ACCT_READ);
 562                 break;
 563             case BLKIF_OP_WRITE:
 564                 block_acct_invalid(blk_get_stats(dataplane->blk),
 565                                    BLOCK_ACCT_WRITE);
 566                 break;
 567             case BLKIF_OP_FLUSH_DISKCACHE:
 568                 block_acct_invalid(blk_get_stats(dataplane->blk),
 569                                    BLOCK_ACCT_FLUSH);
 570             default:
 571                 break;
 572             };
 573
 574             xen_block_complete_request(request);
 575             continue;
 576         }
 577
 578         if (inflight_atstart > IO_PLUG_THRESHOLD &&
 579             batched >= inflight_atstart) {
 580             defer_call_end();
 581         }
 582         xen_block_do_aio(request);
 583         if (inflight_atstart > IO_PLUG_THRESHOLD) {
 584             if (batched >= inflight_atstart) {
 585                 defer_call_begin();
 586                 batched = 0;
 587             } else {
 588                 batched++;
 589             }
 590         }
 591     }
 592     if (inflight_atstart > IO_PLUG_THRESHOLD) {
 593         defer_call_end();
 594     }
 595
 596     return done_something;
 597 }
 598
 599 static void xen_block_dataplane_bh(void *opaque)
 600 {
 601     XenBlockDataPlane *dataplane = opaque;
 602
 603     aio_context_acquire(dataplane->ctx);
 604     xen_block_handle_requests(dataplane);
 605     aio_context_release(dataplane->ctx);
 606 }
 607
 608 static bool xen_block_dataplane_event(void *opaque)
 609 {
 610     XenBlockDataPlane *dataplane = opaque;
 611
 612     return xen_block_handle_requests(dataplane);
 613 }
 614
 615 XenBlockDataPlane *xen_block_dataplane_create(XenDevice *xendev,
 616                                               BlockBackend *blk,
 617                                               unsigned int sector_size,
 618                                               IOThread *iothread)
 619 {
 620     XenBlockDataPlane *dataplane = g_new0(XenBlockDataPlane, 1);
 621
 622     dataplane->xendev = xendev;
 623     dataplane->blk = blk;
 624     dataplane->sector_size = sector_size;
 625
 626     QLIST_INIT(&dataplane->inflight);
 627     QLIST_INIT(&dataplane->freelist);
 628
 629     if (iothread) {
 630         dataplane->iothread = iothread;
 631         object_ref(OBJECT(dataplane->iothread));
 632         dataplane->ctx = iothread_get_aio_context(dataplane->iothread);
 633     } else {
 634         dataplane->ctx = qemu_get_aio_context();
 635     }
 636     dataplane->bh = aio_bh_new_guarded(dataplane->ctx, xen_block_dataplane_bh,
 637                                        dataplane,
 638                                        &DEVICE(xendev)->mem_reentrancy_guard);
 639
 640     return dataplane;
 641 }
 642
 643 void xen_block_dataplane_destroy(XenBlockDataPlane *dataplane)
 644 {
 645     XenBlockRequest *request;
 646
 647     if (!dataplane) {
 648         return;
 649     }
 650
 651     while (!QLIST_EMPTY(&dataplane->freelist)) {
 652         request = QLIST_FIRST(&dataplane->freelist);
 653         QLIST_REMOVE(request, list);
 654         qemu_iovec_destroy(&request->v);
 655         qemu_vfree(request->buf);
 656         g_free(request);
 657     }
 658
 659     qemu_bh_delete(dataplane->bh);
 660     if (dataplane->iothread) {
 661         object_unref(OBJECT(dataplane->iothread));
 662     }
 663
 664     g_free(dataplane);
 665 }
 666
 667 void xen_block_dataplane_detach(XenBlockDataPlane *dataplane)
 668 {
 669     if (!dataplane || !dataplane->event_channel) {
 670         return;
 671     }
 672
 673     /* Only reason for failure is a NULL channel */
 674     xen_device_set_event_channel_context(dataplane->xendev,
 675                                          dataplane->event_channel,
 676                                          NULL, &error_abort);
 677 }
 678
 679 void xen_block_dataplane_attach(XenBlockDataPlane *dataplane)
 680 {
 681     if (!dataplane || !dataplane->event_channel) {
 682         return;
 683     }
 684
 685     /* Only reason for failure is a NULL channel */
 686     xen_device_set_event_channel_context(dataplane->xendev,
 687                                          dataplane->event_channel,
 688                                          dataplane->ctx, &error_abort);
 689 }
 690
 691 void xen_block_dataplane_stop(XenBlockDataPlane *dataplane)
 692 {
 693     XenDevice *xendev;
 694
 695     if (!dataplane) {
 696         return;
 697     }
 698
 699     xendev = dataplane->xendev;
 700
 701     if (!blk_in_drain(dataplane->blk)) {
 702         xen_block_dataplane_detach(dataplane);
 703     }
 704
 705     aio_context_acquire(dataplane->ctx);
 706     /* Xen doesn't have multiple users for nodes, so this can't fail */
 707     blk_set_aio_context(dataplane->blk, qemu_get_aio_context(), &error_abort);
 708     aio_context_release(dataplane->ctx);
 709
 710     /*
 711      * Now that the context has been moved onto the main thread, cancel
 712      * further processing.
 713      */
 714     qemu_bh_cancel(dataplane->bh);
 715
 716     if (dataplane->event_channel) {
 717         Error *local_err = NULL;
 718
 719         xen_device_unbind_event_channel(xendev, dataplane->event_channel,
 720                                         &local_err);
 721         dataplane->event_channel = NULL;
 722
 723         if (local_err) {
 724             error_report_err(local_err);
 725         }
 726     }
 727
 728     if (dataplane->sring) {
 729         Error *local_err = NULL;
 730
 731         xen_device_unmap_grant_refs(xendev, dataplane->sring,
 732                                     dataplane->ring_ref,
 733                                     dataplane->nr_ring_ref, &local_err);
 734         dataplane->sring = NULL;
 735
 736         if (local_err) {
 737             error_report_err(local_err);
 738         }
 739     }
 740
 741     g_free(dataplane->ring_ref);
 742     dataplane->ring_ref = NULL;
 743 }
 744
 745 void xen_block_dataplane_start(XenBlockDataPlane *dataplane,
 746                                const unsigned int ring_ref[],
 747                                unsigned int nr_ring_ref,
 748                                unsigned int event_channel,
 749                                unsigned int protocol,
 750                                Error **errp)
 751 {
 752     ERRP_GUARD();
 753     XenDevice *xendev = dataplane->xendev;
 754     AioContext *old_context;
 755     unsigned int ring_size;
 756     unsigned int i;
 757
 758     dataplane->nr_ring_ref = nr_ring_ref;
 759     dataplane->ring_ref = g_new(unsigned int, nr_ring_ref);
 760
 761     for (i = 0; i < nr_ring_ref; i++) {
 762         dataplane->ring_ref[i] = ring_ref[i];
 763     }
 764
 765     dataplane->protocol = protocol;
 766
 767     ring_size = XEN_PAGE_SIZE * dataplane->nr_ring_ref;
 768     switch (dataplane->protocol) {
 769     case BLKIF_PROTOCOL_NATIVE:
 770     {
 771         dataplane->max_requests = __CONST_RING_SIZE(blkif, ring_size);
 772         break;
 773     }
 774     case BLKIF_PROTOCOL_X86_32:
 775     {
 776         dataplane->max_requests = __CONST_RING_SIZE(blkif_x86_32, ring_size);
 777         break;
 778     }
 779     case BLKIF_PROTOCOL_X86_64:
 780     {
 781         dataplane->max_requests = __CONST_RING_SIZE(blkif_x86_64, ring_size);
 782         break;
 783     }
 784     default:
 785         error_setg(errp, "unknown protocol %u", dataplane->protocol);
 786         return;
 787     }
 788
 789     xen_device_set_max_grant_refs(xendev, dataplane->nr_ring_ref,
 790                                   errp);
 791     if (*errp) {
 792         goto stop;
 793     }
 794
 795     dataplane->sring = xen_device_map_grant_refs(xendev,
 796                                               dataplane->ring_ref,
 797                                               dataplane->nr_ring_ref,
 798                                               PROT_READ | PROT_WRITE,
 799                                               errp);
 800     if (*errp) {
 801         goto stop;
 802     }
 803
 804     switch (dataplane->protocol) {
 805     case BLKIF_PROTOCOL_NATIVE:
 806     {
 807         blkif_sring_t *sring_native = dataplane->sring;
 808
 809         BACK_RING_INIT(&dataplane->rings.native, sring_native, ring_size);
 810         break;
 811     }
 812     case BLKIF_PROTOCOL_X86_32:
 813     {
 814         blkif_x86_32_sring_t *sring_x86_32 = dataplane->sring;
 815
 816         BACK_RING_INIT(&dataplane->rings.x86_32_part, sring_x86_32,
 817                        ring_size);
 818         break;
 819     }
 820     case BLKIF_PROTOCOL_X86_64:
 821     {
 822         blkif_x86_64_sring_t *sring_x86_64 = dataplane->sring;
 823
 824         BACK_RING_INIT(&dataplane->rings.x86_64_part, sring_x86_64,
 825                        ring_size);
 826         break;
 827     }
 828     }
 829
 830     dataplane->event_channel =
 831         xen_device_bind_event_channel(xendev, event_channel,
 832                                       xen_block_dataplane_event, dataplane,
 833                                       errp);
 834     if (*errp) {
 835         goto stop;
 836     }
 837
 838     old_context = blk_get_aio_context(dataplane->blk);
 839     aio_context_acquire(old_context);
 840     /* If other users keep the BlockBackend in the iothread, that's ok */
 841     blk_set_aio_context(dataplane->blk, dataplane->ctx, NULL);
 842     aio_context_release(old_context);
 843
 844     if (!blk_in_drain(dataplane->blk)) {
 845         xen_block_dataplane_attach(dataplane);
 846     }
 847
 848     return;
 849
 850 stop:
 851     xen_block_dataplane_stop(dataplane);
 852 }