hw/block/dataplane/xen-block.c

   1 /*
   2  * Copyright (c) 2018  Citrix Systems Inc.
   3  * (c) Gerd Hoffmann <kraxel@redhat.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; under version 2 of the License.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along
  15  * with this program; if not, see <http://www.gnu.org/licenses/>.
  16  *
  17  * Contributions after 2012-01-13 are licensed under the terms of the
  18  * GNU GPL, version 2 or (at your option) any later version.
  19  */
  20
  21 #include "qemu/osdep.h"
  22 #include "qemu/error-report.h"
  23 #include "qapi/error.h"
  24 #include "hw/hw.h"
  25 #include "hw/xen/xen_common.h"
  26 #include "hw/block/xen_blkif.h"
  27 #include "sysemu/block-backend.h"
  28 #include "sysemu/iothread.h"
  29 #include "xen-block.h"
  30
  31 typedef struct XenBlockRequest {
  32     blkif_request_t req;
  33     int16_t status;
  34     off_t start;
  35     QEMUIOVector v;
  36     void *buf;
  37     size_t size;
  38     int presync;
  39     int aio_inflight;
  40     int aio_errors;
  41     XenBlockDataPlane *dataplane;
  42     QLIST_ENTRY(XenBlockRequest) list;
  43     BlockAcctCookie acct;
  44 } XenBlockRequest;
  45
  46 struct XenBlockDataPlane {
  47     XenDevice *xendev;
  48     XenEventChannel *event_channel;
  49     unsigned int *ring_ref;
  50     unsigned int nr_ring_ref;
  51     void *sring;
  52     int64_t file_blk;
  53     int protocol;
  54     blkif_back_rings_t rings;
  55     int more_work;
  56     QLIST_HEAD(inflight_head, XenBlockRequest) inflight;
  57     QLIST_HEAD(freelist_head, XenBlockRequest) freelist;
  58     int requests_total;
  59     int requests_inflight;
  60     unsigned int max_requests;
  61     BlockBackend *blk;
  62     QEMUBH *bh;
  63     IOThread *iothread;
  64     AioContext *ctx;
  65 };
  66
  67 static void reset_request(XenBlockRequest *request)
  68 {
  69     memset(&request->req, 0, sizeof(request->req));
  70     request->status = 0;
  71     request->start = 0;
  72     request->size = 0;
  73     request->presync = 0;
  74
  75     request->aio_inflight = 0;
  76     request->aio_errors = 0;
  77
  78     request->dataplane = NULL;
  79     memset(&request->list, 0, sizeof(request->list));
  80     memset(&request->acct, 0, sizeof(request->acct));
  81
  82     qemu_iovec_reset(&request->v);
  83 }
  84
  85 static XenBlockRequest *xen_block_start_request(XenBlockDataPlane *dataplane)
  86 {
  87     XenBlockRequest *request = NULL;
  88
  89     if (QLIST_EMPTY(&dataplane->freelist)) {
  90         if (dataplane->requests_total >= dataplane->max_requests) {
  91             goto out;
  92         }
  93         /* allocate new struct */
  94         request = g_malloc0(sizeof(*request));
  95         request->dataplane = dataplane;
  96         /*
  97          * We cannot need more pages per requests than this, and since we
  98          * re-use requests, allocate the memory once here. It will be freed
  99          * xen_block_dataplane_destroy() when the request list is freed.
 100          */
 101         request->buf = qemu_memalign(XC_PAGE_SIZE,
 102                                      BLKIF_MAX_SEGMENTS_PER_REQUEST *
 103                                      XC_PAGE_SIZE);
 104         dataplane->requests_total++;
 105         qemu_iovec_init(&request->v, 1);
 106     } else {
 107         /* get one from freelist */
 108         request = QLIST_FIRST(&dataplane->freelist);
 109         QLIST_REMOVE(request, list);
 110     }
 111     QLIST_INSERT_HEAD(&dataplane->inflight, request, list);
 112     dataplane->requests_inflight++;
 113
 114 out:
 115     return request;
 116 }
 117
 118 static void xen_block_finish_request(XenBlockRequest *request)
 119 {
 120     XenBlockDataPlane *dataplane = request->dataplane;
 121
 122     QLIST_REMOVE(request, list);
 123     dataplane->requests_inflight--;
 124 }
 125
 126 static void xen_block_release_request(XenBlockRequest *request)
 127 {
 128     XenBlockDataPlane *dataplane = request->dataplane;
 129
 130     QLIST_REMOVE(request, list);
 131     reset_request(request);
 132     request->dataplane = dataplane;
 133     QLIST_INSERT_HEAD(&dataplane->freelist, request, list);
 134     dataplane->requests_inflight--;
 135 }
 136
 137 /*
 138  * translate request into iovec + start offset
 139  * do sanity checks along the way
 140  */
 141 static int xen_block_parse_request(XenBlockRequest *request)
 142 {
 143     XenBlockDataPlane *dataplane = request->dataplane;
 144     size_t len;
 145     int i;
 146
 147     switch (request->req.operation) {
 148     case BLKIF_OP_READ:
 149         break;
 150     case BLKIF_OP_FLUSH_DISKCACHE:
 151         request->presync = 1;
 152         if (!request->req.nr_segments) {
 153             return 0;
 154         }
 155         /* fall through */
 156     case BLKIF_OP_WRITE:
 157         break;
 158     case BLKIF_OP_DISCARD:
 159         return 0;
 160     default:
 161         error_report("error: unknown operation (%d)", request->req.operation);
 162         goto err;
 163     };
 164
 165     if (request->req.operation != BLKIF_OP_READ &&
 166         blk_is_read_only(dataplane->blk)) {
 167         error_report("error: write req for ro device");
 168         goto err;
 169     }
 170
 171     request->start = request->req.sector_number * dataplane->file_blk;
 172     for (i = 0; i < request->req.nr_segments; i++) {
 173         if (i == BLKIF_MAX_SEGMENTS_PER_REQUEST) {
 174             error_report("error: nr_segments too big");
 175             goto err;
 176         }
 177         if (request->req.seg[i].first_sect > request->req.seg[i].last_sect) {
 178             error_report("error: first > last sector");
 179             goto err;
 180         }
 181         if (request->req.seg[i].last_sect * dataplane->file_blk >=
 182             XC_PAGE_SIZE) {
 183             error_report("error: page crossing");
 184             goto err;
 185         }
 186
 187         len = (request->req.seg[i].last_sect -
 188                request->req.seg[i].first_sect + 1) * dataplane->file_blk;
 189         request->size += len;
 190     }
 191     if (request->start + request->size > blk_getlength(dataplane->blk)) {
 192         error_report("error: access beyond end of file");
 193         goto err;
 194     }
 195     return 0;
 196
 197 err:
 198     request->status = BLKIF_RSP_ERROR;
 199     return -1;
 200 }
 201
 202 static int xen_block_copy_request(XenBlockRequest *request)
 203 {
 204     XenBlockDataPlane *dataplane = request->dataplane;
 205     XenDevice *xendev = dataplane->xendev;
 206     XenDeviceGrantCopySegment segs[BLKIF_MAX_SEGMENTS_PER_REQUEST];
 207     int i, count;
 208     int64_t file_blk = dataplane->file_blk;
 209     bool to_domain = (request->req.operation == BLKIF_OP_READ);
 210     void *virt = request->buf;
 211     Error *local_err = NULL;
 212
 213     if (request->req.nr_segments == 0) {
 214         return 0;
 215     }
 216
 217     count = request->req.nr_segments;
 218
 219     for (i = 0; i < count; i++) {
 220         if (to_domain) {
 221             segs[i].dest.foreign.ref = request->req.seg[i].gref;
 222             segs[i].dest.foreign.offset = request->req.seg[i].first_sect *
 223                 file_blk;
 224             segs[i].source.virt = virt;
 225         } else {
 226             segs[i].source.foreign.ref = request->req.seg[i].gref;
 227             segs[i].source.foreign.offset = request->req.seg[i].first_sect *
 228                 file_blk;
 229             segs[i].dest.virt = virt;
 230         }
 231         segs[i].len = (request->req.seg[i].last_sect -
 232                        request->req.seg[i].first_sect + 1) * file_blk;
 233         virt += segs[i].len;
 234     }
 235
 236     xen_device_copy_grant_refs(xendev, to_domain, segs, count, &local_err);
 237
 238     if (local_err) {
 239         error_reportf_err(local_err, "failed to copy data: ");
 240
 241         request->aio_errors++;
 242         return -1;
 243     }
 244
 245     return 0;
 246 }
 247
 248 static int xen_block_do_aio(XenBlockRequest *request);
 249 static int xen_block_send_response(XenBlockRequest *request);
 250
 251 static void xen_block_complete_aio(void *opaque, int ret)
 252 {
 253     XenBlockRequest *request = opaque;
 254     XenBlockDataPlane *dataplane = request->dataplane;
 255
 256     aio_context_acquire(dataplane->ctx);
 257
 258     if (ret != 0) {
 259         error_report("%s I/O error",
 260                      request->req.operation == BLKIF_OP_READ ?
 261                      "read" : "write");
 262         request->aio_errors++;
 263     }
 264
 265     request->aio_inflight--;
 266     if (request->presync) {
 267         request->presync = 0;
 268         xen_block_do_aio(request);
 269         goto done;
 270     }
 271     if (request->aio_inflight > 0) {
 272         goto done;
 273     }
 274
 275     switch (request->req.operation) {
 276     case BLKIF_OP_READ:
 277         /* in case of failure request->aio_errors is increased */
 278         if (ret == 0) {
 279             xen_block_copy_request(request);
 280         }
 281         break;
 282     case BLKIF_OP_WRITE:
 283     case BLKIF_OP_FLUSH_DISKCACHE:
 284         if (!request->req.nr_segments) {
 285             break;
 286         }
 287         break;
 288     default:
 289         break;
 290     }
 291
 292     request->status = request->aio_errors ? BLKIF_RSP_ERROR : BLKIF_RSP_OKAY;
 293     xen_block_finish_request(request);
 294
 295     switch (request->req.operation) {
 296     case BLKIF_OP_WRITE:
 297     case BLKIF_OP_FLUSH_DISKCACHE:
 298         if (!request->req.nr_segments) {
 299             break;
 300         }
 301     case BLKIF_OP_READ:
 302         if (request->status == BLKIF_RSP_OKAY) {
 303             block_acct_done(blk_get_stats(dataplane->blk), &request->acct);
 304         } else {
 305             block_acct_failed(blk_get_stats(dataplane->blk), &request->acct);
 306         }
 307         break;
 308     case BLKIF_OP_DISCARD:
 309     default:
 310         break;
 311     }
 312     if (xen_block_send_response(request)) {
 313         Error *local_err = NULL;
 314
 315         xen_device_notify_event_channel(dataplane->xendev,
 316                                         dataplane->event_channel,
 317                                         &local_err);
 318         if (local_err) {
 319             error_report_err(local_err);
 320         }
 321     }
 322     xen_block_release_request(request);
 323
 324     qemu_bh_schedule(dataplane->bh);
 325
 326 done:
 327     aio_context_release(dataplane->ctx);
 328 }
 329
 330 static bool xen_block_split_discard(XenBlockRequest *request,
 331                                     blkif_sector_t sector_number,
 332                                     uint64_t nr_sectors)
 333 {
 334     XenBlockDataPlane *dataplane = request->dataplane;
 335     int64_t byte_offset;
 336     int byte_chunk;
 337     uint64_t byte_remaining, limit;
 338     uint64_t sec_start = sector_number;
 339     uint64_t sec_count = nr_sectors;
 340
 341     /* Wrap around, or overflowing byte limit? */
 342     if (sec_start + sec_count < sec_count ||
 343         sec_start + sec_count > INT64_MAX / dataplane->file_blk) {
 344         return false;
 345     }
 346
 347     limit = BDRV_REQUEST_MAX_SECTORS * dataplane->file_blk;
 348     byte_offset = sec_start * dataplane->file_blk;
 349     byte_remaining = sec_count * dataplane->file_blk;
 350
 351     do {
 352         byte_chunk = byte_remaining > limit ? limit : byte_remaining;
 353         request->aio_inflight++;
 354         blk_aio_pdiscard(dataplane->blk, byte_offset, byte_chunk,
 355                          xen_block_complete_aio, request);
 356         byte_remaining -= byte_chunk;
 357         byte_offset += byte_chunk;
 358     } while (byte_remaining > 0);
 359
 360     return true;
 361 }
 362
 363 static int xen_block_do_aio(XenBlockRequest *request)
 364 {
 365     XenBlockDataPlane *dataplane = request->dataplane;
 366
 367     if (request->req.nr_segments &&
 368         (request->req.operation == BLKIF_OP_WRITE ||
 369          request->req.operation == BLKIF_OP_FLUSH_DISKCACHE) &&
 370         xen_block_copy_request(request)) {
 371         goto err;
 372     }
 373
 374     request->aio_inflight++;
 375     if (request->presync) {
 376         blk_aio_flush(request->dataplane->blk, xen_block_complete_aio,
 377                       request);
 378         return 0;
 379     }
 380
 381     switch (request->req.operation) {
 382     case BLKIF_OP_READ:
 383         qemu_iovec_add(&request->v, request->buf, request->size);
 384         block_acct_start(blk_get_stats(dataplane->blk), &request->acct,
 385                          request->v.size, BLOCK_ACCT_READ);
 386         request->aio_inflight++;
 387         blk_aio_preadv(dataplane->blk, request->start, &request->v, 0,
 388                        xen_block_complete_aio, request);
 389         break;
 390     case BLKIF_OP_WRITE:
 391     case BLKIF_OP_FLUSH_DISKCACHE:
 392         if (!request->req.nr_segments) {
 393             break;
 394         }
 395
 396         qemu_iovec_add(&request->v, request->buf, request->size);
 397         block_acct_start(blk_get_stats(dataplane->blk), &request->acct,
 398                          request->v.size,
 399                          request->req.operation == BLKIF_OP_WRITE ?
 400                          BLOCK_ACCT_WRITE : BLOCK_ACCT_FLUSH);
 401         request->aio_inflight++;
 402         blk_aio_pwritev(dataplane->blk, request->start, &request->v, 0,
 403                         xen_block_complete_aio, request);
 404         break;
 405     case BLKIF_OP_DISCARD:
 406     {
 407         struct blkif_request_discard *req = (void *)&request->req;
 408         if (!xen_block_split_discard(request, req->sector_number,
 409                                      req->nr_sectors)) {
 410             goto err;
 411         }
 412         break;
 413     }
 414     default:
 415         /* unknown operation (shouldn't happen -- parse catches this) */
 416         goto err;
 417     }
 418
 419     xen_block_complete_aio(request, 0);
 420
 421     return 0;
 422
 423 err:
 424     xen_block_finish_request(request);
 425     request->status = BLKIF_RSP_ERROR;
 426     return -1;
 427 }
 428
 429 static int xen_block_send_response(XenBlockRequest *request)
 430 {
 431     XenBlockDataPlane *dataplane = request->dataplane;
 432     int send_notify = 0;
 433     int have_requests = 0;
 434     blkif_response_t *resp;
 435
 436     /* Place on the response ring for the relevant domain. */
 437     switch (dataplane->protocol) {
 438     case BLKIF_PROTOCOL_NATIVE:
 439         resp = (blkif_response_t *)RING_GET_RESPONSE(
 440             &dataplane->rings.native,
 441             dataplane->rings.native.rsp_prod_pvt);
 442         break;
 443     case BLKIF_PROTOCOL_X86_32:
 444         resp = (blkif_response_t *)RING_GET_RESPONSE(
 445             &dataplane->rings.x86_32_part,
 446             dataplane->rings.x86_32_part.rsp_prod_pvt);
 447         break;
 448     case BLKIF_PROTOCOL_X86_64:
 449         resp = (blkif_response_t *)RING_GET_RESPONSE(
 450             &dataplane->rings.x86_64_part,
 451             dataplane->rings.x86_64_part.rsp_prod_pvt);
 452         break;
 453     default:
 454         return 0;
 455     }
 456
 457     resp->id = request->req.id;
 458     resp->operation = request->req.operation;
 459     resp->status = request->status;
 460
 461     dataplane->rings.common.rsp_prod_pvt++;
 462
 463     RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&dataplane->rings.common,
 464                                          send_notify);
 465     if (dataplane->rings.common.rsp_prod_pvt ==
 466         dataplane->rings.common.req_cons) {
 467         /*
 468          * Tail check for pending requests. Allows frontend to avoid
 469          * notifications if requests are already in flight (lower
 470          * overheads and promotes batching).
 471          */
 472         RING_FINAL_CHECK_FOR_REQUESTS(&dataplane->rings.common,
 473                                       have_requests);
 474     } else if (RING_HAS_UNCONSUMED_REQUESTS(&dataplane->rings.common)) {
 475         have_requests = 1;
 476     }
 477
 478     if (have_requests) {
 479         dataplane->more_work++;
 480     }
 481     return send_notify;
 482 }
 483
 484 static int xen_block_get_request(XenBlockDataPlane *dataplane,
 485                                  XenBlockRequest *request, RING_IDX rc)
 486 {
 487     switch (dataplane->protocol) {
 488     case BLKIF_PROTOCOL_NATIVE: {
 489         blkif_request_t *req =
 490             RING_GET_REQUEST(&dataplane->rings.native, rc);
 491
 492         memcpy(&request->req, req, sizeof(request->req));
 493         break;
 494     }
 495     case BLKIF_PROTOCOL_X86_32: {
 496         blkif_x86_32_request_t *req =
 497             RING_GET_REQUEST(&dataplane->rings.x86_32_part, rc);
 498
 499         blkif_get_x86_32_req(&request->req, req);
 500         break;
 501     }
 502     case BLKIF_PROTOCOL_X86_64: {
 503         blkif_x86_64_request_t *req =
 504             RING_GET_REQUEST(&dataplane->rings.x86_64_part, rc);
 505
 506         blkif_get_x86_64_req(&request->req, req);
 507         break;
 508     }
 509     }
 510     /* Prevent the compiler from accessing the on-ring fields instead. */
 511     barrier();
 512     return 0;
 513 }
 514
 515 /*
 516  * Threshold of in-flight requests above which we will start using
 517  * blk_io_plug()/blk_io_unplug() to batch requests.
 518  */
 519 #define IO_PLUG_THRESHOLD 1
 520
 521 static void xen_block_handle_requests(XenBlockDataPlane *dataplane)
 522 {
 523     RING_IDX rc, rp;
 524     XenBlockRequest *request;
 525     int inflight_atstart = dataplane->requests_inflight;
 526     int batched = 0;
 527
 528     dataplane->more_work = 0;
 529
 530     rc = dataplane->rings.common.req_cons;
 531     rp = dataplane->rings.common.sring->req_prod;
 532     xen_rmb(); /* Ensure we see queued requests up to 'rp'. */
 533
 534     /*
 535      * If there was more than IO_PLUG_THRESHOLD requests in flight
 536      * when we got here, this is an indication that there the bottleneck
 537      * is below us, so it's worth beginning to batch up I/O requests
 538      * rather than submitting them immediately. The maximum number
 539      * of requests we're willing to batch is the number already in
 540      * flight, so it can grow up to max_requests when the bottleneck
 541      * is below us.
 542      */
 543     if (inflight_atstart > IO_PLUG_THRESHOLD) {
 544         blk_io_plug(dataplane->blk);
 545     }
 546     while (rc != rp) {
 547         /* pull request from ring */
 548         if (RING_REQUEST_CONS_OVERFLOW(&dataplane->rings.common, rc)) {
 549             break;
 550         }
 551         request = xen_block_start_request(dataplane);
 552         if (request == NULL) {
 553             dataplane->more_work++;
 554             break;
 555         }
 556         xen_block_get_request(dataplane, request, rc);
 557         dataplane->rings.common.req_cons = ++rc;
 558
 559         /* parse them */
 560         if (xen_block_parse_request(request) != 0) {
 561             switch (request->req.operation) {
 562             case BLKIF_OP_READ:
 563                 block_acct_invalid(blk_get_stats(dataplane->blk),
 564                                    BLOCK_ACCT_READ);
 565                 break;
 566             case BLKIF_OP_WRITE:
 567                 block_acct_invalid(blk_get_stats(dataplane->blk),
 568                                    BLOCK_ACCT_WRITE);
 569                 break;
 570             case BLKIF_OP_FLUSH_DISKCACHE:
 571                 block_acct_invalid(blk_get_stats(dataplane->blk),
 572                                    BLOCK_ACCT_FLUSH);
 573             default:
 574                 break;
 575             };
 576
 577             if (xen_block_send_response(request)) {
 578                 Error *local_err = NULL;
 579
 580                 xen_device_notify_event_channel(dataplane->xendev,
 581                                                 dataplane->event_channel,
 582                                                 &local_err);
 583                 if (local_err) {
 584                     error_report_err(local_err);
 585                 }
 586             }
 587             xen_block_release_request(request);
 588             continue;
 589         }
 590
 591         if (inflight_atstart > IO_PLUG_THRESHOLD &&
 592             batched >= inflight_atstart) {
 593             blk_io_unplug(dataplane->blk);
 594         }
 595         xen_block_do_aio(request);
 596         if (inflight_atstart > IO_PLUG_THRESHOLD) {
 597             if (batched >= inflight_atstart) {
 598                 blk_io_plug(dataplane->blk);
 599                 batched = 0;
 600             } else {
 601                 batched++;
 602             }
 603         }
 604     }
 605     if (inflight_atstart > IO_PLUG_THRESHOLD) {
 606         blk_io_unplug(dataplane->blk);
 607     }
 608
 609     if (dataplane->more_work &&
 610         dataplane->requests_inflight < dataplane->max_requests) {
 611         qemu_bh_schedule(dataplane->bh);
 612     }
 613 }
 614
 615 static void xen_block_dataplane_bh(void *opaque)
 616 {
 617     XenBlockDataPlane *dataplane = opaque;
 618
 619     aio_context_acquire(dataplane->ctx);
 620     xen_block_handle_requests(dataplane);
 621     aio_context_release(dataplane->ctx);
 622 }
 623
 624 static void xen_block_dataplane_event(void *opaque)
 625 {
 626     XenBlockDataPlane *dataplane = opaque;
 627
 628     qemu_bh_schedule(dataplane->bh);
 629 }
 630
 631 XenBlockDataPlane *xen_block_dataplane_create(XenDevice *xendev,
 632                                               BlockConf *conf,
 633                                               IOThread *iothread)
 634 {
 635     XenBlockDataPlane *dataplane = g_new0(XenBlockDataPlane, 1);
 636
 637     dataplane->xendev = xendev;
 638     dataplane->file_blk = conf->logical_block_size;
 639     dataplane->blk = conf->blk;
 640
 641     QLIST_INIT(&dataplane->inflight);
 642     QLIST_INIT(&dataplane->freelist);
 643
 644     if (iothread) {
 645         dataplane->iothread = iothread;
 646         object_ref(OBJECT(dataplane->iothread));
 647         dataplane->ctx = iothread_get_aio_context(dataplane->iothread);
 648     } else {
 649         dataplane->ctx = qemu_get_aio_context();
 650     }
 651     dataplane->bh = aio_bh_new(dataplane->ctx, xen_block_dataplane_bh,
 652                                dataplane);
 653
 654     return dataplane;
 655 }
 656
 657 void xen_block_dataplane_destroy(XenBlockDataPlane *dataplane)
 658 {
 659     XenBlockRequest *request;
 660
 661     if (!dataplane) {
 662         return;
 663     }
 664
 665     while (!QLIST_EMPTY(&dataplane->freelist)) {
 666         request = QLIST_FIRST(&dataplane->freelist);
 667         QLIST_REMOVE(request, list);
 668         qemu_iovec_destroy(&request->v);
 669         qemu_vfree(request->buf);
 670         g_free(request);
 671     }
 672
 673     qemu_bh_delete(dataplane->bh);
 674     if (dataplane->iothread) {
 675         object_unref(OBJECT(dataplane->iothread));
 676     }
 677
 678     g_free(dataplane);
 679 }
 680
 681 void xen_block_dataplane_stop(XenBlockDataPlane *dataplane)
 682 {
 683     XenDevice *xendev;
 684
 685     if (!dataplane) {
 686         return;
 687     }
 688
 689     aio_context_acquire(dataplane->ctx);
 690     blk_set_aio_context(dataplane->blk, qemu_get_aio_context());
 691     aio_context_release(dataplane->ctx);
 692
 693     xendev = dataplane->xendev;
 694
 695     if (dataplane->event_channel) {
 696         Error *local_err = NULL;
 697
 698         xen_device_unbind_event_channel(xendev, dataplane->event_channel,
 699                                         &local_err);
 700         dataplane->event_channel = NULL;
 701
 702         if (local_err) {
 703             error_report_err(local_err);
 704         }
 705     }
 706
 707     if (dataplane->sring) {
 708         Error *local_err = NULL;
 709
 710         xen_device_unmap_grant_refs(xendev, dataplane->sring,
 711                                     dataplane->nr_ring_ref, &local_err);
 712         dataplane->sring = NULL;
 713
 714         if (local_err) {
 715             error_report_err(local_err);
 716         }
 717     }
 718
 719     g_free(dataplane->ring_ref);
 720     dataplane->ring_ref = NULL;
 721 }
 722
 723 void xen_block_dataplane_start(XenBlockDataPlane *dataplane,
 724                                const unsigned int ring_ref[],
 725                                unsigned int nr_ring_ref,
 726                                unsigned int event_channel,
 727                                unsigned int protocol,
 728                                Error **errp)
 729 {
 730     XenDevice *xendev = dataplane->xendev;
 731     Error *local_err = NULL;
 732     unsigned int ring_size;
 733     unsigned int i;
 734
 735     dataplane->nr_ring_ref = nr_ring_ref;
 736     dataplane->ring_ref = g_new(unsigned int, nr_ring_ref);
 737
 738     for (i = 0; i < nr_ring_ref; i++) {
 739         dataplane->ring_ref[i] = ring_ref[i];
 740     }
 741
 742     dataplane->protocol = protocol;
 743
 744     ring_size = XC_PAGE_SIZE * dataplane->nr_ring_ref;
 745     switch (dataplane->protocol) {
 746     case BLKIF_PROTOCOL_NATIVE:
 747     {
 748         dataplane->max_requests = __CONST_RING_SIZE(blkif, ring_size);
 749         break;
 750     }
 751     case BLKIF_PROTOCOL_X86_32:
 752     {
 753         dataplane->max_requests = __CONST_RING_SIZE(blkif_x86_32, ring_size);
 754         break;
 755     }
 756     case BLKIF_PROTOCOL_X86_64:
 757     {
 758         dataplane->max_requests = __CONST_RING_SIZE(blkif_x86_64, ring_size);
 759         break;
 760     }
 761     default:
 762         error_setg(errp, "unknown protocol %u", dataplane->protocol);
 763         return;
 764     }
 765
 766     xen_device_set_max_grant_refs(xendev, dataplane->nr_ring_ref,
 767                                   &local_err);
 768     if (local_err) {
 769         error_propagate(errp, local_err);
 770         goto stop;
 771     }
 772
 773     dataplane->sring = xen_device_map_grant_refs(xendev,
 774                                               dataplane->ring_ref,
 775                                               dataplane->nr_ring_ref,
 776                                               PROT_READ | PROT_WRITE,
 777                                               &local_err);
 778     if (local_err) {
 779         error_propagate(errp, local_err);
 780         goto stop;
 781     }
 782
 783     switch (dataplane->protocol) {
 784     case BLKIF_PROTOCOL_NATIVE:
 785     {
 786         blkif_sring_t *sring_native = dataplane->sring;
 787
 788         BACK_RING_INIT(&dataplane->rings.native, sring_native, ring_size);
 789         break;
 790     }
 791     case BLKIF_PROTOCOL_X86_32:
 792     {
 793         blkif_x86_32_sring_t *sring_x86_32 = dataplane->sring;
 794
 795         BACK_RING_INIT(&dataplane->rings.x86_32_part, sring_x86_32,
 796                        ring_size);
 797         break;
 798     }
 799     case BLKIF_PROTOCOL_X86_64:
 800     {
 801         blkif_x86_64_sring_t *sring_x86_64 = dataplane->sring;
 802
 803         BACK_RING_INIT(&dataplane->rings.x86_64_part, sring_x86_64,
 804                        ring_size);
 805         break;
 806     }
 807     }
 808
 809     dataplane->event_channel =
 810         xen_device_bind_event_channel(xendev, event_channel,
 811                                       xen_block_dataplane_event, dataplane,
 812                                       &local_err);
 813     if (local_err) {
 814         error_propagate(errp, local_err);
 815         goto stop;
 816     }
 817
 818     aio_context_acquire(dataplane->ctx);
 819     blk_set_aio_context(dataplane->blk, dataplane->ctx);
 820     aio_context_release(dataplane->ctx);
 821     return;
 822
 823 stop:
 824     xen_block_dataplane_stop(dataplane);
 825 }