hw/virtio/virtio-crypto.c

   1 /*
   2  * Virtio crypto Support
   3  *
   4  * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD.
   5  *
   6  * Authors:
   7  *    Gonglei <arei.gonglei@huawei.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or
  10  * (at your option) any later version.  See the COPYING file in the
  11  * top-level directory.
  12  */
  13 #include "qemu/osdep.h"
  14 #include "qemu/iov.h"
  15 #include "hw/qdev.h"
  16 #include "qapi/error.h"
  17 #include "qemu/error-report.h"
  18
  19 #include "hw/virtio/virtio.h"
  20 #include "hw/virtio/virtio-crypto.h"
  21 #include "hw/virtio/virtio-access.h"
  22 #include "standard-headers/linux/virtio_ids.h"
  23 #include "sysemu/cryptodev-vhost.h"
  24
  25 #define VIRTIO_CRYPTO_VM_VERSION 1
  26
  27 /*
  28  * Transfer virtqueue index to crypto queue index.
  29  * The control virtqueue is after the data virtqueues
  30  * so the input value doesn't need to be adjusted
  31  */
  32 static inline int virtio_crypto_vq2q(int queue_index)
  33 {
  34     return queue_index;
  35 }
  36
  37 static int
  38 virtio_crypto_cipher_session_helper(VirtIODevice *vdev,
  39            CryptoDevBackendSymSessionInfo *info,
  40            struct virtio_crypto_cipher_session_para *cipher_para,
  41            struct iovec **iov, unsigned int *out_num)
  42 {
  43     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
  44     unsigned int num = *out_num;
  45
  46     info->cipher_alg = ldl_le_p(&cipher_para->algo);
  47     info->key_len = ldl_le_p(&cipher_para->keylen);
  48     info->direction = ldl_le_p(&cipher_para->op);
  49     DPRINTF("cipher_alg=%" PRIu32 ", info->direction=%" PRIu32 "\n",
  50              info->cipher_alg, info->direction);
  51
  52     if (info->key_len > vcrypto->conf.max_cipher_key_len) {
  53         error_report("virtio-crypto length of cipher key is too big: %u",
  54                      info->key_len);
  55         return -VIRTIO_CRYPTO_ERR;
  56     }
  57     /* Get cipher key */
  58     if (info->key_len > 0) {
  59         size_t s;
  60         DPRINTF("keylen=%" PRIu32 "\n", info->key_len);
  61
  62         info->cipher_key = g_malloc(info->key_len);
  63         s = iov_to_buf(*iov, num, 0, info->cipher_key, info->key_len);
  64         if (unlikely(s != info->key_len)) {
  65             virtio_error(vdev, "virtio-crypto cipher key incorrect");
  66             return -EFAULT;
  67         }
  68         iov_discard_front(iov, &num, info->key_len);
  69         *out_num = num;
  70     }
  71
  72     return 0;
  73 }
  74
  75 static int64_t
  76 virtio_crypto_create_sym_session(VirtIOCrypto *vcrypto,
  77                struct virtio_crypto_sym_create_session_req *sess_req,
  78                uint32_t queue_id,
  79                uint32_t opcode,
  80                struct iovec *iov, unsigned int out_num)
  81 {
  82     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
  83     CryptoDevBackendSymSessionInfo info;
  84     int64_t session_id;
  85     int queue_index;
  86     uint32_t op_type;
  87     Error *local_err = NULL;
  88     int ret;
  89
  90     memset(&info, 0, sizeof(info));
  91     op_type = ldl_le_p(&sess_req->op_type);
  92     info.op_type = op_type;
  93     info.op_code = opcode;
  94
  95     if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
  96         ret = virtio_crypto_cipher_session_helper(vdev, &info,
  97                            &sess_req->u.cipher.para,
  98                            &iov, &out_num);
  99         if (ret < 0) {
 100             goto err;
 101         }
 102     } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
 103         size_t s;
 104         /* cipher part */
 105         ret = virtio_crypto_cipher_session_helper(vdev, &info,
 106                            &sess_req->u.chain.para.cipher_param,
 107                            &iov, &out_num);
 108         if (ret < 0) {
 109             goto err;
 110         }
 111         /* hash part */
 112         info.alg_chain_order = ldl_le_p(
 113                                      &sess_req->u.chain.para.alg_chain_order);
 114         info.add_len = ldl_le_p(&sess_req->u.chain.para.aad_len);
 115         info.hash_mode = ldl_le_p(&sess_req->u.chain.para.hash_mode);
 116         if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH) {
 117             info.hash_alg = ldl_le_p(&sess_req->u.chain.para.u.mac_param.algo);
 118             info.auth_key_len = ldl_le_p(
 119                              &sess_req->u.chain.para.u.mac_param.auth_key_len);
 120             info.hash_result_len = ldl_le_p(
 121                            &sess_req->u.chain.para.u.mac_param.hash_result_len);
 122             if (info.auth_key_len > vcrypto->conf.max_auth_key_len) {
 123                 error_report("virtio-crypto length of auth key is too big: %u",
 124                              info.auth_key_len);
 125                 ret = -VIRTIO_CRYPTO_ERR;
 126                 goto err;
 127             }
 128             /* get auth key */
 129             if (info.auth_key_len > 0) {
 130                 DPRINTF("auth_keylen=%" PRIu32 "\n", info.auth_key_len);
 131                 info.auth_key = g_malloc(info.auth_key_len);
 132                 s = iov_to_buf(iov, out_num, 0, info.auth_key,
 133                                info.auth_key_len);
 134                 if (unlikely(s != info.auth_key_len)) {
 135                     virtio_error(vdev,
 136                           "virtio-crypto authenticated key incorrect");
 137                     ret = -EFAULT;
 138                     goto err;
 139                 }
 140                 iov_discard_front(&iov, &out_num, info.auth_key_len);
 141             }
 142         } else if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN) {
 143             info.hash_alg = ldl_le_p(
 144                              &sess_req->u.chain.para.u.hash_param.algo);
 145             info.hash_result_len = ldl_le_p(
 146                         &sess_req->u.chain.para.u.hash_param.hash_result_len);
 147         } else {
 148             /* VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */
 149             error_report("unsupported hash mode");
 150             ret = -VIRTIO_CRYPTO_NOTSUPP;
 151             goto err;
 152         }
 153     } else {
 154         /* VIRTIO_CRYPTO_SYM_OP_NONE */
 155         error_report("unsupported cipher op_type: VIRTIO_CRYPTO_SYM_OP_NONE");
 156         ret = -VIRTIO_CRYPTO_NOTSUPP;
 157         goto err;
 158     }
 159
 160     queue_index = virtio_crypto_vq2q(queue_id);
 161     session_id = cryptodev_backend_sym_create_session(
 162                                      vcrypto->cryptodev,
 163                                      &info, queue_index, &local_err);
 164     if (session_id >= 0) {
 165         DPRINTF("create session_id=%" PRIu64 " successfully\n",
 166                 session_id);
 167
 168         ret = session_id;
 169     } else {
 170         if (local_err) {
 171             error_report_err(local_err);
 172         }
 173         ret = -VIRTIO_CRYPTO_ERR;
 174     }
 175
 176 err:
 177     g_free(info.cipher_key);
 178     g_free(info.auth_key);
 179     return ret;
 180 }
 181
 182 static uint8_t
 183 virtio_crypto_handle_close_session(VirtIOCrypto *vcrypto,
 184          struct virtio_crypto_destroy_session_req *close_sess_req,
 185          uint32_t queue_id)
 186 {
 187     int ret;
 188     uint64_t session_id;
 189     uint32_t status;
 190     Error *local_err = NULL;
 191
 192     session_id = ldq_le_p(&close_sess_req->session_id);
 193     DPRINTF("close session, id=%" PRIu64 "\n", session_id);
 194
 195     ret = cryptodev_backend_sym_close_session(
 196               vcrypto->cryptodev, session_id, queue_id, &local_err);
 197     if (ret == 0) {
 198         status = VIRTIO_CRYPTO_OK;
 199     } else {
 200         if (local_err) {
 201             error_report_err(local_err);
 202         } else {
 203             error_report("destroy session failed");
 204         }
 205         status = VIRTIO_CRYPTO_ERR;
 206     }
 207
 208     return status;
 209 }
 210
 211 static void virtio_crypto_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
 212 {
 213     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 214     struct virtio_crypto_op_ctrl_req ctrl;
 215     VirtQueueElement *elem;
 216     struct iovec *in_iov;
 217     struct iovec *out_iov;
 218     unsigned in_num;
 219     unsigned out_num;
 220     uint32_t queue_id;
 221     uint32_t opcode;
 222     struct virtio_crypto_session_input input;
 223     int64_t session_id;
 224     uint8_t status;
 225     size_t s;
 226
 227     for (;;) {
 228         elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
 229         if (!elem) {
 230             break;
 231         }
 232         if (elem->out_num < 1 || elem->in_num < 1) {
 233             virtio_error(vdev, "virtio-crypto ctrl missing headers");
 234             virtqueue_detach_element(vq, elem, 0);
 235             g_free(elem);
 236             break;
 237         }
 238
 239         out_num = elem->out_num;
 240         out_iov = elem->out_sg;
 241         in_num = elem->in_num;
 242         in_iov = elem->in_sg;
 243         if (unlikely(iov_to_buf(out_iov, out_num, 0, &ctrl, sizeof(ctrl))
 244                     != sizeof(ctrl))) {
 245             virtio_error(vdev, "virtio-crypto request ctrl_hdr too short");
 246             virtqueue_detach_element(vq, elem, 0);
 247             g_free(elem);
 248             break;
 249         }
 250         iov_discard_front(&out_iov, &out_num, sizeof(ctrl));
 251
 252         opcode = ldl_le_p(&ctrl.header.opcode);
 253         queue_id = ldl_le_p(&ctrl.header.queue_id);
 254
 255         switch (opcode) {
 256         case VIRTIO_CRYPTO_CIPHER_CREATE_SESSION:
 257             memset(&input, 0, sizeof(input));
 258             session_id = virtio_crypto_create_sym_session(vcrypto,
 259                              &ctrl.u.sym_create_session,
 260                              queue_id, opcode,
 261                              out_iov, out_num);
 262             /* Serious errors, need to reset virtio crypto device */
 263             if (session_id == -EFAULT) {
 264                 virtqueue_detach_element(vq, elem, 0);
 265                 break;
 266             } else if (session_id == -VIRTIO_CRYPTO_NOTSUPP) {
 267                 stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP);
 268             } else if (session_id == -VIRTIO_CRYPTO_ERR) {
 269                 stl_le_p(&input.status, VIRTIO_CRYPTO_ERR);
 270             } else {
 271                 /* Set the session id */
 272                 stq_le_p(&input.session_id, session_id);
 273                 stl_le_p(&input.status, VIRTIO_CRYPTO_OK);
 274             }
 275
 276             s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input));
 277             if (unlikely(s != sizeof(input))) {
 278                 virtio_error(vdev, "virtio-crypto input incorrect");
 279                 virtqueue_detach_element(vq, elem, 0);
 280                 break;
 281             }
 282             virtqueue_push(vq, elem, sizeof(input));
 283             virtio_notify(vdev, vq);
 284             break;
 285         case VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION:
 286         case VIRTIO_CRYPTO_HASH_DESTROY_SESSION:
 287         case VIRTIO_CRYPTO_MAC_DESTROY_SESSION:
 288         case VIRTIO_CRYPTO_AEAD_DESTROY_SESSION:
 289             status = virtio_crypto_handle_close_session(vcrypto,
 290                    &ctrl.u.destroy_session, queue_id);
 291             /* The status only occupy one byte, we can directly use it */
 292             s = iov_from_buf(in_iov, in_num, 0, &status, sizeof(status));
 293             if (unlikely(s != sizeof(status))) {
 294                 virtio_error(vdev, "virtio-crypto status incorrect");
 295                 virtqueue_detach_element(vq, elem, 0);
 296                 break;
 297             }
 298             virtqueue_push(vq, elem, sizeof(status));
 299             virtio_notify(vdev, vq);
 300             break;
 301         case VIRTIO_CRYPTO_HASH_CREATE_SESSION:
 302         case VIRTIO_CRYPTO_MAC_CREATE_SESSION:
 303         case VIRTIO_CRYPTO_AEAD_CREATE_SESSION:
 304         default:
 305             error_report("virtio-crypto unsupported ctrl opcode: %d", opcode);
 306             memset(&input, 0, sizeof(input));
 307             stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP);
 308             s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input));
 309             if (unlikely(s != sizeof(input))) {
 310                 virtio_error(vdev, "virtio-crypto input incorrect");
 311                 virtqueue_detach_element(vq, elem, 0);
 312                 break;
 313             }
 314             virtqueue_push(vq, elem, sizeof(input));
 315             virtio_notify(vdev, vq);
 316
 317             break;
 318         } /* end switch case */
 319
 320         g_free(elem);
 321     } /* end for loop */
 322 }
 323
 324 static void virtio_crypto_init_request(VirtIOCrypto *vcrypto, VirtQueue *vq,
 325                                 VirtIOCryptoReq *req)
 326 {
 327     req->vcrypto = vcrypto;
 328     req->vq = vq;
 329     req->in = NULL;
 330     req->in_iov = NULL;
 331     req->in_num = 0;
 332     req->in_len = 0;
 333     req->flags = CRYPTODEV_BACKEND_ALG__MAX;
 334     req->u.sym_op_info = NULL;
 335 }
 336
 337 static void virtio_crypto_free_request(VirtIOCryptoReq *req)
 338 {
 339     if (req) {
 340         if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) {
 341             size_t max_len;
 342             CryptoDevBackendSymOpInfo *op_info = req->u.sym_op_info;
 343
 344             max_len = op_info->iv_len +
 345                       op_info->aad_len +
 346                       op_info->src_len +
 347                       op_info->dst_len +
 348                       op_info->digest_result_len;
 349
 350             /* Zeroize and free request data structure */
 351             memset(op_info, 0, sizeof(*op_info) + max_len);
 352             g_free(op_info);
 353         }
 354         g_free(req);
 355     }
 356 }
 357
 358 static void
 359 virtio_crypto_sym_input_data_helper(VirtIODevice *vdev,
 360                 VirtIOCryptoReq *req,
 361                 uint32_t status,
 362                 CryptoDevBackendSymOpInfo *sym_op_info)
 363 {
 364     size_t s, len;
 365
 366     if (status != VIRTIO_CRYPTO_OK) {
 367         return;
 368     }
 369
 370     len = sym_op_info->src_len;
 371     /* Save the cipher result */
 372     s = iov_from_buf(req->in_iov, req->in_num, 0, sym_op_info->dst, len);
 373     if (s != len) {
 374         virtio_error(vdev, "virtio-crypto dest data incorrect");
 375         return;
 376     }
 377
 378     iov_discard_front(&req->in_iov, &req->in_num, len);
 379
 380     if (sym_op_info->op_type ==
 381                       VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
 382         /* Save the digest result */
 383         s = iov_from_buf(req->in_iov, req->in_num, 0,
 384                          sym_op_info->digest_result,
 385                          sym_op_info->digest_result_len);
 386         if (s != sym_op_info->digest_result_len) {
 387             virtio_error(vdev, "virtio-crypto digest result incorrect");
 388         }
 389     }
 390 }
 391
 392 static void virtio_crypto_req_complete(VirtIOCryptoReq *req, uint8_t status)
 393 {
 394     VirtIOCrypto *vcrypto = req->vcrypto;
 395     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 396
 397     if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) {
 398         virtio_crypto_sym_input_data_helper(vdev, req, status,
 399                                             req->u.sym_op_info);
 400     }
 401     stb_p(&req->in->status, status);
 402     virtqueue_push(req->vq, &req->elem, req->in_len);
 403     virtio_notify(vdev, req->vq);
 404 }
 405
 406 static VirtIOCryptoReq *
 407 virtio_crypto_get_request(VirtIOCrypto *s, VirtQueue *vq)
 408 {
 409     VirtIOCryptoReq *req = virtqueue_pop(vq, sizeof(VirtIOCryptoReq));
 410
 411     if (req) {
 412         virtio_crypto_init_request(s, vq, req);
 413     }
 414     return req;
 415 }
 416
 417 static CryptoDevBackendSymOpInfo *
 418 virtio_crypto_sym_op_helper(VirtIODevice *vdev,
 419            struct virtio_crypto_cipher_para *cipher_para,
 420            struct virtio_crypto_alg_chain_data_para *alg_chain_para,
 421            struct iovec *iov, unsigned int out_num)
 422 {
 423     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 424     CryptoDevBackendSymOpInfo *op_info;
 425     uint32_t src_len = 0, dst_len = 0;
 426     uint32_t iv_len = 0;
 427     uint32_t aad_len = 0, hash_result_len = 0;
 428     uint32_t hash_start_src_offset = 0, len_to_hash = 0;
 429     uint32_t cipher_start_src_offset = 0, len_to_cipher = 0;
 430
 431     uint64_t max_len, curr_size = 0;
 432     size_t s;
 433
 434     /* Plain cipher */
 435     if (cipher_para) {
 436         iv_len = ldl_le_p(&cipher_para->iv_len);
 437         src_len = ldl_le_p(&cipher_para->src_data_len);
 438         dst_len = ldl_le_p(&cipher_para->dst_data_len);
 439     } else if (alg_chain_para) { /* Algorithm chain */
 440         iv_len = ldl_le_p(&alg_chain_para->iv_len);
 441         src_len = ldl_le_p(&alg_chain_para->src_data_len);
 442         dst_len = ldl_le_p(&alg_chain_para->dst_data_len);
 443
 444         aad_len = ldl_le_p(&alg_chain_para->aad_len);
 445         hash_result_len = ldl_le_p(&alg_chain_para->hash_result_len);
 446         hash_start_src_offset = ldl_le_p(
 447                          &alg_chain_para->hash_start_src_offset);
 448         cipher_start_src_offset = ldl_le_p(
 449                          &alg_chain_para->cipher_start_src_offset);
 450         len_to_cipher = ldl_le_p(&alg_chain_para->len_to_cipher);
 451         len_to_hash = ldl_le_p(&alg_chain_para->len_to_hash);
 452     } else {
 453         return NULL;
 454     }
 455
 456     max_len = (uint64_t)iv_len + aad_len + src_len + dst_len + hash_result_len;
 457     if (unlikely(max_len > vcrypto->conf.max_size)) {
 458         virtio_error(vdev, "virtio-crypto too big length");
 459         return NULL;
 460     }
 461
 462     op_info = g_malloc0(sizeof(CryptoDevBackendSymOpInfo) + max_len);
 463     op_info->iv_len = iv_len;
 464     op_info->src_len = src_len;
 465     op_info->dst_len = dst_len;
 466     op_info->aad_len = aad_len;
 467     op_info->digest_result_len = hash_result_len;
 468     op_info->hash_start_src_offset = hash_start_src_offset;
 469     op_info->len_to_hash = len_to_hash;
 470     op_info->cipher_start_src_offset = cipher_start_src_offset;
 471     op_info->len_to_cipher = len_to_cipher;
 472     /* Handle the initilization vector */
 473     if (op_info->iv_len > 0) {
 474         DPRINTF("iv_len=%" PRIu32 "\n", op_info->iv_len);
 475         op_info->iv = op_info->data + curr_size;
 476
 477         s = iov_to_buf(iov, out_num, 0, op_info->iv, op_info->iv_len);
 478         if (unlikely(s != op_info->iv_len)) {
 479             virtio_error(vdev, "virtio-crypto iv incorrect");
 480             goto err;
 481         }
 482         iov_discard_front(&iov, &out_num, op_info->iv_len);
 483         curr_size += op_info->iv_len;
 484     }
 485
 486     /* Handle additional authentication data if exists */
 487     if (op_info->aad_len > 0) {
 488         DPRINTF("aad_len=%" PRIu32 "\n", op_info->aad_len);
 489         op_info->aad_data = op_info->data + curr_size;
 490
 491         s = iov_to_buf(iov, out_num, 0, op_info->aad_data, op_info->aad_len);
 492         if (unlikely(s != op_info->aad_len)) {
 493             virtio_error(vdev, "virtio-crypto additional auth data incorrect");
 494             goto err;
 495         }
 496         iov_discard_front(&iov, &out_num, op_info->aad_len);
 497
 498         curr_size += op_info->aad_len;
 499     }
 500
 501     /* Handle the source data */
 502     if (op_info->src_len > 0) {
 503         DPRINTF("src_len=%" PRIu32 "\n", op_info->src_len);
 504         op_info->src = op_info->data + curr_size;
 505
 506         s = iov_to_buf(iov, out_num, 0, op_info->src, op_info->src_len);
 507         if (unlikely(s != op_info->src_len)) {
 508             virtio_error(vdev, "virtio-crypto source data incorrect");
 509             goto err;
 510         }
 511         iov_discard_front(&iov, &out_num, op_info->src_len);
 512
 513         curr_size += op_info->src_len;
 514     }
 515
 516     /* Handle the destination data */
 517     op_info->dst = op_info->data + curr_size;
 518     curr_size += op_info->dst_len;
 519
 520     DPRINTF("dst_len=%" PRIu32 "\n", op_info->dst_len);
 521
 522     /* Handle the hash digest result */
 523     if (hash_result_len > 0) {
 524         DPRINTF("hash_result_len=%" PRIu32 "\n", hash_result_len);
 525         op_info->digest_result = op_info->data + curr_size;
 526     }
 527
 528     return op_info;
 529
 530 err:
 531     g_free(op_info);
 532     return NULL;
 533 }
 534
 535 static int
 536 virtio_crypto_handle_sym_req(VirtIOCrypto *vcrypto,
 537                struct virtio_crypto_sym_data_req *req,
 538                CryptoDevBackendSymOpInfo **sym_op_info,
 539                struct iovec *iov, unsigned int out_num)
 540 {
 541     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 542     uint32_t op_type;
 543     CryptoDevBackendSymOpInfo *op_info;
 544
 545     op_type = ldl_le_p(&req->op_type);
 546
 547     if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
 548         op_info = virtio_crypto_sym_op_helper(vdev, &req->u.cipher.para,
 549                                               NULL, iov, out_num);
 550         if (!op_info) {
 551             return -EFAULT;
 552         }
 553         op_info->op_type = op_type;
 554     } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
 555         op_info = virtio_crypto_sym_op_helper(vdev, NULL,
 556                                               &req->u.chain.para,
 557                                               iov, out_num);
 558         if (!op_info) {
 559             return -EFAULT;
 560         }
 561         op_info->op_type = op_type;
 562     } else {
 563         /* VIRTIO_CRYPTO_SYM_OP_NONE */
 564         error_report("virtio-crypto unsupported cipher type");
 565         return -VIRTIO_CRYPTO_NOTSUPP;
 566     }
 567
 568     *sym_op_info = op_info;
 569
 570     return 0;
 571 }
 572
 573 static int
 574 virtio_crypto_handle_request(VirtIOCryptoReq *request)
 575 {
 576     VirtIOCrypto *vcrypto = request->vcrypto;
 577     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 578     VirtQueueElement *elem = &request->elem;
 579     int queue_index = virtio_crypto_vq2q(virtio_get_queue_index(request->vq));
 580     struct virtio_crypto_op_data_req req;
 581     int ret;
 582     struct iovec *in_iov;
 583     struct iovec *out_iov;
 584     unsigned in_num;
 585     unsigned out_num;
 586     uint32_t opcode;
 587     uint8_t status = VIRTIO_CRYPTO_ERR;
 588     uint64_t session_id;
 589     CryptoDevBackendSymOpInfo *sym_op_info = NULL;
 590     Error *local_err = NULL;
 591
 592     if (elem->out_num < 1 || elem->in_num < 1) {
 593         virtio_error(vdev, "virtio-crypto dataq missing headers");
 594         return -1;
 595     }
 596
 597     out_num = elem->out_num;
 598     out_iov = elem->out_sg;
 599     in_num = elem->in_num;
 600     in_iov = elem->in_sg;
 601     if (unlikely(iov_to_buf(out_iov, out_num, 0, &req, sizeof(req))
 602                 != sizeof(req))) {
 603         virtio_error(vdev, "virtio-crypto request outhdr too short");
 604         return -1;
 605     }
 606     iov_discard_front(&out_iov, &out_num, sizeof(req));
 607
 608     if (in_iov[in_num - 1].iov_len <
 609             sizeof(struct virtio_crypto_inhdr)) {
 610         virtio_error(vdev, "virtio-crypto request inhdr too short");
 611         return -1;
 612     }
 613     /* We always touch the last byte, so just see how big in_iov is. */
 614     request->in_len = iov_size(in_iov, in_num);
 615     request->in = (void *)in_iov[in_num - 1].iov_base
 616               + in_iov[in_num - 1].iov_len
 617               - sizeof(struct virtio_crypto_inhdr);
 618     iov_discard_back(in_iov, &in_num, sizeof(struct virtio_crypto_inhdr));
 619
 620     /*
 621      * The length of operation result, including dest_data
 622      * and digest_result if exists.
 623      */
 624     request->in_num = in_num;
 625     request->in_iov = in_iov;
 626
 627     opcode = ldl_le_p(&req.header.opcode);
 628     session_id = ldq_le_p(&req.header.session_id);
 629
 630     switch (opcode) {
 631     case VIRTIO_CRYPTO_CIPHER_ENCRYPT:
 632     case VIRTIO_CRYPTO_CIPHER_DECRYPT:
 633         ret = virtio_crypto_handle_sym_req(vcrypto,
 634                          &req.u.sym_req,
 635                          &sym_op_info,
 636                          out_iov, out_num);
 637         /* Serious errors, need to reset virtio crypto device */
 638         if (ret == -EFAULT) {
 639             return -1;
 640         } else if (ret == -VIRTIO_CRYPTO_NOTSUPP) {
 641             virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP);
 642             virtio_crypto_free_request(request);
 643         } else {
 644             sym_op_info->session_id = session_id;
 645
 646             /* Set request's parameter */
 647             request->flags = CRYPTODEV_BACKEND_ALG_SYM;
 648             request->u.sym_op_info = sym_op_info;
 649             ret = cryptodev_backend_crypto_operation(vcrypto->cryptodev,
 650                                     request, queue_index, &local_err);
 651             if (ret < 0) {
 652                 status = -ret;
 653                 if (local_err) {
 654                     error_report_err(local_err);
 655                 }
 656             } else { /* ret == VIRTIO_CRYPTO_OK */
 657                 status = ret;
 658             }
 659             virtio_crypto_req_complete(request, status);
 660             virtio_crypto_free_request(request);
 661         }
 662         break;
 663     case VIRTIO_CRYPTO_HASH:
 664     case VIRTIO_CRYPTO_MAC:
 665     case VIRTIO_CRYPTO_AEAD_ENCRYPT:
 666     case VIRTIO_CRYPTO_AEAD_DECRYPT:
 667     default:
 668         error_report("virtio-crypto unsupported dataq opcode: %u",
 669                      opcode);
 670         virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP);
 671         virtio_crypto_free_request(request);
 672     }
 673
 674     return 0;
 675 }
 676
 677 static void virtio_crypto_handle_dataq(VirtIODevice *vdev, VirtQueue *vq)
 678 {
 679     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 680     VirtIOCryptoReq *req;
 681
 682     while ((req = virtio_crypto_get_request(vcrypto, vq))) {
 683         if (virtio_crypto_handle_request(req) < 0) {
 684             virtqueue_detach_element(req->vq, &req->elem, 0);
 685             virtio_crypto_free_request(req);
 686             break;
 687         }
 688     }
 689 }
 690
 691 static void virtio_crypto_dataq_bh(void *opaque)
 692 {
 693     VirtIOCryptoQueue *q = opaque;
 694     VirtIOCrypto *vcrypto = q->vcrypto;
 695     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 696
 697     /* This happens when device was stopped but BH wasn't. */
 698     if (!vdev->vm_running) {
 699         return;
 700     }
 701
 702     /* Just in case the driver is not ready on more */
 703     if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) {
 704         return;
 705     }
 706
 707     for (;;) {
 708         virtio_crypto_handle_dataq(vdev, q->dataq);
 709         virtio_queue_set_notification(q->dataq, 1);
 710
 711         /* Are we done or did the guest add more buffers? */
 712         if (virtio_queue_empty(q->dataq)) {
 713             break;
 714         }
 715
 716         virtio_queue_set_notification(q->dataq, 0);
 717     }
 718 }
 719
 720 static void
 721 virtio_crypto_handle_dataq_bh(VirtIODevice *vdev, VirtQueue *vq)
 722 {
 723     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 724     VirtIOCryptoQueue *q =
 725          &vcrypto->vqs[virtio_crypto_vq2q(virtio_get_queue_index(vq))];
 726
 727     /* This happens when device was stopped but VCPU wasn't. */
 728     if (!vdev->vm_running) {
 729         return;
 730     }
 731     virtio_queue_set_notification(vq, 0);
 732     qemu_bh_schedule(q->dataq_bh);
 733 }
 734
 735 static uint64_t virtio_crypto_get_features(VirtIODevice *vdev,
 736                                            uint64_t features,
 737                                            Error **errp)
 738 {
 739     return features;
 740 }
 741
 742 static void virtio_crypto_reset(VirtIODevice *vdev)
 743 {
 744     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 745     /* multiqueue is disabled by default */
 746     vcrypto->curr_queues = 1;
 747     if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) {
 748         vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY;
 749     } else {
 750         vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY;
 751     }
 752 }
 753
 754 static void virtio_crypto_init_config(VirtIODevice *vdev)
 755 {
 756     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 757
 758     vcrypto->conf.crypto_services =
 759                      vcrypto->conf.cryptodev->conf.crypto_services;
 760     vcrypto->conf.cipher_algo_l =
 761                      vcrypto->conf.cryptodev->conf.cipher_algo_l;
 762     vcrypto->conf.cipher_algo_h =
 763                      vcrypto->conf.cryptodev->conf.cipher_algo_h;
 764     vcrypto->conf.hash_algo = vcrypto->conf.cryptodev->conf.hash_algo;
 765     vcrypto->conf.mac_algo_l = vcrypto->conf.cryptodev->conf.mac_algo_l;
 766     vcrypto->conf.mac_algo_h = vcrypto->conf.cryptodev->conf.mac_algo_h;
 767     vcrypto->conf.aead_algo = vcrypto->conf.cryptodev->conf.aead_algo;
 768     vcrypto->conf.max_cipher_key_len =
 769                   vcrypto->conf.cryptodev->conf.max_cipher_key_len;
 770     vcrypto->conf.max_auth_key_len =
 771                   vcrypto->conf.cryptodev->conf.max_auth_key_len;
 772     vcrypto->conf.max_size = vcrypto->conf.cryptodev->conf.max_size;
 773 }
 774
 775 static void virtio_crypto_device_realize(DeviceState *dev, Error **errp)
 776 {
 777     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 778     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev);
 779     int i;
 780
 781     vcrypto->cryptodev = vcrypto->conf.cryptodev;
 782     if (vcrypto->cryptodev == NULL) {
 783         error_setg(errp, "'cryptodev' parameter expects a valid object");
 784         return;
 785     } else if (cryptodev_backend_is_used(vcrypto->cryptodev)) {
 786         char *path = object_get_canonical_path_component(OBJECT(vcrypto->conf.cryptodev));
 787         error_setg(errp, "can't use already used cryptodev backend: %s", path);
 788         g_free(path);
 789         return;
 790     }
 791
 792     vcrypto->max_queues = MAX(vcrypto->cryptodev->conf.peers.queues, 1);
 793     if (vcrypto->max_queues + 1 > VIRTIO_QUEUE_MAX) {
 794         error_setg(errp, "Invalid number of queues (= %" PRIu32 "), "
 795                    "must be a positive integer less than %d.",
 796                    vcrypto->max_queues, VIRTIO_QUEUE_MAX);
 797         return;
 798     }
 799
 800     virtio_init(vdev, "virtio-crypto", VIRTIO_ID_CRYPTO, vcrypto->config_size);
 801     vcrypto->curr_queues = 1;
 802     vcrypto->vqs = g_malloc0(sizeof(VirtIOCryptoQueue) * vcrypto->max_queues);
 803     for (i = 0; i < vcrypto->max_queues; i++) {
 804         vcrypto->vqs[i].dataq =
 805                  virtio_add_queue(vdev, 1024, virtio_crypto_handle_dataq_bh);
 806         vcrypto->vqs[i].dataq_bh =
 807                  qemu_bh_new(virtio_crypto_dataq_bh, &vcrypto->vqs[i]);
 808         vcrypto->vqs[i].vcrypto = vcrypto;
 809     }
 810
 811     vcrypto->ctrl_vq = virtio_add_queue(vdev, 64, virtio_crypto_handle_ctrl);
 812     if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) {
 813         vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY;
 814     } else {
 815         vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY;
 816     }
 817
 818     virtio_crypto_init_config(vdev);
 819     cryptodev_backend_set_used(vcrypto->cryptodev, true);
 820 }
 821
 822 static void virtio_crypto_device_unrealize(DeviceState *dev, Error **errp)
 823 {
 824     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 825     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev);
 826     VirtIOCryptoQueue *q;
 827     int i, max_queues;
 828
 829     max_queues = vcrypto->multiqueue ? vcrypto->max_queues : 1;
 830     for (i = 0; i < max_queues; i++) {
 831         virtio_del_queue(vdev, i);
 832         q = &vcrypto->vqs[i];
 833         qemu_bh_delete(q->dataq_bh);
 834     }
 835
 836     g_free(vcrypto->vqs);
 837
 838     virtio_cleanup(vdev);
 839     cryptodev_backend_set_used(vcrypto->cryptodev, false);
 840 }
 841
 842 static const VMStateDescription vmstate_virtio_crypto = {
 843     .name = "virtio-crypto",
 844     .unmigratable = 1,
 845     .minimum_version_id = VIRTIO_CRYPTO_VM_VERSION,
 846     .version_id = VIRTIO_CRYPTO_VM_VERSION,
 847     .fields = (VMStateField[]) {
 848         VMSTATE_VIRTIO_DEVICE,
 849         VMSTATE_END_OF_LIST()
 850     },
 851 };
 852
 853 static Property virtio_crypto_properties[] = {
 854     DEFINE_PROP_LINK("cryptodev", VirtIOCrypto, conf.cryptodev,
 855                      TYPE_CRYPTODEV_BACKEND, CryptoDevBackend *),
 856     DEFINE_PROP_END_OF_LIST(),
 857 };
 858
 859 static void virtio_crypto_get_config(VirtIODevice *vdev, uint8_t *config)
 860 {
 861     VirtIOCrypto *c = VIRTIO_CRYPTO(vdev);
 862     struct virtio_crypto_config crypto_cfg = {};
 863
 864     /*
 865      * Virtio-crypto device conforms to VIRTIO 1.0 which is always LE,
 866      * so we can use LE accessors directly.
 867      */
 868     stl_le_p(&crypto_cfg.status, c->status);
 869     stl_le_p(&crypto_cfg.max_dataqueues, c->max_queues);
 870     stl_le_p(&crypto_cfg.crypto_services, c->conf.crypto_services);
 871     stl_le_p(&crypto_cfg.cipher_algo_l, c->conf.cipher_algo_l);
 872     stl_le_p(&crypto_cfg.cipher_algo_h, c->conf.cipher_algo_h);
 873     stl_le_p(&crypto_cfg.hash_algo, c->conf.hash_algo);
 874     stl_le_p(&crypto_cfg.mac_algo_l, c->conf.mac_algo_l);
 875     stl_le_p(&crypto_cfg.mac_algo_h, c->conf.mac_algo_h);
 876     stl_le_p(&crypto_cfg.aead_algo, c->conf.aead_algo);
 877     stl_le_p(&crypto_cfg.max_cipher_key_len, c->conf.max_cipher_key_len);
 878     stl_le_p(&crypto_cfg.max_auth_key_len, c->conf.max_auth_key_len);
 879     stq_le_p(&crypto_cfg.max_size, c->conf.max_size);
 880
 881     memcpy(config, &crypto_cfg, c->config_size);
 882 }
 883
 884 static bool virtio_crypto_started(VirtIOCrypto *c, uint8_t status)
 885 {
 886     VirtIODevice *vdev = VIRTIO_DEVICE(c);
 887     return (status & VIRTIO_CONFIG_S_DRIVER_OK) &&
 888         (c->status & VIRTIO_CRYPTO_S_HW_READY) && vdev->vm_running;
 889 }
 890
 891 static void virtio_crypto_vhost_status(VirtIOCrypto *c, uint8_t status)
 892 {
 893     VirtIODevice *vdev = VIRTIO_DEVICE(c);
 894     int queues = c->multiqueue ? c->max_queues : 1;
 895     CryptoDevBackend *b = c->cryptodev;
 896     CryptoDevBackendClient *cc = b->conf.peers.ccs[0];
 897
 898     if (!cryptodev_get_vhost(cc, b, 0)) {
 899         return;
 900     }
 901
 902     if ((virtio_crypto_started(c, status)) == !!c->vhost_started) {
 903         return;
 904     }
 905
 906     if (!c->vhost_started) {
 907         int r;
 908
 909         c->vhost_started = 1;
 910         r = cryptodev_vhost_start(vdev, queues);
 911         if (r < 0) {
 912             error_report("unable to start vhost crypto: %d: "
 913                          "falling back on userspace virtio", -r);
 914             c->vhost_started = 0;
 915         }
 916     } else {
 917         cryptodev_vhost_stop(vdev, queues);
 918         c->vhost_started = 0;
 919     }
 920 }
 921
 922 static void virtio_crypto_set_status(VirtIODevice *vdev, uint8_t status)
 923 {
 924     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 925
 926     virtio_crypto_vhost_status(vcrypto, status);
 927 }
 928
 929 static void virtio_crypto_guest_notifier_mask(VirtIODevice *vdev, int idx,
 930                                            bool mask)
 931 {
 932     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 933     int queue = virtio_crypto_vq2q(idx);
 934
 935     assert(vcrypto->vhost_started);
 936
 937     cryptodev_vhost_virtqueue_mask(vdev, queue, idx, mask);
 938 }
 939
 940 static bool virtio_crypto_guest_notifier_pending(VirtIODevice *vdev, int idx)
 941 {
 942     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 943     int queue = virtio_crypto_vq2q(idx);
 944
 945     assert(vcrypto->vhost_started);
 946
 947     return cryptodev_vhost_virtqueue_pending(vdev, queue, idx);
 948 }
 949
 950 static void virtio_crypto_class_init(ObjectClass *klass, void *data)
 951 {
 952     DeviceClass *dc = DEVICE_CLASS(klass);
 953     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
 954
 955     dc->props = virtio_crypto_properties;
 956     dc->vmsd = &vmstate_virtio_crypto;
 957     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 958     vdc->realize = virtio_crypto_device_realize;
 959     vdc->unrealize = virtio_crypto_device_unrealize;
 960     vdc->get_config = virtio_crypto_get_config;
 961     vdc->get_features = virtio_crypto_get_features;
 962     vdc->reset = virtio_crypto_reset;
 963     vdc->set_status = virtio_crypto_set_status;
 964     vdc->guest_notifier_mask = virtio_crypto_guest_notifier_mask;
 965     vdc->guest_notifier_pending = virtio_crypto_guest_notifier_pending;
 966 }
 967
 968 static void virtio_crypto_instance_init(Object *obj)
 969 {
 970     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(obj);
 971
 972     /*
 973      * The default config_size is sizeof(struct virtio_crypto_config).
 974      * Can be overriden with virtio_crypto_set_config_size.
 975      */
 976     vcrypto->config_size = sizeof(struct virtio_crypto_config);
 977 }
 978
 979 static const TypeInfo virtio_crypto_info = {
 980     .name = TYPE_VIRTIO_CRYPTO,
 981     .parent = TYPE_VIRTIO_DEVICE,
 982     .instance_size = sizeof(VirtIOCrypto),
 983     .instance_init = virtio_crypto_instance_init,
 984     .class_init = virtio_crypto_class_init,
 985 };
 986
 987 static void virtio_register_types(void)
 988 {
 989     type_register_static(&virtio_crypto_info);
 990 }
 991
 992 type_init(virtio_register_types)