hw/virtio/virtio-crypto.c

   1 /*
   2  * Virtio crypto Support
   3  *
   4  * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD.
   5  *
   6  * Authors:
   7  *    Gonglei <arei.gonglei@huawei.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or
  10  * (at your option) any later version.  See the COPYING file in the
  11  * top-level directory.
  12  */
  13
  14 #include "qemu/osdep.h"
  15 #include "qemu/iov.h"
  16 #include "qemu/main-loop.h"
  17 #include "qemu/module.h"
  18 #include "qapi/error.h"
  19 #include "qemu/error-report.h"
  20
  21 #include "hw/virtio/virtio.h"
  22 #include "hw/virtio/virtio-crypto.h"
  23 #include "hw/qdev-properties.h"
  24 #include "hw/virtio/virtio-access.h"
  25 #include "standard-headers/linux/virtio_ids.h"
  26 #include "sysemu/cryptodev-vhost.h"
  27
  28 #define VIRTIO_CRYPTO_VM_VERSION 1
  29
  30 /*
  31  * Transfer virtqueue index to crypto queue index.
  32  * The control virtqueue is after the data virtqueues
  33  * so the input value doesn't need to be adjusted
  34  */
  35 static inline int virtio_crypto_vq2q(int queue_index)
  36 {
  37     return queue_index;
  38 }
  39
  40 static int
  41 virtio_crypto_cipher_session_helper(VirtIODevice *vdev,
  42            CryptoDevBackendSymSessionInfo *info,
  43            struct virtio_crypto_cipher_session_para *cipher_para,
  44            struct iovec **iov, unsigned int *out_num)
  45 {
  46     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
  47     unsigned int num = *out_num;
  48
  49     info->cipher_alg = ldl_le_p(&cipher_para->algo);
  50     info->key_len = ldl_le_p(&cipher_para->keylen);
  51     info->direction = ldl_le_p(&cipher_para->op);
  52     DPRINTF("cipher_alg=%" PRIu32 ", info->direction=%" PRIu32 "\n",
  53              info->cipher_alg, info->direction);
  54
  55     if (info->key_len > vcrypto->conf.max_cipher_key_len) {
  56         error_report("virtio-crypto length of cipher key is too big: %u",
  57                      info->key_len);
  58         return -VIRTIO_CRYPTO_ERR;
  59     }
  60     /* Get cipher key */
  61     if (info->key_len > 0) {
  62         size_t s;
  63         DPRINTF("keylen=%" PRIu32 "\n", info->key_len);
  64
  65         info->cipher_key = g_malloc(info->key_len);
  66         s = iov_to_buf(*iov, num, 0, info->cipher_key, info->key_len);
  67         if (unlikely(s != info->key_len)) {
  68             virtio_error(vdev, "virtio-crypto cipher key incorrect");
  69             return -EFAULT;
  70         }
  71         iov_discard_front(iov, &num, info->key_len);
  72         *out_num = num;
  73     }
  74
  75     return 0;
  76 }
  77
  78 static int64_t
  79 virtio_crypto_create_sym_session(VirtIOCrypto *vcrypto,
  80                struct virtio_crypto_sym_create_session_req *sess_req,
  81                uint32_t queue_id,
  82                uint32_t opcode,
  83                struct iovec *iov, unsigned int out_num)
  84 {
  85     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
  86     CryptoDevBackendSymSessionInfo info;
  87     int64_t session_id;
  88     int queue_index;
  89     uint32_t op_type;
  90     Error *local_err = NULL;
  91     int ret;
  92
  93     memset(&info, 0, sizeof(info));
  94     op_type = ldl_le_p(&sess_req->op_type);
  95     info.op_type = op_type;
  96     info.op_code = opcode;
  97
  98     if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
  99         ret = virtio_crypto_cipher_session_helper(vdev, &info,
 100                            &sess_req->u.cipher.para,
 101                            &iov, &out_num);
 102         if (ret < 0) {
 103             goto err;
 104         }
 105     } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
 106         size_t s;
 107         /* cipher part */
 108         ret = virtio_crypto_cipher_session_helper(vdev, &info,
 109                            &sess_req->u.chain.para.cipher_param,
 110                            &iov, &out_num);
 111         if (ret < 0) {
 112             goto err;
 113         }
 114         /* hash part */
 115         info.alg_chain_order = ldl_le_p(
 116                                      &sess_req->u.chain.para.alg_chain_order);
 117         info.add_len = ldl_le_p(&sess_req->u.chain.para.aad_len);
 118         info.hash_mode = ldl_le_p(&sess_req->u.chain.para.hash_mode);
 119         if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH) {
 120             info.hash_alg = ldl_le_p(&sess_req->u.chain.para.u.mac_param.algo);
 121             info.auth_key_len = ldl_le_p(
 122                              &sess_req->u.chain.para.u.mac_param.auth_key_len);
 123             info.hash_result_len = ldl_le_p(
 124                            &sess_req->u.chain.para.u.mac_param.hash_result_len);
 125             if (info.auth_key_len > vcrypto->conf.max_auth_key_len) {
 126                 error_report("virtio-crypto length of auth key is too big: %u",
 127                              info.auth_key_len);
 128                 ret = -VIRTIO_CRYPTO_ERR;
 129                 goto err;
 130             }
 131             /* get auth key */
 132             if (info.auth_key_len > 0) {
 133                 DPRINTF("auth_keylen=%" PRIu32 "\n", info.auth_key_len);
 134                 info.auth_key = g_malloc(info.auth_key_len);
 135                 s = iov_to_buf(iov, out_num, 0, info.auth_key,
 136                                info.auth_key_len);
 137                 if (unlikely(s != info.auth_key_len)) {
 138                     virtio_error(vdev,
 139                           "virtio-crypto authenticated key incorrect");
 140                     ret = -EFAULT;
 141                     goto err;
 142                 }
 143                 iov_discard_front(&iov, &out_num, info.auth_key_len);
 144             }
 145         } else if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN) {
 146             info.hash_alg = ldl_le_p(
 147                              &sess_req->u.chain.para.u.hash_param.algo);
 148             info.hash_result_len = ldl_le_p(
 149                         &sess_req->u.chain.para.u.hash_param.hash_result_len);
 150         } else {
 151             /* VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */
 152             error_report("unsupported hash mode");
 153             ret = -VIRTIO_CRYPTO_NOTSUPP;
 154             goto err;
 155         }
 156     } else {
 157         /* VIRTIO_CRYPTO_SYM_OP_NONE */
 158         error_report("unsupported cipher op_type: VIRTIO_CRYPTO_SYM_OP_NONE");
 159         ret = -VIRTIO_CRYPTO_NOTSUPP;
 160         goto err;
 161     }
 162
 163     queue_index = virtio_crypto_vq2q(queue_id);
 164     session_id = cryptodev_backend_sym_create_session(
 165                                      vcrypto->cryptodev,
 166                                      &info, queue_index, &local_err);
 167     if (session_id >= 0) {
 168         DPRINTF("create session_id=%" PRIu64 " successfully\n",
 169                 session_id);
 170
 171         ret = session_id;
 172     } else {
 173         if (local_err) {
 174             error_report_err(local_err);
 175         }
 176         ret = -VIRTIO_CRYPTO_ERR;
 177     }
 178
 179 err:
 180     g_free(info.cipher_key);
 181     g_free(info.auth_key);
 182     return ret;
 183 }
 184
 185 static uint8_t
 186 virtio_crypto_handle_close_session(VirtIOCrypto *vcrypto,
 187          struct virtio_crypto_destroy_session_req *close_sess_req,
 188          uint32_t queue_id)
 189 {
 190     int ret;
 191     uint64_t session_id;
 192     uint32_t status;
 193     Error *local_err = NULL;
 194
 195     session_id = ldq_le_p(&close_sess_req->session_id);
 196     DPRINTF("close session, id=%" PRIu64 "\n", session_id);
 197
 198     ret = cryptodev_backend_sym_close_session(
 199               vcrypto->cryptodev, session_id, queue_id, &local_err);
 200     if (ret == 0) {
 201         status = VIRTIO_CRYPTO_OK;
 202     } else {
 203         if (local_err) {
 204             error_report_err(local_err);
 205         } else {
 206             error_report("destroy session failed");
 207         }
 208         status = VIRTIO_CRYPTO_ERR;
 209     }
 210
 211     return status;
 212 }
 213
 214 static void virtio_crypto_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
 215 {
 216     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 217     struct virtio_crypto_op_ctrl_req ctrl;
 218     VirtQueueElement *elem;
 219     struct iovec *in_iov;
 220     struct iovec *out_iov;
 221     unsigned in_num;
 222     unsigned out_num;
 223     uint32_t queue_id;
 224     uint32_t opcode;
 225     struct virtio_crypto_session_input input;
 226     int64_t session_id;
 227     uint8_t status;
 228     size_t s;
 229
 230     for (;;) {
 231         g_autofree struct iovec *out_iov_copy = NULL;
 232
 233         elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
 234         if (!elem) {
 235             break;
 236         }
 237         if (elem->out_num < 1 || elem->in_num < 1) {
 238             virtio_error(vdev, "virtio-crypto ctrl missing headers");
 239             virtqueue_detach_element(vq, elem, 0);
 240             g_free(elem);
 241             break;
 242         }
 243
 244         out_num = elem->out_num;
 245         out_iov_copy = g_memdup2(elem->out_sg, sizeof(out_iov[0]) * out_num);
 246         out_iov = out_iov_copy;
 247
 248         in_num = elem->in_num;
 249         in_iov = elem->in_sg;
 250
 251         if (unlikely(iov_to_buf(out_iov, out_num, 0, &ctrl, sizeof(ctrl))
 252                     != sizeof(ctrl))) {
 253             virtio_error(vdev, "virtio-crypto request ctrl_hdr too short");
 254             virtqueue_detach_element(vq, elem, 0);
 255             g_free(elem);
 256             break;
 257         }
 258         iov_discard_front(&out_iov, &out_num, sizeof(ctrl));
 259
 260         opcode = ldl_le_p(&ctrl.header.opcode);
 261         queue_id = ldl_le_p(&ctrl.header.queue_id);
 262
 263         switch (opcode) {
 264         case VIRTIO_CRYPTO_CIPHER_CREATE_SESSION:
 265             memset(&input, 0, sizeof(input));
 266             session_id = virtio_crypto_create_sym_session(vcrypto,
 267                              &ctrl.u.sym_create_session,
 268                              queue_id, opcode,
 269                              out_iov, out_num);
 270             /* Serious errors, need to reset virtio crypto device */
 271             if (session_id == -EFAULT) {
 272                 virtqueue_detach_element(vq, elem, 0);
 273                 break;
 274             } else if (session_id == -VIRTIO_CRYPTO_NOTSUPP) {
 275                 stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP);
 276             } else if (session_id == -VIRTIO_CRYPTO_ERR) {
 277                 stl_le_p(&input.status, VIRTIO_CRYPTO_ERR);
 278             } else {
 279                 /* Set the session id */
 280                 stq_le_p(&input.session_id, session_id);
 281                 stl_le_p(&input.status, VIRTIO_CRYPTO_OK);
 282             }
 283
 284             s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input));
 285             if (unlikely(s != sizeof(input))) {
 286                 virtio_error(vdev, "virtio-crypto input incorrect");
 287                 virtqueue_detach_element(vq, elem, 0);
 288                 break;
 289             }
 290             virtqueue_push(vq, elem, sizeof(input));
 291             virtio_notify(vdev, vq);
 292             break;
 293         case VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION:
 294         case VIRTIO_CRYPTO_HASH_DESTROY_SESSION:
 295         case VIRTIO_CRYPTO_MAC_DESTROY_SESSION:
 296         case VIRTIO_CRYPTO_AEAD_DESTROY_SESSION:
 297             status = virtio_crypto_handle_close_session(vcrypto,
 298                    &ctrl.u.destroy_session, queue_id);
 299             /* The status only occupy one byte, we can directly use it */
 300             s = iov_from_buf(in_iov, in_num, 0, &status, sizeof(status));
 301             if (unlikely(s != sizeof(status))) {
 302                 virtio_error(vdev, "virtio-crypto status incorrect");
 303                 virtqueue_detach_element(vq, elem, 0);
 304                 break;
 305             }
 306             virtqueue_push(vq, elem, sizeof(status));
 307             virtio_notify(vdev, vq);
 308             break;
 309         case VIRTIO_CRYPTO_HASH_CREATE_SESSION:
 310         case VIRTIO_CRYPTO_MAC_CREATE_SESSION:
 311         case VIRTIO_CRYPTO_AEAD_CREATE_SESSION:
 312         default:
 313             error_report("virtio-crypto unsupported ctrl opcode: %d", opcode);
 314             memset(&input, 0, sizeof(input));
 315             stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP);
 316             s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input));
 317             if (unlikely(s != sizeof(input))) {
 318                 virtio_error(vdev, "virtio-crypto input incorrect");
 319                 virtqueue_detach_element(vq, elem, 0);
 320                 break;
 321             }
 322             virtqueue_push(vq, elem, sizeof(input));
 323             virtio_notify(vdev, vq);
 324
 325             break;
 326         } /* end switch case */
 327
 328         g_free(elem);
 329     } /* end for loop */
 330 }
 331
 332 static void virtio_crypto_init_request(VirtIOCrypto *vcrypto, VirtQueue *vq,
 333                                 VirtIOCryptoReq *req)
 334 {
 335     req->vcrypto = vcrypto;
 336     req->vq = vq;
 337     req->in = NULL;
 338     req->in_iov = NULL;
 339     req->in_num = 0;
 340     req->in_len = 0;
 341     req->flags = CRYPTODEV_BACKEND_ALG__MAX;
 342     req->u.sym_op_info = NULL;
 343 }
 344
 345 static void virtio_crypto_free_request(VirtIOCryptoReq *req)
 346 {
 347     if (req) {
 348         if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) {
 349             size_t max_len;
 350             CryptoDevBackendSymOpInfo *op_info = req->u.sym_op_info;
 351
 352             max_len = op_info->iv_len +
 353                       op_info->aad_len +
 354                       op_info->src_len +
 355                       op_info->dst_len +
 356                       op_info->digest_result_len;
 357
 358             /* Zeroize and free request data structure */
 359             memset(op_info, 0, sizeof(*op_info) + max_len);
 360             g_free(op_info);
 361         }
 362         g_free(req);
 363     }
 364 }
 365
 366 static void
 367 virtio_crypto_sym_input_data_helper(VirtIODevice *vdev,
 368                 VirtIOCryptoReq *req,
 369                 uint32_t status,
 370                 CryptoDevBackendSymOpInfo *sym_op_info)
 371 {
 372     size_t s, len;
 373
 374     if (status != VIRTIO_CRYPTO_OK) {
 375         return;
 376     }
 377
 378     len = sym_op_info->src_len;
 379     /* Save the cipher result */
 380     s = iov_from_buf(req->in_iov, req->in_num, 0, sym_op_info->dst, len);
 381     if (s != len) {
 382         virtio_error(vdev, "virtio-crypto dest data incorrect");
 383         return;
 384     }
 385
 386     iov_discard_front(&req->in_iov, &req->in_num, len);
 387
 388     if (sym_op_info->op_type ==
 389                       VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
 390         /* Save the digest result */
 391         s = iov_from_buf(req->in_iov, req->in_num, 0,
 392                          sym_op_info->digest_result,
 393                          sym_op_info->digest_result_len);
 394         if (s != sym_op_info->digest_result_len) {
 395             virtio_error(vdev, "virtio-crypto digest result incorrect");
 396         }
 397     }
 398 }
 399
 400 static void virtio_crypto_req_complete(VirtIOCryptoReq *req, uint8_t status)
 401 {
 402     VirtIOCrypto *vcrypto = req->vcrypto;
 403     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 404
 405     if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) {
 406         virtio_crypto_sym_input_data_helper(vdev, req, status,
 407                                             req->u.sym_op_info);
 408     }
 409     stb_p(&req->in->status, status);
 410     virtqueue_push(req->vq, &req->elem, req->in_len);
 411     virtio_notify(vdev, req->vq);
 412 }
 413
 414 static VirtIOCryptoReq *
 415 virtio_crypto_get_request(VirtIOCrypto *s, VirtQueue *vq)
 416 {
 417     VirtIOCryptoReq *req = virtqueue_pop(vq, sizeof(VirtIOCryptoReq));
 418
 419     if (req) {
 420         virtio_crypto_init_request(s, vq, req);
 421     }
 422     return req;
 423 }
 424
 425 static CryptoDevBackendSymOpInfo *
 426 virtio_crypto_sym_op_helper(VirtIODevice *vdev,
 427            struct virtio_crypto_cipher_para *cipher_para,
 428            struct virtio_crypto_alg_chain_data_para *alg_chain_para,
 429            struct iovec *iov, unsigned int out_num)
 430 {
 431     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 432     CryptoDevBackendSymOpInfo *op_info;
 433     uint32_t src_len = 0, dst_len = 0;
 434     uint32_t iv_len = 0;
 435     uint32_t aad_len = 0, hash_result_len = 0;
 436     uint32_t hash_start_src_offset = 0, len_to_hash = 0;
 437     uint32_t cipher_start_src_offset = 0, len_to_cipher = 0;
 438
 439     uint64_t max_len, curr_size = 0;
 440     size_t s;
 441
 442     /* Plain cipher */
 443     if (cipher_para) {
 444         iv_len = ldl_le_p(&cipher_para->iv_len);
 445         src_len = ldl_le_p(&cipher_para->src_data_len);
 446         dst_len = ldl_le_p(&cipher_para->dst_data_len);
 447     } else if (alg_chain_para) { /* Algorithm chain */
 448         iv_len = ldl_le_p(&alg_chain_para->iv_len);
 449         src_len = ldl_le_p(&alg_chain_para->src_data_len);
 450         dst_len = ldl_le_p(&alg_chain_para->dst_data_len);
 451
 452         aad_len = ldl_le_p(&alg_chain_para->aad_len);
 453         hash_result_len = ldl_le_p(&alg_chain_para->hash_result_len);
 454         hash_start_src_offset = ldl_le_p(
 455                          &alg_chain_para->hash_start_src_offset);
 456         cipher_start_src_offset = ldl_le_p(
 457                          &alg_chain_para->cipher_start_src_offset);
 458         len_to_cipher = ldl_le_p(&alg_chain_para->len_to_cipher);
 459         len_to_hash = ldl_le_p(&alg_chain_para->len_to_hash);
 460     } else {
 461         return NULL;
 462     }
 463
 464     max_len = (uint64_t)iv_len + aad_len + src_len + dst_len + hash_result_len;
 465     if (unlikely(max_len > vcrypto->conf.max_size)) {
 466         virtio_error(vdev, "virtio-crypto too big length");
 467         return NULL;
 468     }
 469
 470     op_info = g_malloc0(sizeof(CryptoDevBackendSymOpInfo) + max_len);
 471     op_info->iv_len = iv_len;
 472     op_info->src_len = src_len;
 473     op_info->dst_len = dst_len;
 474     op_info->aad_len = aad_len;
 475     op_info->digest_result_len = hash_result_len;
 476     op_info->hash_start_src_offset = hash_start_src_offset;
 477     op_info->len_to_hash = len_to_hash;
 478     op_info->cipher_start_src_offset = cipher_start_src_offset;
 479     op_info->len_to_cipher = len_to_cipher;
 480     /* Handle the initilization vector */
 481     if (op_info->iv_len > 0) {
 482         DPRINTF("iv_len=%" PRIu32 "\n", op_info->iv_len);
 483         op_info->iv = op_info->data + curr_size;
 484
 485         s = iov_to_buf(iov, out_num, 0, op_info->iv, op_info->iv_len);
 486         if (unlikely(s != op_info->iv_len)) {
 487             virtio_error(vdev, "virtio-crypto iv incorrect");
 488             goto err;
 489         }
 490         iov_discard_front(&iov, &out_num, op_info->iv_len);
 491         curr_size += op_info->iv_len;
 492     }
 493
 494     /* Handle additional authentication data if exists */
 495     if (op_info->aad_len > 0) {
 496         DPRINTF("aad_len=%" PRIu32 "\n", op_info->aad_len);
 497         op_info->aad_data = op_info->data + curr_size;
 498
 499         s = iov_to_buf(iov, out_num, 0, op_info->aad_data, op_info->aad_len);
 500         if (unlikely(s != op_info->aad_len)) {
 501             virtio_error(vdev, "virtio-crypto additional auth data incorrect");
 502             goto err;
 503         }
 504         iov_discard_front(&iov, &out_num, op_info->aad_len);
 505
 506         curr_size += op_info->aad_len;
 507     }
 508
 509     /* Handle the source data */
 510     if (op_info->src_len > 0) {
 511         DPRINTF("src_len=%" PRIu32 "\n", op_info->src_len);
 512         op_info->src = op_info->data + curr_size;
 513
 514         s = iov_to_buf(iov, out_num, 0, op_info->src, op_info->src_len);
 515         if (unlikely(s != op_info->src_len)) {
 516             virtio_error(vdev, "virtio-crypto source data incorrect");
 517             goto err;
 518         }
 519         iov_discard_front(&iov, &out_num, op_info->src_len);
 520
 521         curr_size += op_info->src_len;
 522     }
 523
 524     /* Handle the destination data */
 525     op_info->dst = op_info->data + curr_size;
 526     curr_size += op_info->dst_len;
 527
 528     DPRINTF("dst_len=%" PRIu32 "\n", op_info->dst_len);
 529
 530     /* Handle the hash digest result */
 531     if (hash_result_len > 0) {
 532         DPRINTF("hash_result_len=%" PRIu32 "\n", hash_result_len);
 533         op_info->digest_result = op_info->data + curr_size;
 534     }
 535
 536     return op_info;
 537
 538 err:
 539     g_free(op_info);
 540     return NULL;
 541 }
 542
 543 static int
 544 virtio_crypto_handle_sym_req(VirtIOCrypto *vcrypto,
 545                struct virtio_crypto_sym_data_req *req,
 546                CryptoDevBackendSymOpInfo **sym_op_info,
 547                struct iovec *iov, unsigned int out_num)
 548 {
 549     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 550     uint32_t op_type;
 551     CryptoDevBackendSymOpInfo *op_info;
 552
 553     op_type = ldl_le_p(&req->op_type);
 554
 555     if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
 556         op_info = virtio_crypto_sym_op_helper(vdev, &req->u.cipher.para,
 557                                               NULL, iov, out_num);
 558         if (!op_info) {
 559             return -EFAULT;
 560         }
 561         op_info->op_type = op_type;
 562     } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
 563         op_info = virtio_crypto_sym_op_helper(vdev, NULL,
 564                                               &req->u.chain.para,
 565                                               iov, out_num);
 566         if (!op_info) {
 567             return -EFAULT;
 568         }
 569         op_info->op_type = op_type;
 570     } else {
 571         /* VIRTIO_CRYPTO_SYM_OP_NONE */
 572         error_report("virtio-crypto unsupported cipher type");
 573         return -VIRTIO_CRYPTO_NOTSUPP;
 574     }
 575
 576     *sym_op_info = op_info;
 577
 578     return 0;
 579 }
 580
 581 static int
 582 virtio_crypto_handle_request(VirtIOCryptoReq *request)
 583 {
 584     VirtIOCrypto *vcrypto = request->vcrypto;
 585     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 586     VirtQueueElement *elem = &request->elem;
 587     int queue_index = virtio_crypto_vq2q(virtio_get_queue_index(request->vq));
 588     struct virtio_crypto_op_data_req req;
 589     int ret;
 590     g_autofree struct iovec *in_iov_copy = NULL;
 591     g_autofree struct iovec *out_iov_copy = NULL;
 592     struct iovec *in_iov;
 593     struct iovec *out_iov;
 594     unsigned in_num;
 595     unsigned out_num;
 596     uint32_t opcode;
 597     uint8_t status = VIRTIO_CRYPTO_ERR;
 598     uint64_t session_id;
 599     CryptoDevBackendSymOpInfo *sym_op_info = NULL;
 600     Error *local_err = NULL;
 601
 602     if (elem->out_num < 1 || elem->in_num < 1) {
 603         virtio_error(vdev, "virtio-crypto dataq missing headers");
 604         return -1;
 605     }
 606
 607     out_num = elem->out_num;
 608     out_iov_copy = g_memdup2(elem->out_sg, sizeof(out_iov[0]) * out_num);
 609     out_iov = out_iov_copy;
 610
 611     in_num = elem->in_num;
 612     in_iov_copy = g_memdup2(elem->in_sg, sizeof(in_iov[0]) * in_num);
 613     in_iov = in_iov_copy;
 614
 615     if (unlikely(iov_to_buf(out_iov, out_num, 0, &req, sizeof(req))
 616                 != sizeof(req))) {
 617         virtio_error(vdev, "virtio-crypto request outhdr too short");
 618         return -1;
 619     }
 620     iov_discard_front(&out_iov, &out_num, sizeof(req));
 621
 622     if (in_iov[in_num - 1].iov_len <
 623             sizeof(struct virtio_crypto_inhdr)) {
 624         virtio_error(vdev, "virtio-crypto request inhdr too short");
 625         return -1;
 626     }
 627     /* We always touch the last byte, so just see how big in_iov is. */
 628     request->in_len = iov_size(in_iov, in_num);
 629     request->in = (void *)in_iov[in_num - 1].iov_base
 630               + in_iov[in_num - 1].iov_len
 631               - sizeof(struct virtio_crypto_inhdr);
 632     iov_discard_back(in_iov, &in_num, sizeof(struct virtio_crypto_inhdr));
 633
 634     /*
 635      * The length of operation result, including dest_data
 636      * and digest_result if exists.
 637      */
 638     request->in_num = in_num;
 639     request->in_iov = in_iov;
 640
 641     opcode = ldl_le_p(&req.header.opcode);
 642     session_id = ldq_le_p(&req.header.session_id);
 643
 644     switch (opcode) {
 645     case VIRTIO_CRYPTO_CIPHER_ENCRYPT:
 646     case VIRTIO_CRYPTO_CIPHER_DECRYPT:
 647         ret = virtio_crypto_handle_sym_req(vcrypto,
 648                          &req.u.sym_req,
 649                          &sym_op_info,
 650                          out_iov, out_num);
 651         /* Serious errors, need to reset virtio crypto device */
 652         if (ret == -EFAULT) {
 653             return -1;
 654         } else if (ret == -VIRTIO_CRYPTO_NOTSUPP) {
 655             virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP);
 656             virtio_crypto_free_request(request);
 657         } else {
 658             sym_op_info->session_id = session_id;
 659
 660             /* Set request's parameter */
 661             request->flags = CRYPTODEV_BACKEND_ALG_SYM;
 662             request->u.sym_op_info = sym_op_info;
 663             ret = cryptodev_backend_crypto_operation(vcrypto->cryptodev,
 664                                     request, queue_index, &local_err);
 665             if (ret < 0) {
 666                 status = -ret;
 667                 if (local_err) {
 668                     error_report_err(local_err);
 669                 }
 670             } else { /* ret == VIRTIO_CRYPTO_OK */
 671                 status = ret;
 672             }
 673             virtio_crypto_req_complete(request, status);
 674             virtio_crypto_free_request(request);
 675         }
 676         break;
 677     case VIRTIO_CRYPTO_HASH:
 678     case VIRTIO_CRYPTO_MAC:
 679     case VIRTIO_CRYPTO_AEAD_ENCRYPT:
 680     case VIRTIO_CRYPTO_AEAD_DECRYPT:
 681     default:
 682         error_report("virtio-crypto unsupported dataq opcode: %u",
 683                      opcode);
 684         virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP);
 685         virtio_crypto_free_request(request);
 686     }
 687
 688     return 0;
 689 }
 690
 691 static void virtio_crypto_handle_dataq(VirtIODevice *vdev, VirtQueue *vq)
 692 {
 693     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 694     VirtIOCryptoReq *req;
 695
 696     while ((req = virtio_crypto_get_request(vcrypto, vq))) {
 697         if (virtio_crypto_handle_request(req) < 0) {
 698             virtqueue_detach_element(req->vq, &req->elem, 0);
 699             virtio_crypto_free_request(req);
 700             break;
 701         }
 702     }
 703 }
 704
 705 static void virtio_crypto_dataq_bh(void *opaque)
 706 {
 707     VirtIOCryptoQueue *q = opaque;
 708     VirtIOCrypto *vcrypto = q->vcrypto;
 709     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 710
 711     /* This happens when device was stopped but BH wasn't. */
 712     if (!vdev->vm_running) {
 713         return;
 714     }
 715
 716     /* Just in case the driver is not ready on more */
 717     if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) {
 718         return;
 719     }
 720
 721     for (;;) {
 722         virtio_crypto_handle_dataq(vdev, q->dataq);
 723         virtio_queue_set_notification(q->dataq, 1);
 724
 725         /* Are we done or did the guest add more buffers? */
 726         if (virtio_queue_empty(q->dataq)) {
 727             break;
 728         }
 729
 730         virtio_queue_set_notification(q->dataq, 0);
 731     }
 732 }
 733
 734 static void
 735 virtio_crypto_handle_dataq_bh(VirtIODevice *vdev, VirtQueue *vq)
 736 {
 737     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 738     VirtIOCryptoQueue *q =
 739          &vcrypto->vqs[virtio_crypto_vq2q(virtio_get_queue_index(vq))];
 740
 741     /* This happens when device was stopped but VCPU wasn't. */
 742     if (!vdev->vm_running) {
 743         return;
 744     }
 745     virtio_queue_set_notification(vq, 0);
 746     qemu_bh_schedule(q->dataq_bh);
 747 }
 748
 749 static uint64_t virtio_crypto_get_features(VirtIODevice *vdev,
 750                                            uint64_t features,
 751                                            Error **errp)
 752 {
 753     return features;
 754 }
 755
 756 static void virtio_crypto_reset(VirtIODevice *vdev)
 757 {
 758     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 759     /* multiqueue is disabled by default */
 760     vcrypto->curr_queues = 1;
 761     if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) {
 762         vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY;
 763     } else {
 764         vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY;
 765     }
 766 }
 767
 768 static void virtio_crypto_init_config(VirtIODevice *vdev)
 769 {
 770     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 771
 772     vcrypto->conf.crypto_services =
 773                      vcrypto->conf.cryptodev->conf.crypto_services;
 774     vcrypto->conf.cipher_algo_l =
 775                      vcrypto->conf.cryptodev->conf.cipher_algo_l;
 776     vcrypto->conf.cipher_algo_h =
 777                      vcrypto->conf.cryptodev->conf.cipher_algo_h;
 778     vcrypto->conf.hash_algo = vcrypto->conf.cryptodev->conf.hash_algo;
 779     vcrypto->conf.mac_algo_l = vcrypto->conf.cryptodev->conf.mac_algo_l;
 780     vcrypto->conf.mac_algo_h = vcrypto->conf.cryptodev->conf.mac_algo_h;
 781     vcrypto->conf.aead_algo = vcrypto->conf.cryptodev->conf.aead_algo;
 782     vcrypto->conf.max_cipher_key_len =
 783                   vcrypto->conf.cryptodev->conf.max_cipher_key_len;
 784     vcrypto->conf.max_auth_key_len =
 785                   vcrypto->conf.cryptodev->conf.max_auth_key_len;
 786     vcrypto->conf.max_size = vcrypto->conf.cryptodev->conf.max_size;
 787 }
 788
 789 static void virtio_crypto_device_realize(DeviceState *dev, Error **errp)
 790 {
 791     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 792     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev);
 793     int i;
 794
 795     vcrypto->cryptodev = vcrypto->conf.cryptodev;
 796     if (vcrypto->cryptodev == NULL) {
 797         error_setg(errp, "'cryptodev' parameter expects a valid object");
 798         return;
 799     } else if (cryptodev_backend_is_used(vcrypto->cryptodev)) {
 800         error_setg(errp, "can't use already used cryptodev backend: %s",
 801                    object_get_canonical_path_component(OBJECT(vcrypto->conf.cryptodev)));
 802         return;
 803     }
 804
 805     vcrypto->max_queues = MAX(vcrypto->cryptodev->conf.peers.queues, 1);
 806     if (vcrypto->max_queues + 1 > VIRTIO_QUEUE_MAX) {
 807         error_setg(errp, "Invalid number of queues (= %" PRIu32 "), "
 808                    "must be a positive integer less than %d.",
 809                    vcrypto->max_queues, VIRTIO_QUEUE_MAX);
 810         return;
 811     }
 812
 813     virtio_init(vdev, VIRTIO_ID_CRYPTO, vcrypto->config_size);
 814     vcrypto->curr_queues = 1;
 815     vcrypto->vqs = g_new0(VirtIOCryptoQueue, vcrypto->max_queues);
 816     for (i = 0; i < vcrypto->max_queues; i++) {
 817         vcrypto->vqs[i].dataq =
 818                  virtio_add_queue(vdev, 1024, virtio_crypto_handle_dataq_bh);
 819         vcrypto->vqs[i].dataq_bh =
 820                  qemu_bh_new(virtio_crypto_dataq_bh, &vcrypto->vqs[i]);
 821         vcrypto->vqs[i].vcrypto = vcrypto;
 822     }
 823
 824     vcrypto->ctrl_vq = virtio_add_queue(vdev, 64, virtio_crypto_handle_ctrl);
 825     if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) {
 826         vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY;
 827     } else {
 828         vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY;
 829     }
 830
 831     virtio_crypto_init_config(vdev);
 832     cryptodev_backend_set_used(vcrypto->cryptodev, true);
 833 }
 834
 835 static void virtio_crypto_device_unrealize(DeviceState *dev)
 836 {
 837     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 838     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev);
 839     VirtIOCryptoQueue *q;
 840     int i, max_queues;
 841
 842     max_queues = vcrypto->multiqueue ? vcrypto->max_queues : 1;
 843     for (i = 0; i < max_queues; i++) {
 844         virtio_delete_queue(vcrypto->vqs[i].dataq);
 845         q = &vcrypto->vqs[i];
 846         qemu_bh_delete(q->dataq_bh);
 847     }
 848
 849     g_free(vcrypto->vqs);
 850     virtio_delete_queue(vcrypto->ctrl_vq);
 851
 852     virtio_cleanup(vdev);
 853     cryptodev_backend_set_used(vcrypto->cryptodev, false);
 854 }
 855
 856 static const VMStateDescription vmstate_virtio_crypto = {
 857     .name = "virtio-crypto",
 858     .unmigratable = 1,
 859     .minimum_version_id = VIRTIO_CRYPTO_VM_VERSION,
 860     .version_id = VIRTIO_CRYPTO_VM_VERSION,
 861     .fields = (VMStateField[]) {
 862         VMSTATE_VIRTIO_DEVICE,
 863         VMSTATE_END_OF_LIST()
 864     },
 865 };
 866
 867 static Property virtio_crypto_properties[] = {
 868     DEFINE_PROP_LINK("cryptodev", VirtIOCrypto, conf.cryptodev,
 869                      TYPE_CRYPTODEV_BACKEND, CryptoDevBackend *),
 870     DEFINE_PROP_END_OF_LIST(),
 871 };
 872
 873 static void virtio_crypto_get_config(VirtIODevice *vdev, uint8_t *config)
 874 {
 875     VirtIOCrypto *c = VIRTIO_CRYPTO(vdev);
 876     struct virtio_crypto_config crypto_cfg = {};
 877
 878     /*
 879      * Virtio-crypto device conforms to VIRTIO 1.0 which is always LE,
 880      * so we can use LE accessors directly.
 881      */
 882     stl_le_p(&crypto_cfg.status, c->status);
 883     stl_le_p(&crypto_cfg.max_dataqueues, c->max_queues);
 884     stl_le_p(&crypto_cfg.crypto_services, c->conf.crypto_services);
 885     stl_le_p(&crypto_cfg.cipher_algo_l, c->conf.cipher_algo_l);
 886     stl_le_p(&crypto_cfg.cipher_algo_h, c->conf.cipher_algo_h);
 887     stl_le_p(&crypto_cfg.hash_algo, c->conf.hash_algo);
 888     stl_le_p(&crypto_cfg.mac_algo_l, c->conf.mac_algo_l);
 889     stl_le_p(&crypto_cfg.mac_algo_h, c->conf.mac_algo_h);
 890     stl_le_p(&crypto_cfg.aead_algo, c->conf.aead_algo);
 891     stl_le_p(&crypto_cfg.max_cipher_key_len, c->conf.max_cipher_key_len);
 892     stl_le_p(&crypto_cfg.max_auth_key_len, c->conf.max_auth_key_len);
 893     stq_le_p(&crypto_cfg.max_size, c->conf.max_size);
 894
 895     memcpy(config, &crypto_cfg, c->config_size);
 896 }
 897
 898 static bool virtio_crypto_started(VirtIOCrypto *c, uint8_t status)
 899 {
 900     VirtIODevice *vdev = VIRTIO_DEVICE(c);
 901     return (status & VIRTIO_CONFIG_S_DRIVER_OK) &&
 902         (c->status & VIRTIO_CRYPTO_S_HW_READY) && vdev->vm_running;
 903 }
 904
 905 static void virtio_crypto_vhost_status(VirtIOCrypto *c, uint8_t status)
 906 {
 907     VirtIODevice *vdev = VIRTIO_DEVICE(c);
 908     int queues = c->multiqueue ? c->max_queues : 1;
 909     CryptoDevBackend *b = c->cryptodev;
 910     CryptoDevBackendClient *cc = b->conf.peers.ccs[0];
 911
 912     if (!cryptodev_get_vhost(cc, b, 0)) {
 913         return;
 914     }
 915
 916     if ((virtio_crypto_started(c, status)) == !!c->vhost_started) {
 917         return;
 918     }
 919
 920     if (!c->vhost_started) {
 921         int r;
 922
 923         c->vhost_started = 1;
 924         r = cryptodev_vhost_start(vdev, queues);
 925         if (r < 0) {
 926             error_report("unable to start vhost crypto: %d: "
 927                          "falling back on userspace virtio", -r);
 928             c->vhost_started = 0;
 929         }
 930     } else {
 931         cryptodev_vhost_stop(vdev, queues);
 932         c->vhost_started = 0;
 933     }
 934 }
 935
 936 static void virtio_crypto_set_status(VirtIODevice *vdev, uint8_t status)
 937 {
 938     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 939
 940     virtio_crypto_vhost_status(vcrypto, status);
 941 }
 942
 943 static void virtio_crypto_guest_notifier_mask(VirtIODevice *vdev, int idx,
 944                                            bool mask)
 945 {
 946     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 947     int queue = virtio_crypto_vq2q(idx);
 948
 949     assert(vcrypto->vhost_started);
 950
 951     cryptodev_vhost_virtqueue_mask(vdev, queue, idx, mask);
 952 }
 953
 954 static bool virtio_crypto_guest_notifier_pending(VirtIODevice *vdev, int idx)
 955 {
 956     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 957     int queue = virtio_crypto_vq2q(idx);
 958
 959     assert(vcrypto->vhost_started);
 960
 961     return cryptodev_vhost_virtqueue_pending(vdev, queue, idx);
 962 }
 963
 964 static struct vhost_dev *virtio_crypto_get_vhost(VirtIODevice *vdev)
 965 {
 966     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 967     CryptoDevBackend *b = vcrypto->cryptodev;
 968     CryptoDevBackendClient *cc = b->conf.peers.ccs[0];
 969     CryptoDevBackendVhost *vhost_crypto = cryptodev_get_vhost(cc, b, 0);
 970     return &vhost_crypto->dev;
 971 }
 972
 973 static void virtio_crypto_class_init(ObjectClass *klass, void *data)
 974 {
 975     DeviceClass *dc = DEVICE_CLASS(klass);
 976     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
 977
 978     device_class_set_props(dc, virtio_crypto_properties);
 979     dc->vmsd = &vmstate_virtio_crypto;
 980     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 981     vdc->realize = virtio_crypto_device_realize;
 982     vdc->unrealize = virtio_crypto_device_unrealize;
 983     vdc->get_config = virtio_crypto_get_config;
 984     vdc->get_features = virtio_crypto_get_features;
 985     vdc->reset = virtio_crypto_reset;
 986     vdc->set_status = virtio_crypto_set_status;
 987     vdc->guest_notifier_mask = virtio_crypto_guest_notifier_mask;
 988     vdc->guest_notifier_pending = virtio_crypto_guest_notifier_pending;
 989     vdc->get_vhost = virtio_crypto_get_vhost;
 990 }
 991
 992 static void virtio_crypto_instance_init(Object *obj)
 993 {
 994     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(obj);
 995
 996     /*
 997      * The default config_size is sizeof(struct virtio_crypto_config).
 998      * Can be overriden with virtio_crypto_set_config_size.
 999      */
1000     vcrypto->config_size = sizeof(struct virtio_crypto_config);
1001 }
1002
1003 static const TypeInfo virtio_crypto_info = {
1004     .name = TYPE_VIRTIO_CRYPTO,
1005     .parent = TYPE_VIRTIO_DEVICE,
1006     .instance_size = sizeof(VirtIOCrypto),
1007     .instance_init = virtio_crypto_instance_init,
1008     .class_init = virtio_crypto_class_init,
1009 };
1010
1011 static void virtio_register_types(void)
1012 {
1013     type_register_static(&virtio_crypto_info);
1014 }
1015
1016 type_init(virtio_register_types)