hw/virtio/virtio-crypto.c

   1 /*
   2  * Virtio crypto Support
   3  *
   4  * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD.
   5  *
   6  * Authors:
   7  *    Gonglei <arei.gonglei@huawei.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or
  10  * (at your option) any later version.  See the COPYING file in the
  11  * top-level directory.
  12  */
  13 #include "qemu/osdep.h"
  14 #include "qemu/iov.h"
  15 #include "hw/qdev.h"
  16 #include "qapi/error.h"
  17 #include "qemu/error-report.h"
  18
  19 #include "hw/virtio/virtio.h"
  20 #include "hw/virtio/virtio-crypto.h"
  21 #include "hw/virtio/virtio-access.h"
  22 #include "standard-headers/linux/virtio_ids.h"
  23
  24 #define VIRTIO_CRYPTO_VM_VERSION 1
  25
  26 /*
  27  * Transfer virtqueue index to crypto queue index.
  28  * The control virtqueue is after the data virtqueues
  29  * so the input value doesn't need to be adjusted
  30  */
  31 static inline int virtio_crypto_vq2q(int queue_index)
  32 {
  33     return queue_index;
  34 }
  35
  36 static int
  37 virtio_crypto_cipher_session_helper(VirtIODevice *vdev,
  38            CryptoDevBackendSymSessionInfo *info,
  39            struct virtio_crypto_cipher_session_para *cipher_para,
  40            struct iovec **iov, unsigned int *out_num)
  41 {
  42     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
  43     unsigned int num = *out_num;
  44
  45     info->cipher_alg = ldl_le_p(&cipher_para->algo);
  46     info->key_len = ldl_le_p(&cipher_para->keylen);
  47     info->direction = ldl_le_p(&cipher_para->op);
  48     DPRINTF("cipher_alg=%" PRIu32 ", info->direction=%" PRIu32 "\n",
  49              info->cipher_alg, info->direction);
  50
  51     if (info->key_len > vcrypto->conf.max_cipher_key_len) {
  52         error_report("virtio-crypto length of cipher key is too big: %u",
  53                      info->key_len);
  54         return -VIRTIO_CRYPTO_ERR;
  55     }
  56     /* Get cipher key */
  57     if (info->key_len > 0) {
  58         size_t s;
  59         DPRINTF("keylen=%" PRIu32 "\n", info->key_len);
  60
  61         info->cipher_key = g_malloc(info->key_len);
  62         s = iov_to_buf(*iov, num, 0, info->cipher_key, info->key_len);
  63         if (unlikely(s != info->key_len)) {
  64             virtio_error(vdev, "virtio-crypto cipher key incorrect");
  65             return -EFAULT;
  66         }
  67         iov_discard_front(iov, &num, info->key_len);
  68         *out_num = num;
  69     }
  70
  71     return 0;
  72 }
  73
  74 static int64_t
  75 virtio_crypto_create_sym_session(VirtIOCrypto *vcrypto,
  76                struct virtio_crypto_sym_create_session_req *sess_req,
  77                uint32_t queue_id,
  78                uint32_t opcode,
  79                struct iovec *iov, unsigned int out_num)
  80 {
  81     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
  82     CryptoDevBackendSymSessionInfo info;
  83     int64_t session_id;
  84     int queue_index;
  85     uint32_t op_type;
  86     Error *local_err = NULL;
  87     int ret;
  88
  89     memset(&info, 0, sizeof(info));
  90     op_type = ldl_le_p(&sess_req->op_type);
  91     info.op_type = op_type;
  92     info.op_code = opcode;
  93
  94     if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
  95         ret = virtio_crypto_cipher_session_helper(vdev, &info,
  96                            &sess_req->u.cipher.para,
  97                            &iov, &out_num);
  98         if (ret < 0) {
  99             goto err;
 100         }
 101     } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
 102         size_t s;
 103         /* cipher part */
 104         ret = virtio_crypto_cipher_session_helper(vdev, &info,
 105                            &sess_req->u.chain.para.cipher_param,
 106                            &iov, &out_num);
 107         if (ret < 0) {
 108             goto err;
 109         }
 110         /* hash part */
 111         info.alg_chain_order = ldl_le_p(
 112                                      &sess_req->u.chain.para.alg_chain_order);
 113         info.add_len = ldl_le_p(&sess_req->u.chain.para.aad_len);
 114         info.hash_mode = ldl_le_p(&sess_req->u.chain.para.hash_mode);
 115         if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH) {
 116             info.hash_alg = ldl_le_p(&sess_req->u.chain.para.u.mac_param.algo);
 117             info.auth_key_len = ldl_le_p(
 118                              &sess_req->u.chain.para.u.mac_param.auth_key_len);
 119             info.hash_result_len = ldl_le_p(
 120                            &sess_req->u.chain.para.u.mac_param.hash_result_len);
 121             if (info.auth_key_len > vcrypto->conf.max_auth_key_len) {
 122                 error_report("virtio-crypto length of auth key is too big: %u",
 123                              info.auth_key_len);
 124                 ret = -VIRTIO_CRYPTO_ERR;
 125                 goto err;
 126             }
 127             /* get auth key */
 128             if (info.auth_key_len > 0) {
 129                 DPRINTF("auth_keylen=%" PRIu32 "\n", info.auth_key_len);
 130                 info.auth_key = g_malloc(info.auth_key_len);
 131                 s = iov_to_buf(iov, out_num, 0, info.auth_key,
 132                                info.auth_key_len);
 133                 if (unlikely(s != info.auth_key_len)) {
 134                     virtio_error(vdev,
 135                           "virtio-crypto authenticated key incorrect");
 136                     ret = -EFAULT;
 137                     goto err;
 138                 }
 139                 iov_discard_front(&iov, &out_num, info.auth_key_len);
 140             }
 141         } else if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN) {
 142             info.hash_alg = ldl_le_p(
 143                              &sess_req->u.chain.para.u.hash_param.algo);
 144             info.hash_result_len = ldl_le_p(
 145                         &sess_req->u.chain.para.u.hash_param.hash_result_len);
 146         } else {
 147             /* VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */
 148             error_report("unsupported hash mode");
 149             ret = -VIRTIO_CRYPTO_NOTSUPP;
 150             goto err;
 151         }
 152     } else {
 153         /* VIRTIO_CRYPTO_SYM_OP_NONE */
 154         error_report("unsupported cipher op_type: VIRTIO_CRYPTO_SYM_OP_NONE");
 155         ret = -VIRTIO_CRYPTO_NOTSUPP;
 156         goto err;
 157     }
 158
 159     queue_index = virtio_crypto_vq2q(queue_id);
 160     session_id = cryptodev_backend_sym_create_session(
 161                                      vcrypto->cryptodev,
 162                                      &info, queue_index, &local_err);
 163     if (session_id >= 0) {
 164         DPRINTF("create session_id=%" PRIu64 " successfully\n",
 165                 session_id);
 166
 167         ret = session_id;
 168     } else {
 169         if (local_err) {
 170             error_report_err(local_err);
 171         }
 172         ret = -VIRTIO_CRYPTO_ERR;
 173     }
 174
 175 err:
 176     g_free(info.cipher_key);
 177     g_free(info.auth_key);
 178     return ret;
 179 }
 180
 181 static uint8_t
 182 virtio_crypto_handle_close_session(VirtIOCrypto *vcrypto,
 183          struct virtio_crypto_destroy_session_req *close_sess_req,
 184          uint32_t queue_id)
 185 {
 186     int ret;
 187     uint64_t session_id;
 188     uint32_t status;
 189     Error *local_err = NULL;
 190
 191     session_id = ldq_le_p(&close_sess_req->session_id);
 192     DPRINTF("close session, id=%" PRIu64 "\n", session_id);
 193
 194     ret = cryptodev_backend_sym_close_session(
 195               vcrypto->cryptodev, session_id, queue_id, &local_err);
 196     if (ret == 0) {
 197         status = VIRTIO_CRYPTO_OK;
 198     } else {
 199         if (local_err) {
 200             error_report_err(local_err);
 201         } else {
 202             error_report("destroy session failed");
 203         }
 204         status = VIRTIO_CRYPTO_ERR;
 205     }
 206
 207     return status;
 208 }
 209
 210 static void virtio_crypto_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
 211 {
 212     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 213     struct virtio_crypto_op_ctrl_req ctrl;
 214     VirtQueueElement *elem;
 215     struct iovec *in_iov;
 216     struct iovec *out_iov;
 217     unsigned in_num;
 218     unsigned out_num;
 219     uint32_t queue_id;
 220     uint32_t opcode;
 221     struct virtio_crypto_session_input input;
 222     int64_t session_id;
 223     uint8_t status;
 224     size_t s;
 225
 226     for (;;) {
 227         elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
 228         if (!elem) {
 229             break;
 230         }
 231         if (elem->out_num < 1 || elem->in_num < 1) {
 232             virtio_error(vdev, "virtio-crypto ctrl missing headers");
 233             virtqueue_detach_element(vq, elem, 0);
 234             g_free(elem);
 235             break;
 236         }
 237
 238         out_num = elem->out_num;
 239         out_iov = elem->out_sg;
 240         in_num = elem->in_num;
 241         in_iov = elem->in_sg;
 242         if (unlikely(iov_to_buf(out_iov, out_num, 0, &ctrl, sizeof(ctrl))
 243                     != sizeof(ctrl))) {
 244             virtio_error(vdev, "virtio-crypto request ctrl_hdr too short");
 245             virtqueue_detach_element(vq, elem, 0);
 246             g_free(elem);
 247             break;
 248         }
 249         iov_discard_front(&out_iov, &out_num, sizeof(ctrl));
 250
 251         opcode = ldl_le_p(&ctrl.header.opcode);
 252         queue_id = ldl_le_p(&ctrl.header.queue_id);
 253
 254         switch (opcode) {
 255         case VIRTIO_CRYPTO_CIPHER_CREATE_SESSION:
 256             memset(&input, 0, sizeof(input));
 257             session_id = virtio_crypto_create_sym_session(vcrypto,
 258                              &ctrl.u.sym_create_session,
 259                              queue_id, opcode,
 260                              out_iov, out_num);
 261             /* Serious errors, need to reset virtio crypto device */
 262             if (session_id == -EFAULT) {
 263                 virtqueue_detach_element(vq, elem, 0);
 264                 break;
 265             } else if (session_id == -VIRTIO_CRYPTO_NOTSUPP) {
 266                 stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP);
 267             } else if (session_id == -VIRTIO_CRYPTO_ERR) {
 268                 stl_le_p(&input.status, VIRTIO_CRYPTO_ERR);
 269             } else {
 270                 /* Set the session id */
 271                 stq_le_p(&input.session_id, session_id);
 272                 stl_le_p(&input.status, VIRTIO_CRYPTO_OK);
 273             }
 274
 275             s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input));
 276             if (unlikely(s != sizeof(input))) {
 277                 virtio_error(vdev, "virtio-crypto input incorrect");
 278                 virtqueue_detach_element(vq, elem, 0);
 279                 break;
 280             }
 281             virtqueue_push(vq, elem, sizeof(input));
 282             virtio_notify(vdev, vq);
 283             break;
 284         case VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION:
 285         case VIRTIO_CRYPTO_HASH_DESTROY_SESSION:
 286         case VIRTIO_CRYPTO_MAC_DESTROY_SESSION:
 287         case VIRTIO_CRYPTO_AEAD_DESTROY_SESSION:
 288             status = virtio_crypto_handle_close_session(vcrypto,
 289                    &ctrl.u.destroy_session, queue_id);
 290             /* The status only occupy one byte, we can directly use it */
 291             s = iov_from_buf(in_iov, in_num, 0, &status, sizeof(status));
 292             if (unlikely(s != sizeof(status))) {
 293                 virtio_error(vdev, "virtio-crypto status incorrect");
 294                 virtqueue_detach_element(vq, elem, 0);
 295                 break;
 296             }
 297             virtqueue_push(vq, elem, sizeof(status));
 298             virtio_notify(vdev, vq);
 299             break;
 300         case VIRTIO_CRYPTO_HASH_CREATE_SESSION:
 301         case VIRTIO_CRYPTO_MAC_CREATE_SESSION:
 302         case VIRTIO_CRYPTO_AEAD_CREATE_SESSION:
 303         default:
 304             error_report("virtio-crypto unsupported ctrl opcode: %d", opcode);
 305             memset(&input, 0, sizeof(input));
 306             stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP);
 307             s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input));
 308             if (unlikely(s != sizeof(input))) {
 309                 virtio_error(vdev, "virtio-crypto input incorrect");
 310                 virtqueue_detach_element(vq, elem, 0);
 311                 break;
 312             }
 313             virtqueue_push(vq, elem, sizeof(input));
 314             virtio_notify(vdev, vq);
 315
 316             break;
 317         } /* end switch case */
 318
 319         g_free(elem);
 320     } /* end for loop */
 321 }
 322
 323 static void virtio_crypto_init_request(VirtIOCrypto *vcrypto, VirtQueue *vq,
 324                                 VirtIOCryptoReq *req)
 325 {
 326     req->vcrypto = vcrypto;
 327     req->vq = vq;
 328     req->in = NULL;
 329     req->in_iov = NULL;
 330     req->in_num = 0;
 331     req->in_len = 0;
 332     req->flags = CRYPTODEV_BACKEND_ALG__MAX;
 333     req->u.sym_op_info = NULL;
 334 }
 335
 336 static void virtio_crypto_free_request(VirtIOCryptoReq *req)
 337 {
 338     if (req) {
 339         if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) {
 340             size_t max_len;
 341             CryptoDevBackendSymOpInfo *op_info = req->u.sym_op_info;
 342
 343             max_len = op_info->iv_len +
 344                       op_info->aad_len +
 345                       op_info->src_len +
 346                       op_info->dst_len +
 347                       op_info->digest_result_len;
 348
 349             /* Zeroize and free request data structure */
 350             memset(op_info, 0, sizeof(*op_info) + max_len);
 351             g_free(op_info);
 352         }
 353         g_free(req);
 354     }
 355 }
 356
 357 static void
 358 virtio_crypto_sym_input_data_helper(VirtIODevice *vdev,
 359                 VirtIOCryptoReq *req,
 360                 uint32_t status,
 361                 CryptoDevBackendSymOpInfo *sym_op_info)
 362 {
 363     size_t s, len;
 364
 365     if (status != VIRTIO_CRYPTO_OK) {
 366         return;
 367     }
 368
 369     len = sym_op_info->src_len;
 370     /* Save the cipher result */
 371     s = iov_from_buf(req->in_iov, req->in_num, 0, sym_op_info->dst, len);
 372     if (s != len) {
 373         virtio_error(vdev, "virtio-crypto dest data incorrect");
 374         return;
 375     }
 376
 377     iov_discard_front(&req->in_iov, &req->in_num, len);
 378
 379     if (sym_op_info->op_type ==
 380                       VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
 381         /* Save the digest result */
 382         s = iov_from_buf(req->in_iov, req->in_num, 0,
 383                          sym_op_info->digest_result,
 384                          sym_op_info->digest_result_len);
 385         if (s != sym_op_info->digest_result_len) {
 386             virtio_error(vdev, "virtio-crypto digest result incorrect");
 387         }
 388     }
 389 }
 390
 391 static void virtio_crypto_req_complete(VirtIOCryptoReq *req, uint8_t status)
 392 {
 393     VirtIOCrypto *vcrypto = req->vcrypto;
 394     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 395
 396     if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) {
 397         virtio_crypto_sym_input_data_helper(vdev, req, status,
 398                                             req->u.sym_op_info);
 399     }
 400     stb_p(&req->in->status, status);
 401     virtqueue_push(req->vq, &req->elem, req->in_len);
 402     virtio_notify(vdev, req->vq);
 403 }
 404
 405 static VirtIOCryptoReq *
 406 virtio_crypto_get_request(VirtIOCrypto *s, VirtQueue *vq)
 407 {
 408     VirtIOCryptoReq *req = virtqueue_pop(vq, sizeof(VirtIOCryptoReq));
 409
 410     if (req) {
 411         virtio_crypto_init_request(s, vq, req);
 412     }
 413     return req;
 414 }
 415
 416 static CryptoDevBackendSymOpInfo *
 417 virtio_crypto_sym_op_helper(VirtIODevice *vdev,
 418            struct virtio_crypto_cipher_para *cipher_para,
 419            struct virtio_crypto_alg_chain_data_para *alg_chain_para,
 420            struct iovec *iov, unsigned int out_num)
 421 {
 422     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 423     CryptoDevBackendSymOpInfo *op_info;
 424     uint32_t src_len = 0, dst_len = 0;
 425     uint32_t iv_len = 0;
 426     uint32_t aad_len = 0, hash_result_len = 0;
 427     uint32_t hash_start_src_offset = 0, len_to_hash = 0;
 428     uint32_t cipher_start_src_offset = 0, len_to_cipher = 0;
 429
 430     uint64_t max_len, curr_size = 0;
 431     size_t s;
 432
 433     /* Plain cipher */
 434     if (cipher_para) {
 435         iv_len = ldl_le_p(&cipher_para->iv_len);
 436         src_len = ldl_le_p(&cipher_para->src_data_len);
 437         dst_len = ldl_le_p(&cipher_para->dst_data_len);
 438     } else if (alg_chain_para) { /* Algorithm chain */
 439         iv_len = ldl_le_p(&alg_chain_para->iv_len);
 440         src_len = ldl_le_p(&alg_chain_para->src_data_len);
 441         dst_len = ldl_le_p(&alg_chain_para->dst_data_len);
 442
 443         aad_len = ldl_le_p(&alg_chain_para->aad_len);
 444         hash_result_len = ldl_le_p(&alg_chain_para->hash_result_len);
 445         hash_start_src_offset = ldl_le_p(
 446                          &alg_chain_para->hash_start_src_offset);
 447         cipher_start_src_offset = ldl_le_p(
 448                          &alg_chain_para->cipher_start_src_offset);
 449         len_to_cipher = ldl_le_p(&alg_chain_para->len_to_cipher);
 450         len_to_hash = ldl_le_p(&alg_chain_para->len_to_hash);
 451     } else {
 452         return NULL;
 453     }
 454
 455     max_len = (uint64_t)iv_len + aad_len + src_len + dst_len + hash_result_len;
 456     if (unlikely(max_len > vcrypto->conf.max_size)) {
 457         virtio_error(vdev, "virtio-crypto too big length");
 458         return NULL;
 459     }
 460
 461     op_info = g_malloc0(sizeof(CryptoDevBackendSymOpInfo) + max_len);
 462     op_info->iv_len = iv_len;
 463     op_info->src_len = src_len;
 464     op_info->dst_len = dst_len;
 465     op_info->aad_len = aad_len;
 466     op_info->digest_result_len = hash_result_len;
 467     op_info->hash_start_src_offset = hash_start_src_offset;
 468     op_info->len_to_hash = len_to_hash;
 469     op_info->cipher_start_src_offset = cipher_start_src_offset;
 470     op_info->len_to_cipher = len_to_cipher;
 471     /* Handle the initilization vector */
 472     if (op_info->iv_len > 0) {
 473         DPRINTF("iv_len=%" PRIu32 "\n", op_info->iv_len);
 474         op_info->iv = op_info->data + curr_size;
 475
 476         s = iov_to_buf(iov, out_num, 0, op_info->iv, op_info->iv_len);
 477         if (unlikely(s != op_info->iv_len)) {
 478             virtio_error(vdev, "virtio-crypto iv incorrect");
 479             goto err;
 480         }
 481         iov_discard_front(&iov, &out_num, op_info->iv_len);
 482         curr_size += op_info->iv_len;
 483     }
 484
 485     /* Handle additional authentication data if exists */
 486     if (op_info->aad_len > 0) {
 487         DPRINTF("aad_len=%" PRIu32 "\n", op_info->aad_len);
 488         op_info->aad_data = op_info->data + curr_size;
 489
 490         s = iov_to_buf(iov, out_num, 0, op_info->aad_data, op_info->aad_len);
 491         if (unlikely(s != op_info->aad_len)) {
 492             virtio_error(vdev, "virtio-crypto additional auth data incorrect");
 493             goto err;
 494         }
 495         iov_discard_front(&iov, &out_num, op_info->aad_len);
 496
 497         curr_size += op_info->aad_len;
 498     }
 499
 500     /* Handle the source data */
 501     if (op_info->src_len > 0) {
 502         DPRINTF("src_len=%" PRIu32 "\n", op_info->src_len);
 503         op_info->src = op_info->data + curr_size;
 504
 505         s = iov_to_buf(iov, out_num, 0, op_info->src, op_info->src_len);
 506         if (unlikely(s != op_info->src_len)) {
 507             virtio_error(vdev, "virtio-crypto source data incorrect");
 508             goto err;
 509         }
 510         iov_discard_front(&iov, &out_num, op_info->src_len);
 511
 512         curr_size += op_info->src_len;
 513     }
 514
 515     /* Handle the destination data */
 516     op_info->dst = op_info->data + curr_size;
 517     curr_size += op_info->dst_len;
 518
 519     DPRINTF("dst_len=%" PRIu32 "\n", op_info->dst_len);
 520
 521     /* Handle the hash digest result */
 522     if (hash_result_len > 0) {
 523         DPRINTF("hash_result_len=%" PRIu32 "\n", hash_result_len);
 524         op_info->digest_result = op_info->data + curr_size;
 525     }
 526
 527     return op_info;
 528
 529 err:
 530     g_free(op_info);
 531     return NULL;
 532 }
 533
 534 static int
 535 virtio_crypto_handle_sym_req(VirtIOCrypto *vcrypto,
 536                struct virtio_crypto_sym_data_req *req,
 537                CryptoDevBackendSymOpInfo **sym_op_info,
 538                struct iovec *iov, unsigned int out_num)
 539 {
 540     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 541     uint32_t op_type;
 542     CryptoDevBackendSymOpInfo *op_info;
 543
 544     op_type = ldl_le_p(&req->op_type);
 545
 546     if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
 547         op_info = virtio_crypto_sym_op_helper(vdev, &req->u.cipher.para,
 548                                               NULL, iov, out_num);
 549         if (!op_info) {
 550             return -EFAULT;
 551         }
 552         op_info->op_type = op_type;
 553     } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
 554         op_info = virtio_crypto_sym_op_helper(vdev, NULL,
 555                                               &req->u.chain.para,
 556                                               iov, out_num);
 557         if (!op_info) {
 558             return -EFAULT;
 559         }
 560         op_info->op_type = op_type;
 561     } else {
 562         /* VIRTIO_CRYPTO_SYM_OP_NONE */
 563         error_report("virtio-crypto unsupported cipher type");
 564         return -VIRTIO_CRYPTO_NOTSUPP;
 565     }
 566
 567     *sym_op_info = op_info;
 568
 569     return 0;
 570 }
 571
 572 static int
 573 virtio_crypto_handle_request(VirtIOCryptoReq *request)
 574 {
 575     VirtIOCrypto *vcrypto = request->vcrypto;
 576     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 577     VirtQueueElement *elem = &request->elem;
 578     int queue_index = virtio_crypto_vq2q(virtio_get_queue_index(request->vq));
 579     struct virtio_crypto_op_data_req req;
 580     int ret;
 581     struct iovec *in_iov;
 582     struct iovec *out_iov;
 583     unsigned in_num;
 584     unsigned out_num;
 585     uint32_t opcode;
 586     uint8_t status = VIRTIO_CRYPTO_ERR;
 587     uint64_t session_id;
 588     CryptoDevBackendSymOpInfo *sym_op_info = NULL;
 589     Error *local_err = NULL;
 590
 591     if (elem->out_num < 1 || elem->in_num < 1) {
 592         virtio_error(vdev, "virtio-crypto dataq missing headers");
 593         return -1;
 594     }
 595
 596     out_num = elem->out_num;
 597     out_iov = elem->out_sg;
 598     in_num = elem->in_num;
 599     in_iov = elem->in_sg;
 600     if (unlikely(iov_to_buf(out_iov, out_num, 0, &req, sizeof(req))
 601                 != sizeof(req))) {
 602         virtio_error(vdev, "virtio-crypto request outhdr too short");
 603         return -1;
 604     }
 605     iov_discard_front(&out_iov, &out_num, sizeof(req));
 606
 607     if (in_iov[in_num - 1].iov_len <
 608             sizeof(struct virtio_crypto_inhdr)) {
 609         virtio_error(vdev, "virtio-crypto request inhdr too short");
 610         return -1;
 611     }
 612     /* We always touch the last byte, so just see how big in_iov is. */
 613     request->in_len = iov_size(in_iov, in_num);
 614     request->in = (void *)in_iov[in_num - 1].iov_base
 615               + in_iov[in_num - 1].iov_len
 616               - sizeof(struct virtio_crypto_inhdr);
 617     iov_discard_back(in_iov, &in_num, sizeof(struct virtio_crypto_inhdr));
 618
 619     /*
 620      * The length of operation result, including dest_data
 621      * and digest_result if exists.
 622      */
 623     request->in_num = in_num;
 624     request->in_iov = in_iov;
 625
 626     opcode = ldl_le_p(&req.header.opcode);
 627     session_id = ldq_le_p(&req.header.session_id);
 628
 629     switch (opcode) {
 630     case VIRTIO_CRYPTO_CIPHER_ENCRYPT:
 631     case VIRTIO_CRYPTO_CIPHER_DECRYPT:
 632         ret = virtio_crypto_handle_sym_req(vcrypto,
 633                          &req.u.sym_req,
 634                          &sym_op_info,
 635                          out_iov, out_num);
 636         /* Serious errors, need to reset virtio crypto device */
 637         if (ret == -EFAULT) {
 638             return -1;
 639         } else if (ret == -VIRTIO_CRYPTO_NOTSUPP) {
 640             virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP);
 641             virtio_crypto_free_request(request);
 642         } else {
 643             sym_op_info->session_id = session_id;
 644
 645             /* Set request's parameter */
 646             request->flags = CRYPTODEV_BACKEND_ALG_SYM;
 647             request->u.sym_op_info = sym_op_info;
 648             ret = cryptodev_backend_crypto_operation(vcrypto->cryptodev,
 649                                     request, queue_index, &local_err);
 650             if (ret < 0) {
 651                 status = -ret;
 652                 if (local_err) {
 653                     error_report_err(local_err);
 654                 }
 655             } else { /* ret == VIRTIO_CRYPTO_OK */
 656                 status = ret;
 657             }
 658             virtio_crypto_req_complete(request, status);
 659             virtio_crypto_free_request(request);
 660         }
 661         break;
 662     case VIRTIO_CRYPTO_HASH:
 663     case VIRTIO_CRYPTO_MAC:
 664     case VIRTIO_CRYPTO_AEAD_ENCRYPT:
 665     case VIRTIO_CRYPTO_AEAD_DECRYPT:
 666     default:
 667         error_report("virtio-crypto unsupported dataq opcode: %u",
 668                      opcode);
 669         virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP);
 670         virtio_crypto_free_request(request);
 671     }
 672
 673     return 0;
 674 }
 675
 676 static void virtio_crypto_handle_dataq(VirtIODevice *vdev, VirtQueue *vq)
 677 {
 678     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 679     VirtIOCryptoReq *req;
 680
 681     while ((req = virtio_crypto_get_request(vcrypto, vq))) {
 682         if (virtio_crypto_handle_request(req) < 0) {
 683             virtqueue_detach_element(req->vq, &req->elem, 0);
 684             virtio_crypto_free_request(req);
 685             break;
 686         }
 687     }
 688 }
 689
 690 static void virtio_crypto_dataq_bh(void *opaque)
 691 {
 692     VirtIOCryptoQueue *q = opaque;
 693     VirtIOCrypto *vcrypto = q->vcrypto;
 694     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 695
 696     /* This happens when device was stopped but BH wasn't. */
 697     if (!vdev->vm_running) {
 698         return;
 699     }
 700
 701     /* Just in case the driver is not ready on more */
 702     if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) {
 703         return;
 704     }
 705
 706     for (;;) {
 707         virtio_crypto_handle_dataq(vdev, q->dataq);
 708         virtio_queue_set_notification(q->dataq, 1);
 709
 710         /* Are we done or did the guest add more buffers? */
 711         if (virtio_queue_empty(q->dataq)) {
 712             break;
 713         }
 714
 715         virtio_queue_set_notification(q->dataq, 0);
 716     }
 717 }
 718
 719 static void
 720 virtio_crypto_handle_dataq_bh(VirtIODevice *vdev, VirtQueue *vq)
 721 {
 722     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 723     VirtIOCryptoQueue *q =
 724          &vcrypto->vqs[virtio_crypto_vq2q(virtio_get_queue_index(vq))];
 725
 726     /* This happens when device was stopped but VCPU wasn't. */
 727     if (!vdev->vm_running) {
 728         return;
 729     }
 730     virtio_queue_set_notification(vq, 0);
 731     qemu_bh_schedule(q->dataq_bh);
 732 }
 733
 734 static uint64_t virtio_crypto_get_features(VirtIODevice *vdev,
 735                                            uint64_t features,
 736                                            Error **errp)
 737 {
 738     return features;
 739 }
 740
 741 static void virtio_crypto_reset(VirtIODevice *vdev)
 742 {
 743     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 744     /* multiqueue is disabled by default */
 745     vcrypto->curr_queues = 1;
 746     if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) {
 747         vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY;
 748     } else {
 749         vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY;
 750     }
 751 }
 752
 753 static void virtio_crypto_init_config(VirtIODevice *vdev)
 754 {
 755     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 756
 757     vcrypto->conf.crypto_services =
 758                      vcrypto->conf.cryptodev->conf.crypto_services;
 759     vcrypto->conf.cipher_algo_l =
 760                      vcrypto->conf.cryptodev->conf.cipher_algo_l;
 761     vcrypto->conf.cipher_algo_h =
 762                      vcrypto->conf.cryptodev->conf.cipher_algo_h;
 763     vcrypto->conf.hash_algo = vcrypto->conf.cryptodev->conf.hash_algo;
 764     vcrypto->conf.mac_algo_l = vcrypto->conf.cryptodev->conf.mac_algo_l;
 765     vcrypto->conf.mac_algo_h = vcrypto->conf.cryptodev->conf.mac_algo_h;
 766     vcrypto->conf.aead_algo = vcrypto->conf.cryptodev->conf.aead_algo;
 767     vcrypto->conf.max_cipher_key_len =
 768                   vcrypto->conf.cryptodev->conf.max_cipher_key_len;
 769     vcrypto->conf.max_auth_key_len =
 770                   vcrypto->conf.cryptodev->conf.max_auth_key_len;
 771     vcrypto->conf.max_size = vcrypto->conf.cryptodev->conf.max_size;
 772 }
 773
 774 static void virtio_crypto_device_realize(DeviceState *dev, Error **errp)
 775 {
 776     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 777     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev);
 778     int i;
 779
 780     vcrypto->cryptodev = vcrypto->conf.cryptodev;
 781     if (vcrypto->cryptodev == NULL) {
 782         error_setg(errp, "'cryptodev' parameter expects a valid object");
 783         return;
 784     }
 785
 786     vcrypto->max_queues = MAX(vcrypto->cryptodev->conf.peers.queues, 1);
 787     if (vcrypto->max_queues + 1 > VIRTIO_QUEUE_MAX) {
 788         error_setg(errp, "Invalid number of queues (= %" PRIu32 "), "
 789                    "must be a positive integer less than %d.",
 790                    vcrypto->max_queues, VIRTIO_QUEUE_MAX);
 791         return;
 792     }
 793
 794     virtio_init(vdev, "virtio-crypto", VIRTIO_ID_CRYPTO, vcrypto->config_size);
 795     vcrypto->curr_queues = 1;
 796     vcrypto->vqs = g_malloc0(sizeof(VirtIOCryptoQueue) * vcrypto->max_queues);
 797     for (i = 0; i < vcrypto->max_queues; i++) {
 798         vcrypto->vqs[i].dataq =
 799                  virtio_add_queue(vdev, 1024, virtio_crypto_handle_dataq_bh);
 800         vcrypto->vqs[i].dataq_bh =
 801                  qemu_bh_new(virtio_crypto_dataq_bh, &vcrypto->vqs[i]);
 802         vcrypto->vqs[i].vcrypto = vcrypto;
 803     }
 804
 805     vcrypto->ctrl_vq = virtio_add_queue(vdev, 64, virtio_crypto_handle_ctrl);
 806     if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) {
 807         vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY;
 808     } else {
 809         vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY;
 810     }
 811
 812     virtio_crypto_init_config(vdev);
 813     cryptodev_backend_set_used(vcrypto->cryptodev, true);
 814 }
 815
 816 static void virtio_crypto_device_unrealize(DeviceState *dev, Error **errp)
 817 {
 818     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 819     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev);
 820     VirtIOCryptoQueue *q;
 821     int i, max_queues;
 822
 823     max_queues = vcrypto->multiqueue ? vcrypto->max_queues : 1;
 824     for (i = 0; i < max_queues; i++) {
 825         virtio_del_queue(vdev, i);
 826         q = &vcrypto->vqs[i];
 827         qemu_bh_delete(q->dataq_bh);
 828     }
 829
 830     g_free(vcrypto->vqs);
 831
 832     virtio_cleanup(vdev);
 833     cryptodev_backend_set_used(vcrypto->cryptodev, false);
 834 }
 835
 836 static const VMStateDescription vmstate_virtio_crypto = {
 837     .name = "virtio-crypto",
 838     .unmigratable = 1,
 839     .minimum_version_id = VIRTIO_CRYPTO_VM_VERSION,
 840     .version_id = VIRTIO_CRYPTO_VM_VERSION,
 841     .fields = (VMStateField[]) {
 842         VMSTATE_VIRTIO_DEVICE,
 843         VMSTATE_END_OF_LIST()
 844     },
 845 };
 846
 847 static Property virtio_crypto_properties[] = {
 848     DEFINE_PROP_END_OF_LIST(),
 849 };
 850
 851 static void virtio_crypto_get_config(VirtIODevice *vdev, uint8_t *config)
 852 {
 853     VirtIOCrypto *c = VIRTIO_CRYPTO(vdev);
 854     struct virtio_crypto_config crypto_cfg = {};
 855
 856     /*
 857      * Virtio-crypto device conforms to VIRTIO 1.0 which is always LE,
 858      * so we can use LE accessors directly.
 859      */
 860     stl_le_p(&crypto_cfg.status, c->status);
 861     stl_le_p(&crypto_cfg.max_dataqueues, c->max_queues);
 862     stl_le_p(&crypto_cfg.crypto_services, c->conf.crypto_services);
 863     stl_le_p(&crypto_cfg.cipher_algo_l, c->conf.cipher_algo_l);
 864     stl_le_p(&crypto_cfg.cipher_algo_h, c->conf.cipher_algo_h);
 865     stl_le_p(&crypto_cfg.hash_algo, c->conf.hash_algo);
 866     stl_le_p(&crypto_cfg.mac_algo_l, c->conf.mac_algo_l);
 867     stl_le_p(&crypto_cfg.mac_algo_h, c->conf.mac_algo_h);
 868     stl_le_p(&crypto_cfg.aead_algo, c->conf.aead_algo);
 869     stl_le_p(&crypto_cfg.max_cipher_key_len, c->conf.max_cipher_key_len);
 870     stl_le_p(&crypto_cfg.max_auth_key_len, c->conf.max_auth_key_len);
 871     stq_le_p(&crypto_cfg.max_size, c->conf.max_size);
 872
 873     memcpy(config, &crypto_cfg, c->config_size);
 874 }
 875
 876 static void virtio_crypto_class_init(ObjectClass *klass, void *data)
 877 {
 878     DeviceClass *dc = DEVICE_CLASS(klass);
 879     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
 880
 881     dc->props = virtio_crypto_properties;
 882     dc->vmsd = &vmstate_virtio_crypto;
 883     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 884     vdc->realize = virtio_crypto_device_realize;
 885     vdc->unrealize = virtio_crypto_device_unrealize;
 886     vdc->get_config = virtio_crypto_get_config;
 887     vdc->get_features = virtio_crypto_get_features;
 888     vdc->reset = virtio_crypto_reset;
 889 }
 890
 891 static void
 892 virtio_crypto_check_cryptodev_is_used(Object *obj, const char *name,
 893                                       Object *val, Error **errp)
 894 {
 895     if (cryptodev_backend_is_used(CRYPTODEV_BACKEND(val))) {
 896         char *path = object_get_canonical_path_component(val);
 897         error_setg(errp,
 898             "can't use already used cryptodev backend: %s", path);
 899         g_free(path);
 900     } else {
 901         qdev_prop_allow_set_link_before_realize(obj, name, val, errp);
 902     }
 903 }
 904
 905 static void virtio_crypto_instance_init(Object *obj)
 906 {
 907     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(obj);
 908
 909     /*
 910      * The default config_size is sizeof(struct virtio_crypto_config).
 911      * Can be overriden with virtio_crypto_set_config_size.
 912      */
 913     vcrypto->config_size = sizeof(struct virtio_crypto_config);
 914
 915     object_property_add_link(obj, "cryptodev",
 916                              TYPE_CRYPTODEV_BACKEND,
 917                              (Object **)&vcrypto->conf.cryptodev,
 918                              virtio_crypto_check_cryptodev_is_used,
 919                              OBJ_PROP_LINK_UNREF_ON_RELEASE, NULL);
 920 }
 921
 922 static const TypeInfo virtio_crypto_info = {
 923     .name = TYPE_VIRTIO_CRYPTO,
 924     .parent = TYPE_VIRTIO_DEVICE,
 925     .instance_size = sizeof(VirtIOCrypto),
 926     .instance_init = virtio_crypto_instance_init,
 927     .class_init = virtio_crypto_class_init,
 928 };
 929
 930 static void virtio_register_types(void)
 931 {
 932     type_register_static(&virtio_crypto_info);
 933 }
 934
 935 type_init(virtio_register_types)