util/bufferiszero: Add simd acceleration for aarch64
[qemu/armbru.git] / backends / cryptodev-lkcf.c
blob45aba1ff67b895a9c22658e7fd9577cfa5bb6264
1 /*
2 * QEMU Cryptodev backend for QEMU cipher APIs
4 * Copyright (c) 2022 Bytedance.Inc
6 * Authors:
7 * lei he <helei.sig11@bytedance.com>
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License as published by the Free Software Foundation; either
12 * version 2.1 of the License, or (at your option) any later version.
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
24 #include "qemu/osdep.h"
25 #include "crypto/cipher.h"
26 #include "crypto/akcipher.h"
27 #include "qapi/error.h"
28 #include "qemu/main-loop.h"
29 #include "qemu/thread.h"
30 #include "qemu/error-report.h"
31 #include "qemu/queue.h"
32 #include "qom/object.h"
33 #include "sysemu/cryptodev.h"
34 #include "standard-headers/linux/virtio_crypto.h"
36 #include <keyutils.h>
37 #include <sys/eventfd.h>
39 /**
40 * @TYPE_CRYPTODEV_BACKEND_LKCF:
41 * name of backend that uses linux kernel crypto framework
43 #define TYPE_CRYPTODEV_BACKEND_LKCF "cryptodev-backend-lkcf"
45 OBJECT_DECLARE_SIMPLE_TYPE(CryptoDevBackendLKCF, CRYPTODEV_BACKEND_LKCF)
47 #define INVALID_KEY_ID -1
48 #define MAX_SESSIONS 256
49 #define NR_WORKER_THREAD 64
51 #define KCTL_KEY_TYPE_PKEY "asymmetric"
52 /**
53 * Here the key is uploaded to the thread-keyring of worker thread, at least
54 * util linux-6.0:
55 * 1. process keyring seems to behave unexpectedly if main-thread does not
56 * create the keyring before creating any other thread.
57 * 2. at present, the guest kernel never perform multiple operations on a
58 * session.
59 * 3. it can reduce the load of the main-loop because the key passed by the
60 * guest kernel has been already checked.
62 #define KCTL_KEY_RING KEY_SPEC_THREAD_KEYRING
64 typedef struct CryptoDevBackendLKCFSession {
65 uint8_t *key;
66 size_t keylen;
67 QCryptoAkCipherKeyType keytype;
68 QCryptoAkCipherOptions akcipher_opts;
69 } CryptoDevBackendLKCFSession;
71 typedef struct CryptoDevBackendLKCF CryptoDevBackendLKCF;
72 typedef struct CryptoDevLKCFTask CryptoDevLKCFTask;
73 struct CryptoDevLKCFTask {
74 CryptoDevBackendLKCFSession *sess;
75 CryptoDevBackendOpInfo *op_info;
76 CryptoDevCompletionFunc cb;
77 void *opaque;
78 int status;
79 CryptoDevBackendLKCF *lkcf;
80 QSIMPLEQ_ENTRY(CryptoDevLKCFTask) queue;
83 typedef struct CryptoDevBackendLKCF {
84 CryptoDevBackend parent_obj;
85 CryptoDevBackendLKCFSession *sess[MAX_SESSIONS];
86 QSIMPLEQ_HEAD(, CryptoDevLKCFTask) requests;
87 QSIMPLEQ_HEAD(, CryptoDevLKCFTask) responses;
88 QemuMutex mutex;
89 QemuCond cond;
90 QemuMutex rsp_mutex;
92 /**
93 * There is no async interface for asymmetric keys like AF_ALG sockets,
94 * we don't seem to have better way than create a lots of thread.
96 QemuThread worker_threads[NR_WORKER_THREAD];
97 bool running;
98 int eventfd;
99 } CryptoDevBackendLKCF;
101 static void *cryptodev_lkcf_worker(void *arg);
102 static int cryptodev_lkcf_close_session(CryptoDevBackend *backend,
103 uint64_t session_id,
104 uint32_t queue_index,
105 CryptoDevCompletionFunc cb,
106 void *opaque);
108 static void cryptodev_lkcf_handle_response(void *opaque)
110 CryptoDevBackendLKCF *lkcf = (CryptoDevBackendLKCF *)opaque;
111 QSIMPLEQ_HEAD(, CryptoDevLKCFTask) responses;
112 CryptoDevLKCFTask *task, *next;
113 eventfd_t nevent;
115 QSIMPLEQ_INIT(&responses);
116 eventfd_read(lkcf->eventfd, &nevent);
118 qemu_mutex_lock(&lkcf->rsp_mutex);
119 QSIMPLEQ_PREPEND(&responses, &lkcf->responses);
120 qemu_mutex_unlock(&lkcf->rsp_mutex);
122 QSIMPLEQ_FOREACH_SAFE(task, &responses, queue, next) {
123 if (task->cb) {
124 task->cb(task->opaque, task->status);
126 g_free(task);
130 static int cryptodev_lkcf_set_op_desc(QCryptoAkCipherOptions *opts,
131 char *key_desc,
132 size_t desc_len,
133 Error **errp)
135 QCryptoAkCipherOptionsRSA *rsa_opt;
136 if (opts->alg != QCRYPTO_AKCIPHER_ALG_RSA) {
137 error_setg(errp, "Unsupported alg: %u", opts->alg);
138 return -1;
141 rsa_opt = &opts->u.rsa;
142 if (rsa_opt->padding_alg == QCRYPTO_RSA_PADDING_ALG_PKCS1) {
143 snprintf(key_desc, desc_len, "enc=%s hash=%s",
144 QCryptoRSAPaddingAlgorithm_str(rsa_opt->padding_alg),
145 QCryptoHashAlgorithm_str(rsa_opt->hash_alg));
147 } else {
148 snprintf(key_desc, desc_len, "enc=%s",
149 QCryptoRSAPaddingAlgorithm_str(rsa_opt->padding_alg));
151 return 0;
154 static int cryptodev_lkcf_set_rsa_opt(int virtio_padding_alg,
155 int virtio_hash_alg,
156 QCryptoAkCipherOptionsRSA *opt,
157 Error **errp)
159 if (virtio_padding_alg == VIRTIO_CRYPTO_RSA_PKCS1_PADDING) {
160 opt->padding_alg = QCRYPTO_RSA_PADDING_ALG_PKCS1;
162 switch (virtio_hash_alg) {
163 case VIRTIO_CRYPTO_RSA_MD5:
164 opt->hash_alg = QCRYPTO_HASH_ALG_MD5;
165 break;
167 case VIRTIO_CRYPTO_RSA_SHA1:
168 opt->hash_alg = QCRYPTO_HASH_ALG_SHA1;
169 break;
171 case VIRTIO_CRYPTO_RSA_SHA256:
172 opt->hash_alg = QCRYPTO_HASH_ALG_SHA256;
173 break;
175 case VIRTIO_CRYPTO_RSA_SHA512:
176 opt->hash_alg = QCRYPTO_HASH_ALG_SHA512;
177 break;
179 default:
180 error_setg(errp, "Unsupported rsa hash algo: %d", virtio_hash_alg);
181 return -1;
183 return 0;
186 if (virtio_padding_alg == VIRTIO_CRYPTO_RSA_RAW_PADDING) {
187 opt->padding_alg = QCRYPTO_RSA_PADDING_ALG_RAW;
188 return 0;
191 error_setg(errp, "Unsupported rsa padding algo: %u", virtio_padding_alg);
192 return -1;
195 static int cryptodev_lkcf_get_unused_session_index(CryptoDevBackendLKCF *lkcf)
197 size_t i;
199 for (i = 0; i < MAX_SESSIONS; i++) {
200 if (lkcf->sess[i] == NULL) {
201 return i;
204 return -1;
207 static void cryptodev_lkcf_init(CryptoDevBackend *backend, Error **errp)
209 /* Only support one queue */
210 int queues = backend->conf.peers.queues, i;
211 CryptoDevBackendClient *cc;
212 CryptoDevBackendLKCF *lkcf =
213 CRYPTODEV_BACKEND_LKCF(backend);
215 if (queues != 1) {
216 error_setg(errp,
217 "Only support one queue in cryptodev-builtin backend");
218 return;
220 lkcf->eventfd = eventfd(0, 0);
221 if (lkcf->eventfd < 0) {
222 error_setg(errp, "Failed to create eventfd: %d", errno);
223 return;
226 cc = cryptodev_backend_new_client();
227 cc->info_str = g_strdup_printf("cryptodev-lkcf0");
228 cc->queue_index = 0;
229 cc->type = QCRYPTODEV_BACKEND_TYPE_LKCF;
230 backend->conf.peers.ccs[0] = cc;
232 backend->conf.crypto_services =
233 1u << QCRYPTODEV_BACKEND_SERVICE_AKCIPHER;
234 backend->conf.akcipher_algo = 1u << VIRTIO_CRYPTO_AKCIPHER_RSA;
235 lkcf->running = true;
237 QSIMPLEQ_INIT(&lkcf->requests);
238 QSIMPLEQ_INIT(&lkcf->responses);
239 qemu_mutex_init(&lkcf->mutex);
240 qemu_mutex_init(&lkcf->rsp_mutex);
241 qemu_cond_init(&lkcf->cond);
242 for (i = 0; i < NR_WORKER_THREAD; i++) {
243 qemu_thread_create(&lkcf->worker_threads[i], "lkcf-worker",
244 cryptodev_lkcf_worker, lkcf, 0);
246 qemu_set_fd_handler(
247 lkcf->eventfd, cryptodev_lkcf_handle_response, NULL, lkcf);
248 cryptodev_backend_set_ready(backend, true);
251 static void cryptodev_lkcf_cleanup(CryptoDevBackend *backend, Error **errp)
253 CryptoDevBackendLKCF *lkcf = CRYPTODEV_BACKEND_LKCF(backend);
254 size_t i;
255 int queues = backend->conf.peers.queues;
256 CryptoDevBackendClient *cc;
257 CryptoDevLKCFTask *task, *next;
259 qemu_mutex_lock(&lkcf->mutex);
260 lkcf->running = false;
261 qemu_mutex_unlock(&lkcf->mutex);
262 qemu_cond_broadcast(&lkcf->cond);
264 close(lkcf->eventfd);
265 for (i = 0; i < NR_WORKER_THREAD; i++) {
266 qemu_thread_join(&lkcf->worker_threads[i]);
269 QSIMPLEQ_FOREACH_SAFE(task, &lkcf->requests, queue, next) {
270 if (task->cb) {
271 task->cb(task->opaque, task->status);
273 g_free(task);
276 QSIMPLEQ_FOREACH_SAFE(task, &lkcf->responses, queue, next) {
277 if (task->cb) {
278 task->cb(task->opaque, task->status);
280 g_free(task);
283 qemu_mutex_destroy(&lkcf->mutex);
284 qemu_cond_destroy(&lkcf->cond);
285 qemu_mutex_destroy(&lkcf->rsp_mutex);
287 for (i = 0; i < MAX_SESSIONS; i++) {
288 if (lkcf->sess[i] != NULL) {
289 cryptodev_lkcf_close_session(backend, i, 0, NULL, NULL);
293 for (i = 0; i < queues; i++) {
294 cc = backend->conf.peers.ccs[i];
295 if (cc) {
296 cryptodev_backend_free_client(cc);
297 backend->conf.peers.ccs[i] = NULL;
301 cryptodev_backend_set_ready(backend, false);
304 static void cryptodev_lkcf_execute_task(CryptoDevLKCFTask *task)
306 CryptoDevBackendLKCFSession *session = task->sess;
307 CryptoDevBackendAsymOpInfo *asym_op_info;
308 bool kick = false;
309 int ret, status, op_code = task->op_info->op_code;
310 size_t p8info_len;
311 g_autofree uint8_t *p8info = NULL;
312 Error *local_error = NULL;
313 key_serial_t key_id = INVALID_KEY_ID;
314 char op_desc[64];
315 g_autoptr(QCryptoAkCipher) akcipher = NULL;
318 * We only offload private key session:
319 * 1. currently, the Linux kernel can only accept public key wrapped
320 * with X.509 certificates, but unfortunately the cost of making a
321 * ceritificate with public key is too expensive.
322 * 2. generally, public key related compution is fast, just compute it with
323 * thread-pool.
325 if (session->keytype == QCRYPTO_AKCIPHER_KEY_TYPE_PRIVATE) {
326 if (qcrypto_akcipher_export_p8info(&session->akcipher_opts,
327 session->key, session->keylen,
328 &p8info, &p8info_len,
329 &local_error) != 0 ||
330 cryptodev_lkcf_set_op_desc(&session->akcipher_opts, op_desc,
331 sizeof(op_desc), &local_error) != 0) {
332 error_report_err(local_error);
333 } else {
334 key_id = add_key(KCTL_KEY_TYPE_PKEY, "lkcf-backend-priv-key",
335 p8info, p8info_len, KCTL_KEY_RING);
339 if (key_id < 0) {
340 if (!qcrypto_akcipher_supports(&session->akcipher_opts)) {
341 status = -VIRTIO_CRYPTO_NOTSUPP;
342 goto out;
344 akcipher = qcrypto_akcipher_new(&session->akcipher_opts,
345 session->keytype,
346 session->key, session->keylen,
347 &local_error);
348 if (!akcipher) {
349 status = -VIRTIO_CRYPTO_ERR;
350 goto out;
354 asym_op_info = task->op_info->u.asym_op_info;
355 switch (op_code) {
356 case VIRTIO_CRYPTO_AKCIPHER_ENCRYPT:
357 if (key_id >= 0) {
358 ret = keyctl_pkey_encrypt(key_id, op_desc,
359 asym_op_info->src, asym_op_info->src_len,
360 asym_op_info->dst, asym_op_info->dst_len);
361 } else {
362 ret = qcrypto_akcipher_encrypt(akcipher,
363 asym_op_info->src, asym_op_info->src_len,
364 asym_op_info->dst, asym_op_info->dst_len, &local_error);
366 break;
368 case VIRTIO_CRYPTO_AKCIPHER_DECRYPT:
369 if (key_id >= 0) {
370 ret = keyctl_pkey_decrypt(key_id, op_desc,
371 asym_op_info->src, asym_op_info->src_len,
372 asym_op_info->dst, asym_op_info->dst_len);
373 } else {
374 ret = qcrypto_akcipher_decrypt(akcipher,
375 asym_op_info->src, asym_op_info->src_len,
376 asym_op_info->dst, asym_op_info->dst_len, &local_error);
378 break;
380 case VIRTIO_CRYPTO_AKCIPHER_SIGN:
381 if (key_id >= 0) {
382 ret = keyctl_pkey_sign(key_id, op_desc,
383 asym_op_info->src, asym_op_info->src_len,
384 asym_op_info->dst, asym_op_info->dst_len);
385 } else {
386 ret = qcrypto_akcipher_sign(akcipher,
387 asym_op_info->src, asym_op_info->src_len,
388 asym_op_info->dst, asym_op_info->dst_len, &local_error);
390 break;
392 case VIRTIO_CRYPTO_AKCIPHER_VERIFY:
393 if (key_id >= 0) {
394 ret = keyctl_pkey_verify(key_id, op_desc,
395 asym_op_info->src, asym_op_info->src_len,
396 asym_op_info->dst, asym_op_info->dst_len);
397 } else {
398 ret = qcrypto_akcipher_verify(akcipher,
399 asym_op_info->src, asym_op_info->src_len,
400 asym_op_info->dst, asym_op_info->dst_len, &local_error);
402 break;
404 default:
405 error_setg(&local_error, "Unknown opcode: %u", op_code);
406 status = -VIRTIO_CRYPTO_ERR;
407 goto out;
410 if (ret < 0) {
411 if (!local_error) {
412 if (errno != EKEYREJECTED) {
413 error_report("Failed do operation with keyctl: %d", errno);
415 } else {
416 error_report_err(local_error);
418 status = op_code == VIRTIO_CRYPTO_AKCIPHER_VERIFY ?
419 -VIRTIO_CRYPTO_KEY_REJECTED : -VIRTIO_CRYPTO_ERR;
420 } else {
421 status = VIRTIO_CRYPTO_OK;
422 asym_op_info->dst_len = ret;
425 out:
426 if (key_id >= 0) {
427 keyctl_unlink(key_id, KCTL_KEY_RING);
429 task->status = status;
431 qemu_mutex_lock(&task->lkcf->rsp_mutex);
432 if (QSIMPLEQ_EMPTY(&task->lkcf->responses)) {
433 kick = true;
435 QSIMPLEQ_INSERT_TAIL(&task->lkcf->responses, task, queue);
436 qemu_mutex_unlock(&task->lkcf->rsp_mutex);
438 if (kick) {
439 eventfd_write(task->lkcf->eventfd, 1);
443 static void *cryptodev_lkcf_worker(void *arg)
445 CryptoDevBackendLKCF *backend = (CryptoDevBackendLKCF *)arg;
446 CryptoDevLKCFTask *task;
448 for (;;) {
449 task = NULL;
450 qemu_mutex_lock(&backend->mutex);
451 while (backend->running && QSIMPLEQ_EMPTY(&backend->requests)) {
452 qemu_cond_wait(&backend->cond, &backend->mutex);
454 if (backend->running) {
455 task = QSIMPLEQ_FIRST(&backend->requests);
456 QSIMPLEQ_REMOVE_HEAD(&backend->requests, queue);
458 qemu_mutex_unlock(&backend->mutex);
460 /* stopped */
461 if (!task) {
462 break;
464 cryptodev_lkcf_execute_task(task);
467 return NULL;
470 static int cryptodev_lkcf_operation(
471 CryptoDevBackend *backend,
472 CryptoDevBackendOpInfo *op_info)
474 CryptoDevBackendLKCF *lkcf =
475 CRYPTODEV_BACKEND_LKCF(backend);
476 CryptoDevBackendLKCFSession *sess;
477 QCryptodevBackendAlgType algtype = op_info->algtype;
478 CryptoDevLKCFTask *task;
480 if (op_info->session_id >= MAX_SESSIONS ||
481 lkcf->sess[op_info->session_id] == NULL) {
482 error_report("Cannot find a valid session id: %" PRIu64 "",
483 op_info->session_id);
484 return -VIRTIO_CRYPTO_INVSESS;
487 sess = lkcf->sess[op_info->session_id];
488 if (algtype != QCRYPTODEV_BACKEND_ALG_ASYM) {
489 error_report("algtype not supported: %u", algtype);
490 return -VIRTIO_CRYPTO_NOTSUPP;
493 task = g_new0(CryptoDevLKCFTask, 1);
494 task->op_info = op_info;
495 task->cb = op_info->cb;
496 task->opaque = op_info->opaque;
497 task->sess = sess;
498 task->lkcf = lkcf;
499 task->status = -VIRTIO_CRYPTO_ERR;
501 qemu_mutex_lock(&lkcf->mutex);
502 QSIMPLEQ_INSERT_TAIL(&lkcf->requests, task, queue);
503 qemu_mutex_unlock(&lkcf->mutex);
504 qemu_cond_signal(&lkcf->cond);
506 return VIRTIO_CRYPTO_OK;
509 static int cryptodev_lkcf_create_asym_session(
510 CryptoDevBackendLKCF *lkcf,
511 CryptoDevBackendAsymSessionInfo *sess_info,
512 uint64_t *session_id)
514 Error *local_error = NULL;
515 int index;
516 g_autofree CryptoDevBackendLKCFSession *sess =
517 g_new0(CryptoDevBackendLKCFSession, 1);
519 switch (sess_info->algo) {
520 case VIRTIO_CRYPTO_AKCIPHER_RSA:
521 sess->akcipher_opts.alg = QCRYPTO_AKCIPHER_ALG_RSA;
522 if (cryptodev_lkcf_set_rsa_opt(
523 sess_info->u.rsa.padding_algo, sess_info->u.rsa.hash_algo,
524 &sess->akcipher_opts.u.rsa, &local_error) != 0) {
525 error_report_err(local_error);
526 return -VIRTIO_CRYPTO_ERR;
528 break;
530 default:
531 error_report("Unsupported asym alg %u", sess_info->algo);
532 return -VIRTIO_CRYPTO_NOTSUPP;
535 switch (sess_info->keytype) {
536 case VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PUBLIC:
537 sess->keytype = QCRYPTO_AKCIPHER_KEY_TYPE_PUBLIC;
538 break;
540 case VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PRIVATE:
541 sess->keytype = QCRYPTO_AKCIPHER_KEY_TYPE_PRIVATE;
542 break;
544 default:
545 error_report("Unknown akcipher keytype: %u", sess_info->keytype);
546 return -VIRTIO_CRYPTO_ERR;
549 index = cryptodev_lkcf_get_unused_session_index(lkcf);
550 if (index < 0) {
551 error_report("Total number of sessions created exceeds %u",
552 MAX_SESSIONS);
553 return -VIRTIO_CRYPTO_ERR;
556 sess->keylen = sess_info->keylen;
557 sess->key = g_malloc(sess_info->keylen);
558 memcpy(sess->key, sess_info->key, sess_info->keylen);
560 lkcf->sess[index] = g_steal_pointer(&sess);
561 *session_id = index;
563 return VIRTIO_CRYPTO_OK;
566 static int cryptodev_lkcf_create_session(
567 CryptoDevBackend *backend,
568 CryptoDevBackendSessionInfo *sess_info,
569 uint32_t queue_index,
570 CryptoDevCompletionFunc cb,
571 void *opaque)
573 CryptoDevBackendAsymSessionInfo *asym_sess_info;
574 CryptoDevBackendLKCF *lkcf =
575 CRYPTODEV_BACKEND_LKCF(backend);
576 int ret;
578 switch (sess_info->op_code) {
579 case VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION:
580 asym_sess_info = &sess_info->u.asym_sess_info;
581 ret = cryptodev_lkcf_create_asym_session(
582 lkcf, asym_sess_info, &sess_info->session_id);
583 break;
585 default:
586 ret = -VIRTIO_CRYPTO_NOTSUPP;
587 error_report("Unsupported opcode: %" PRIu32 "",
588 sess_info->op_code);
589 break;
591 if (cb) {
592 cb(opaque, ret);
594 return 0;
597 static int cryptodev_lkcf_close_session(CryptoDevBackend *backend,
598 uint64_t session_id,
599 uint32_t queue_index,
600 CryptoDevCompletionFunc cb,
601 void *opaque)
603 CryptoDevBackendLKCF *lkcf = CRYPTODEV_BACKEND_LKCF(backend);
604 CryptoDevBackendLKCFSession *session;
606 assert(session_id < MAX_SESSIONS && lkcf->sess[session_id]);
607 session = lkcf->sess[session_id];
608 lkcf->sess[session_id] = NULL;
610 g_free(session->key);
611 g_free(session);
613 if (cb) {
614 cb(opaque, VIRTIO_CRYPTO_OK);
616 return 0;
619 static void cryptodev_lkcf_class_init(ObjectClass *oc, void *data)
621 CryptoDevBackendClass *bc = CRYPTODEV_BACKEND_CLASS(oc);
623 bc->init = cryptodev_lkcf_init;
624 bc->cleanup = cryptodev_lkcf_cleanup;
625 bc->create_session = cryptodev_lkcf_create_session;
626 bc->close_session = cryptodev_lkcf_close_session;
627 bc->do_op = cryptodev_lkcf_operation;
630 static const TypeInfo cryptodev_builtin_info = {
631 .name = TYPE_CRYPTODEV_BACKEND_LKCF,
632 .parent = TYPE_CRYPTODEV_BACKEND,
633 .class_init = cryptodev_lkcf_class_init,
634 .instance_size = sizeof(CryptoDevBackendLKCF),
637 static void cryptodev_lkcf_register_types(void)
639 type_register_static(&cryptodev_builtin_info);
642 type_init(cryptodev_lkcf_register_types);