2 * algif_aead: User-space interface for AEAD algorithms
4 * Copyright (C) 2014, Stephan Mueller <smueller@chronox.de>
6 * This file provides the user-space API for AEAD ciphers.
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
13 * The following concept of the memory management is used:
15 * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is
16 * filled by user space with the data submitted via sendpage/sendmsg. Filling
17 * up the TX SGL does not cause a crypto operation -- the data will only be
18 * tracked by the kernel. Upon receipt of one recvmsg call, the caller must
19 * provide a buffer which is tracked with the RX SGL.
21 * During the processing of the recvmsg operation, the cipher request is
22 * allocated and prepared. As part of the recvmsg operation, the processed
23 * TX buffers are extracted from the TX SGL into a separate SGL.
25 * After the completion of the crypto operation, the RX SGL and the cipher
26 * request is released. The extracted TX SGL parts are released together with
30 #include <crypto/internal/aead.h>
31 #include <crypto/scatterwalk.h>
32 #include <crypto/if_alg.h>
33 #include <crypto/skcipher.h>
34 #include <crypto/null.h>
35 #include <linux/init.h>
36 #include <linux/list.h>
37 #include <linux/kernel.h>
39 #include <linux/module.h>
40 #include <linux/net.h>
44 struct crypto_aead
*aead
;
46 struct crypto_skcipher
*null_tfm
;
49 static inline bool aead_sufficient_data(struct sock
*sk
)
51 struct alg_sock
*ask
= alg_sk(sk
);
52 struct sock
*psk
= ask
->parent
;
53 struct alg_sock
*pask
= alg_sk(psk
);
54 struct af_alg_ctx
*ctx
= ask
->private;
55 struct aead_tfm
*aeadc
= pask
->private;
56 struct crypto_aead
*tfm
= aeadc
->aead
;
57 unsigned int as
= crypto_aead_authsize(tfm
);
60 * The minimum amount of memory needed for an AEAD cipher is
61 * the AAD and in case of decryption the tag.
63 return ctx
->used
>= ctx
->aead_assoclen
+ (ctx
->enc
? 0 : as
);
66 static int aead_sendmsg(struct socket
*sock
, struct msghdr
*msg
, size_t size
)
68 struct sock
*sk
= sock
->sk
;
69 struct alg_sock
*ask
= alg_sk(sk
);
70 struct sock
*psk
= ask
->parent
;
71 struct alg_sock
*pask
= alg_sk(psk
);
72 struct aead_tfm
*aeadc
= pask
->private;
73 struct crypto_aead
*tfm
= aeadc
->aead
;
74 unsigned int ivsize
= crypto_aead_ivsize(tfm
);
76 return af_alg_sendmsg(sock
, msg
, size
, ivsize
);
79 static int crypto_aead_copy_sgl(struct crypto_skcipher
*null_tfm
,
80 struct scatterlist
*src
,
81 struct scatterlist
*dst
, unsigned int len
)
83 SKCIPHER_REQUEST_ON_STACK(skreq
, null_tfm
);
85 skcipher_request_set_tfm(skreq
, null_tfm
);
86 skcipher_request_set_callback(skreq
, CRYPTO_TFM_REQ_MAY_BACKLOG
,
88 skcipher_request_set_crypt(skreq
, src
, dst
, len
, NULL
);
90 return crypto_skcipher_encrypt(skreq
);
93 static int _aead_recvmsg(struct socket
*sock
, struct msghdr
*msg
,
94 size_t ignored
, int flags
)
96 struct sock
*sk
= sock
->sk
;
97 struct alg_sock
*ask
= alg_sk(sk
);
98 struct sock
*psk
= ask
->parent
;
99 struct alg_sock
*pask
= alg_sk(psk
);
100 struct af_alg_ctx
*ctx
= ask
->private;
101 struct aead_tfm
*aeadc
= pask
->private;
102 struct crypto_aead
*tfm
= aeadc
->aead
;
103 struct crypto_skcipher
*null_tfm
= aeadc
->null_tfm
;
104 unsigned int as
= crypto_aead_authsize(tfm
);
105 struct af_alg_async_req
*areq
;
106 struct af_alg_tsgl
*tsgl
;
107 struct scatterlist
*src
;
109 size_t used
= 0; /* [in] TX bufs to be en/decrypted */
110 size_t outlen
= 0; /* [out] RX bufs produced by kernel */
111 size_t usedpages
= 0; /* [in] RX bufs to be used from user */
112 size_t processed
= 0; /* [in] TX bufs to be consumed */
115 * Data length provided by caller via sendmsg/sendpage that has not
116 * yet been processed.
121 * Make sure sufficient data is present -- note, the same check is
122 * is also present in sendmsg/sendpage. The checks in sendpage/sendmsg
123 * shall provide an information to the data sender that something is
124 * wrong, but they are irrelevant to maintain the kernel integrity.
125 * We need this check here too in case user space decides to not honor
126 * the error message in sendmsg/sendpage and still call recvmsg. This
127 * check here protects the kernel integrity.
129 if (!aead_sufficient_data(sk
))
133 * Calculate the minimum output buffer size holding the result of the
134 * cipher operation. When encrypting data, the receiving buffer is
135 * larger by the tag length compared to the input buffer as the
136 * encryption operation generates the tag. For decryption, the input
137 * buffer provides the tag which is consumed resulting in only the
138 * plaintext without a buffer for the tag returned to the caller.
146 * The cipher operation input data is reduced by the associated data
147 * length as this data is processed separately later on.
149 used
-= ctx
->aead_assoclen
;
151 /* Allocate cipher request for current operation. */
152 areq
= af_alg_alloc_areq(sk
, sizeof(struct af_alg_async_req
) +
153 crypto_aead_reqsize(tfm
));
155 return PTR_ERR(areq
);
157 /* convert iovecs of output buffers into RX SGL */
158 err
= af_alg_get_rsgl(sk
, msg
, flags
, areq
, outlen
, &usedpages
);
163 * Ensure output buffer is sufficiently large. If the caller provides
164 * less buffer space, only use the relative required input size. This
165 * allows AIO operation where the caller sent all data to be processed
166 * and the AIO operation performs the operation on the different chunks
169 if (usedpages
< outlen
) {
170 size_t less
= outlen
- usedpages
;
180 processed
= used
+ ctx
->aead_assoclen
;
181 tsgl
= list_first_entry(&ctx
->tsgl_list
, struct af_alg_tsgl
, list
);
184 * Copy of AAD from source to destination
186 * The AAD is copied to the destination buffer without change. Even
187 * when user space uses an in-place cipher operation, the kernel
188 * will copy the data as it does not see whether such in-place operation
191 * To ensure efficiency, the following implementation ensure that the
192 * ciphers are invoked to perform a crypto operation in-place. This
193 * is achieved by memory management specified as follows.
196 /* Use the RX SGL as source (and destination) for crypto op. */
197 src
= areq
->first_rsgl
.sgl
.sg
;
201 * Encryption operation - The in-place cipher operation is
202 * achieved by the following operation:
208 * RX SGL: AAD || PT || Tag
210 err
= crypto_aead_copy_sgl(null_tfm
, tsgl
->sg
,
211 areq
->first_rsgl
.sgl
.sg
, processed
);
214 af_alg_pull_tsgl(sk
, processed
, NULL
, 0);
217 * Decryption operation - To achieve an in-place cipher
218 * operation, the following SGL structure is used:
220 * TX SGL: AAD || CT || Tag
222 * | copy | | Create SGL link.
224 * RX SGL: AAD || CT ----+
227 /* Copy AAD || CT to RX SGL buffer for in-place operation. */
228 err
= crypto_aead_copy_sgl(null_tfm
, tsgl
->sg
,
229 areq
->first_rsgl
.sgl
.sg
, outlen
);
233 /* Create TX SGL for tag and chain it to RX SGL. */
234 areq
->tsgl_entries
= af_alg_count_tsgl(sk
, processed
,
236 if (!areq
->tsgl_entries
)
237 areq
->tsgl_entries
= 1;
238 areq
->tsgl
= sock_kmalloc(sk
, sizeof(*areq
->tsgl
) *
245 sg_init_table(areq
->tsgl
, areq
->tsgl_entries
);
247 /* Release TX SGL, except for tag data and reassign tag data. */
248 af_alg_pull_tsgl(sk
, processed
, areq
->tsgl
, processed
- as
);
250 /* chain the areq TX SGL holding the tag with RX SGL */
253 struct af_alg_sgl
*sgl_prev
= &areq
->last_rsgl
->sgl
;
255 sg_unmark_end(sgl_prev
->sg
+ sgl_prev
->npages
- 1);
256 sg_chain(sgl_prev
->sg
, sgl_prev
->npages
+ 1,
259 /* no RX SGL present (e.g. authentication only) */
263 /* Initialize the crypto operation */
264 aead_request_set_crypt(&areq
->cra_u
.aead_req
, src
,
265 areq
->first_rsgl
.sgl
.sg
, used
, ctx
->iv
);
266 aead_request_set_ad(&areq
->cra_u
.aead_req
, ctx
->aead_assoclen
);
267 aead_request_set_tfm(&areq
->cra_u
.aead_req
, tfm
);
269 if (msg
->msg_iocb
&& !is_sync_kiocb(msg
->msg_iocb
)) {
271 areq
->iocb
= msg
->msg_iocb
;
272 aead_request_set_callback(&areq
->cra_u
.aead_req
,
273 CRYPTO_TFM_REQ_MAY_BACKLOG
,
274 af_alg_async_cb
, areq
);
275 err
= ctx
->enc
? crypto_aead_encrypt(&areq
->cra_u
.aead_req
) :
276 crypto_aead_decrypt(&areq
->cra_u
.aead_req
);
278 /* Synchronous operation */
279 aead_request_set_callback(&areq
->cra_u
.aead_req
,
280 CRYPTO_TFM_REQ_MAY_BACKLOG
,
281 crypto_req_done
, &ctx
->wait
);
282 err
= crypto_wait_req(ctx
->enc
?
283 crypto_aead_encrypt(&areq
->cra_u
.aead_req
) :
284 crypto_aead_decrypt(&areq
->cra_u
.aead_req
),
288 /* AIO operation in progress */
289 if (err
== -EINPROGRESS
) {
292 /* Remember output size that will be generated. */
293 areq
->outlen
= outlen
;
299 af_alg_free_areq_sgls(areq
);
300 sock_kfree_s(sk
, areq
, areq
->areqlen
);
302 return err
? err
: outlen
;
305 static int aead_recvmsg(struct socket
*sock
, struct msghdr
*msg
,
306 size_t ignored
, int flags
)
308 struct sock
*sk
= sock
->sk
;
312 while (msg_data_left(msg
)) {
313 int err
= _aead_recvmsg(sock
, msg
, ignored
, flags
);
316 * This error covers -EIOCBQUEUED which implies that we can
317 * only handle one AIO request. If the caller wants to have
318 * multiple AIO requests in parallel, he must make multiple
319 * separate AIO calls.
321 * Also return the error if no data has been processed so far.
324 if (err
== -EIOCBQUEUED
|| err
== -EBADMSG
|| !ret
)
333 af_alg_wmem_wakeup(sk
);
338 static struct proto_ops algif_aead_ops
= {
341 .connect
= sock_no_connect
,
342 .socketpair
= sock_no_socketpair
,
343 .getname
= sock_no_getname
,
344 .ioctl
= sock_no_ioctl
,
345 .listen
= sock_no_listen
,
346 .shutdown
= sock_no_shutdown
,
347 .getsockopt
= sock_no_getsockopt
,
348 .mmap
= sock_no_mmap
,
349 .bind
= sock_no_bind
,
350 .accept
= sock_no_accept
,
351 .setsockopt
= sock_no_setsockopt
,
353 .release
= af_alg_release
,
354 .sendmsg
= aead_sendmsg
,
355 .sendpage
= af_alg_sendpage
,
356 .recvmsg
= aead_recvmsg
,
360 static int aead_check_key(struct socket
*sock
)
364 struct alg_sock
*pask
;
365 struct aead_tfm
*tfm
;
366 struct sock
*sk
= sock
->sk
;
367 struct alg_sock
*ask
= alg_sk(sk
);
374 pask
= alg_sk(ask
->parent
);
378 lock_sock_nested(psk
, SINGLE_DEPTH_NESTING
);
398 static int aead_sendmsg_nokey(struct socket
*sock
, struct msghdr
*msg
,
403 err
= aead_check_key(sock
);
407 return aead_sendmsg(sock
, msg
, size
);
410 static ssize_t
aead_sendpage_nokey(struct socket
*sock
, struct page
*page
,
411 int offset
, size_t size
, int flags
)
415 err
= aead_check_key(sock
);
419 return af_alg_sendpage(sock
, page
, offset
, size
, flags
);
422 static int aead_recvmsg_nokey(struct socket
*sock
, struct msghdr
*msg
,
423 size_t ignored
, int flags
)
427 err
= aead_check_key(sock
);
431 return aead_recvmsg(sock
, msg
, ignored
, flags
);
434 static struct proto_ops algif_aead_ops_nokey
= {
437 .connect
= sock_no_connect
,
438 .socketpair
= sock_no_socketpair
,
439 .getname
= sock_no_getname
,
440 .ioctl
= sock_no_ioctl
,
441 .listen
= sock_no_listen
,
442 .shutdown
= sock_no_shutdown
,
443 .getsockopt
= sock_no_getsockopt
,
444 .mmap
= sock_no_mmap
,
445 .bind
= sock_no_bind
,
446 .accept
= sock_no_accept
,
447 .setsockopt
= sock_no_setsockopt
,
449 .release
= af_alg_release
,
450 .sendmsg
= aead_sendmsg_nokey
,
451 .sendpage
= aead_sendpage_nokey
,
452 .recvmsg
= aead_recvmsg_nokey
,
456 static void *aead_bind(const char *name
, u32 type
, u32 mask
)
458 struct aead_tfm
*tfm
;
459 struct crypto_aead
*aead
;
460 struct crypto_skcipher
*null_tfm
;
462 tfm
= kzalloc(sizeof(*tfm
), GFP_KERNEL
);
464 return ERR_PTR(-ENOMEM
);
466 aead
= crypto_alloc_aead(name
, type
, mask
);
469 return ERR_CAST(aead
);
472 null_tfm
= crypto_get_default_null_skcipher2();
473 if (IS_ERR(null_tfm
)) {
474 crypto_free_aead(aead
);
476 return ERR_CAST(null_tfm
);
480 tfm
->null_tfm
= null_tfm
;
485 static void aead_release(void *private)
487 struct aead_tfm
*tfm
= private;
489 crypto_free_aead(tfm
->aead
);
493 static int aead_setauthsize(void *private, unsigned int authsize
)
495 struct aead_tfm
*tfm
= private;
497 return crypto_aead_setauthsize(tfm
->aead
, authsize
);
500 static int aead_setkey(void *private, const u8
*key
, unsigned int keylen
)
502 struct aead_tfm
*tfm
= private;
505 err
= crypto_aead_setkey(tfm
->aead
, key
, keylen
);
511 static void aead_sock_destruct(struct sock
*sk
)
513 struct alg_sock
*ask
= alg_sk(sk
);
514 struct af_alg_ctx
*ctx
= ask
->private;
515 struct sock
*psk
= ask
->parent
;
516 struct alg_sock
*pask
= alg_sk(psk
);
517 struct aead_tfm
*aeadc
= pask
->private;
518 struct crypto_aead
*tfm
= aeadc
->aead
;
519 unsigned int ivlen
= crypto_aead_ivsize(tfm
);
521 af_alg_pull_tsgl(sk
, ctx
->used
, NULL
, 0);
522 crypto_put_default_null_skcipher2();
523 sock_kzfree_s(sk
, ctx
->iv
, ivlen
);
524 sock_kfree_s(sk
, ctx
, ctx
->len
);
525 af_alg_release_parent(sk
);
528 static int aead_accept_parent_nokey(void *private, struct sock
*sk
)
530 struct af_alg_ctx
*ctx
;
531 struct alg_sock
*ask
= alg_sk(sk
);
532 struct aead_tfm
*tfm
= private;
533 struct crypto_aead
*aead
= tfm
->aead
;
534 unsigned int len
= sizeof(*ctx
);
535 unsigned int ivlen
= crypto_aead_ivsize(aead
);
537 ctx
= sock_kmalloc(sk
, len
, GFP_KERNEL
);
542 ctx
->iv
= sock_kmalloc(sk
, ivlen
, GFP_KERNEL
);
544 sock_kfree_s(sk
, ctx
, len
);
547 memset(ctx
->iv
, 0, ivlen
);
549 INIT_LIST_HEAD(&ctx
->tsgl_list
);
556 ctx
->aead_assoclen
= 0;
557 crypto_init_wait(&ctx
->wait
);
561 sk
->sk_destruct
= aead_sock_destruct
;
566 static int aead_accept_parent(void *private, struct sock
*sk
)
568 struct aead_tfm
*tfm
= private;
573 return aead_accept_parent_nokey(private, sk
);
576 static const struct af_alg_type algif_type_aead
= {
578 .release
= aead_release
,
579 .setkey
= aead_setkey
,
580 .setauthsize
= aead_setauthsize
,
581 .accept
= aead_accept_parent
,
582 .accept_nokey
= aead_accept_parent_nokey
,
583 .ops
= &algif_aead_ops
,
584 .ops_nokey
= &algif_aead_ops_nokey
,
589 static int __init
algif_aead_init(void)
591 return af_alg_register_type(&algif_type_aead
);
594 static void __exit
algif_aead_exit(void)
596 int err
= af_alg_unregister_type(&algif_type_aead
);
600 module_init(algif_aead_init
);
601 module_exit(algif_aead_exit
);
602 MODULE_LICENSE("GPL");
603 MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
604 MODULE_DESCRIPTION("AEAD kernel crypto API user space interface");