2 * pcrypt - Parallel crypto wrapper.
4 * Copyright (C) 2009 secunet Security Networks AG
5 * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * You should have received a copy of the GNU General Public License along with
17 * this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
21 #include <crypto/algapi.h>
22 #include <crypto/internal/aead.h>
23 #include <linux/err.h>
24 #include <linux/init.h>
25 #include <linux/module.h>
26 #include <linux/slab.h>
27 #include <linux/notifier.h>
28 #include <crypto/pcrypt.h>
30 struct pcrypt_instance
{
31 struct padata_instance
*pinst
;
32 struct workqueue_struct
*wq
;
35 * Cpumask for callback CPUs. It should be
36 * equal to serial cpumask of corresponding padata instance,
37 * so it is updated when padata notifies us about serial
40 * cb_cpumask is protected by RCU. This fact prevents us from
41 * using cpumask_var_t directly because the actual type of
42 * cpumsak_var_t depends on kernel configuration(particularly on
43 * CONFIG_CPUMASK_OFFSTACK macro). Depending on the configuration
44 * cpumask_var_t may be either a pointer to the struct cpumask
45 * or a variable allocated on the stack. Thus we can not safely use
46 * cpumask_var_t with RCU operations such as rcu_assign_pointer or
47 * rcu_dereference. So cpumask_var_t is wrapped with struct
48 * pcrypt_cpumask which makes possible to use it with RCU.
50 struct pcrypt_cpumask
{
53 struct notifier_block nblock
;
56 static struct pcrypt_instance pencrypt
;
57 static struct pcrypt_instance pdecrypt
;
60 struct pcrypt_instance_ctx
{
61 struct crypto_spawn spawn
;
62 unsigned int tfm_count
;
65 struct pcrypt_aead_ctx
{
66 struct crypto_aead
*child
;
70 static int pcrypt_do_parallel(struct padata_priv
*padata
, unsigned int *cb_cpu
,
71 struct pcrypt_instance
*pcrypt
)
73 unsigned int cpu_index
, cpu
, i
;
74 struct pcrypt_cpumask
*cpumask
;
79 cpumask
= rcu_dereference(pcrypt
->cb_cpumask
);
80 if (cpumask_test_cpu(cpu
, cpumask
->mask
))
83 cpu_index
= cpu
% cpumask_weight(cpumask
->mask
);
85 cpu
= cpumask_first(cpumask
->mask
);
86 for (i
= 0; i
< cpu_index
; i
++)
87 cpu
= cpumask_next(cpu
, cpumask
->mask
);
93 return padata_do_parallel(pcrypt
->pinst
, padata
, cpu
);
96 static int pcrypt_aead_setkey(struct crypto_aead
*parent
,
97 const u8
*key
, unsigned int keylen
)
99 struct pcrypt_aead_ctx
*ctx
= crypto_aead_ctx(parent
);
101 return crypto_aead_setkey(ctx
->child
, key
, keylen
);
104 static int pcrypt_aead_setauthsize(struct crypto_aead
*parent
,
105 unsigned int authsize
)
107 struct pcrypt_aead_ctx
*ctx
= crypto_aead_ctx(parent
);
109 return crypto_aead_setauthsize(ctx
->child
, authsize
);
112 static void pcrypt_aead_serial(struct padata_priv
*padata
)
114 struct pcrypt_request
*preq
= pcrypt_padata_request(padata
);
115 struct aead_request
*req
= pcrypt_request_ctx(preq
);
117 aead_request_complete(req
->base
.data
, padata
->info
);
120 static void pcrypt_aead_giv_serial(struct padata_priv
*padata
)
122 struct pcrypt_request
*preq
= pcrypt_padata_request(padata
);
123 struct aead_givcrypt_request
*req
= pcrypt_request_ctx(preq
);
125 aead_request_complete(req
->areq
.base
.data
, padata
->info
);
128 static void pcrypt_aead_done(struct crypto_async_request
*areq
, int err
)
130 struct aead_request
*req
= areq
->data
;
131 struct pcrypt_request
*preq
= aead_request_ctx(req
);
132 struct padata_priv
*padata
= pcrypt_request_padata(preq
);
135 req
->base
.flags
&= ~CRYPTO_TFM_REQ_MAY_SLEEP
;
137 padata_do_serial(padata
);
140 static void pcrypt_aead_enc(struct padata_priv
*padata
)
142 struct pcrypt_request
*preq
= pcrypt_padata_request(padata
);
143 struct aead_request
*req
= pcrypt_request_ctx(preq
);
145 padata
->info
= crypto_aead_encrypt(req
);
147 if (padata
->info
== -EINPROGRESS
)
150 padata_do_serial(padata
);
153 static int pcrypt_aead_encrypt(struct aead_request
*req
)
156 struct pcrypt_request
*preq
= aead_request_ctx(req
);
157 struct aead_request
*creq
= pcrypt_request_ctx(preq
);
158 struct padata_priv
*padata
= pcrypt_request_padata(preq
);
159 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
160 struct pcrypt_aead_ctx
*ctx
= crypto_aead_ctx(aead
);
161 u32 flags
= aead_request_flags(req
);
163 memset(padata
, 0, sizeof(struct padata_priv
));
165 padata
->parallel
= pcrypt_aead_enc
;
166 padata
->serial
= pcrypt_aead_serial
;
168 aead_request_set_tfm(creq
, ctx
->child
);
169 aead_request_set_callback(creq
, flags
& ~CRYPTO_TFM_REQ_MAY_SLEEP
,
170 pcrypt_aead_done
, req
);
171 aead_request_set_crypt(creq
, req
->src
, req
->dst
,
172 req
->cryptlen
, req
->iv
);
173 aead_request_set_assoc(creq
, req
->assoc
, req
->assoclen
);
175 err
= pcrypt_do_parallel(padata
, &ctx
->cb_cpu
, &pencrypt
);
182 static void pcrypt_aead_dec(struct padata_priv
*padata
)
184 struct pcrypt_request
*preq
= pcrypt_padata_request(padata
);
185 struct aead_request
*req
= pcrypt_request_ctx(preq
);
187 padata
->info
= crypto_aead_decrypt(req
);
189 if (padata
->info
== -EINPROGRESS
)
192 padata_do_serial(padata
);
195 static int pcrypt_aead_decrypt(struct aead_request
*req
)
198 struct pcrypt_request
*preq
= aead_request_ctx(req
);
199 struct aead_request
*creq
= pcrypt_request_ctx(preq
);
200 struct padata_priv
*padata
= pcrypt_request_padata(preq
);
201 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
202 struct pcrypt_aead_ctx
*ctx
= crypto_aead_ctx(aead
);
203 u32 flags
= aead_request_flags(req
);
205 memset(padata
, 0, sizeof(struct padata_priv
));
207 padata
->parallel
= pcrypt_aead_dec
;
208 padata
->serial
= pcrypt_aead_serial
;
210 aead_request_set_tfm(creq
, ctx
->child
);
211 aead_request_set_callback(creq
, flags
& ~CRYPTO_TFM_REQ_MAY_SLEEP
,
212 pcrypt_aead_done
, req
);
213 aead_request_set_crypt(creq
, req
->src
, req
->dst
,
214 req
->cryptlen
, req
->iv
);
215 aead_request_set_assoc(creq
, req
->assoc
, req
->assoclen
);
217 err
= pcrypt_do_parallel(padata
, &ctx
->cb_cpu
, &pdecrypt
);
224 static void pcrypt_aead_givenc(struct padata_priv
*padata
)
226 struct pcrypt_request
*preq
= pcrypt_padata_request(padata
);
227 struct aead_givcrypt_request
*req
= pcrypt_request_ctx(preq
);
229 padata
->info
= crypto_aead_givencrypt(req
);
231 if (padata
->info
== -EINPROGRESS
)
234 padata_do_serial(padata
);
237 static int pcrypt_aead_givencrypt(struct aead_givcrypt_request
*req
)
240 struct aead_request
*areq
= &req
->areq
;
241 struct pcrypt_request
*preq
= aead_request_ctx(areq
);
242 struct aead_givcrypt_request
*creq
= pcrypt_request_ctx(preq
);
243 struct padata_priv
*padata
= pcrypt_request_padata(preq
);
244 struct crypto_aead
*aead
= aead_givcrypt_reqtfm(req
);
245 struct pcrypt_aead_ctx
*ctx
= crypto_aead_ctx(aead
);
246 u32 flags
= aead_request_flags(areq
);
248 memset(padata
, 0, sizeof(struct padata_priv
));
250 padata
->parallel
= pcrypt_aead_givenc
;
251 padata
->serial
= pcrypt_aead_giv_serial
;
253 aead_givcrypt_set_tfm(creq
, ctx
->child
);
254 aead_givcrypt_set_callback(creq
, flags
& ~CRYPTO_TFM_REQ_MAY_SLEEP
,
255 pcrypt_aead_done
, areq
);
256 aead_givcrypt_set_crypt(creq
, areq
->src
, areq
->dst
,
257 areq
->cryptlen
, areq
->iv
);
258 aead_givcrypt_set_assoc(creq
, areq
->assoc
, areq
->assoclen
);
259 aead_givcrypt_set_giv(creq
, req
->giv
, req
->seq
);
261 err
= pcrypt_do_parallel(padata
, &ctx
->cb_cpu
, &pencrypt
);
268 static int pcrypt_aead_init_tfm(struct crypto_tfm
*tfm
)
271 struct crypto_instance
*inst
= crypto_tfm_alg_instance(tfm
);
272 struct pcrypt_instance_ctx
*ictx
= crypto_instance_ctx(inst
);
273 struct pcrypt_aead_ctx
*ctx
= crypto_tfm_ctx(tfm
);
274 struct crypto_aead
*cipher
;
278 cpu_index
= ictx
->tfm_count
% cpumask_weight(cpu_active_mask
);
280 ctx
->cb_cpu
= cpumask_first(cpu_active_mask
);
281 for (cpu
= 0; cpu
< cpu_index
; cpu
++)
282 ctx
->cb_cpu
= cpumask_next(ctx
->cb_cpu
, cpu_active_mask
);
284 cipher
= crypto_spawn_aead(crypto_instance_ctx(inst
));
287 return PTR_ERR(cipher
);
290 tfm
->crt_aead
.reqsize
= sizeof(struct pcrypt_request
)
291 + sizeof(struct aead_givcrypt_request
)
292 + crypto_aead_reqsize(cipher
);
297 static void pcrypt_aead_exit_tfm(struct crypto_tfm
*tfm
)
299 struct pcrypt_aead_ctx
*ctx
= crypto_tfm_ctx(tfm
);
301 crypto_free_aead(ctx
->child
);
304 static struct crypto_instance
*pcrypt_alloc_instance(struct crypto_alg
*alg
)
306 struct crypto_instance
*inst
;
307 struct pcrypt_instance_ctx
*ctx
;
310 inst
= kzalloc(sizeof(*inst
) + sizeof(*ctx
), GFP_KERNEL
);
312 inst
= ERR_PTR(-ENOMEM
);
317 if (snprintf(inst
->alg
.cra_driver_name
, CRYPTO_MAX_ALG_NAME
,
318 "pcrypt(%s)", alg
->cra_driver_name
) >= CRYPTO_MAX_ALG_NAME
)
321 memcpy(inst
->alg
.cra_name
, alg
->cra_name
, CRYPTO_MAX_ALG_NAME
);
323 ctx
= crypto_instance_ctx(inst
);
324 err
= crypto_init_spawn(&ctx
->spawn
, alg
, inst
,
325 CRYPTO_ALG_TYPE_MASK
);
329 inst
->alg
.cra_priority
= alg
->cra_priority
+ 100;
330 inst
->alg
.cra_blocksize
= alg
->cra_blocksize
;
331 inst
->alg
.cra_alignmask
= alg
->cra_alignmask
;
342 static struct crypto_instance
*pcrypt_alloc_aead(struct rtattr
**tb
,
345 struct crypto_instance
*inst
;
346 struct crypto_alg
*alg
;
348 alg
= crypto_get_attr_alg(tb
, type
, (mask
& CRYPTO_ALG_TYPE_MASK
));
350 return ERR_CAST(alg
);
352 inst
= pcrypt_alloc_instance(alg
);
356 inst
->alg
.cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
;
357 inst
->alg
.cra_type
= &crypto_aead_type
;
359 inst
->alg
.cra_aead
.ivsize
= alg
->cra_aead
.ivsize
;
360 inst
->alg
.cra_aead
.geniv
= alg
->cra_aead
.geniv
;
361 inst
->alg
.cra_aead
.maxauthsize
= alg
->cra_aead
.maxauthsize
;
363 inst
->alg
.cra_ctxsize
= sizeof(struct pcrypt_aead_ctx
);
365 inst
->alg
.cra_init
= pcrypt_aead_init_tfm
;
366 inst
->alg
.cra_exit
= pcrypt_aead_exit_tfm
;
368 inst
->alg
.cra_aead
.setkey
= pcrypt_aead_setkey
;
369 inst
->alg
.cra_aead
.setauthsize
= pcrypt_aead_setauthsize
;
370 inst
->alg
.cra_aead
.encrypt
= pcrypt_aead_encrypt
;
371 inst
->alg
.cra_aead
.decrypt
= pcrypt_aead_decrypt
;
372 inst
->alg
.cra_aead
.givencrypt
= pcrypt_aead_givencrypt
;
379 static struct crypto_instance
*pcrypt_alloc(struct rtattr
**tb
)
381 struct crypto_attr_type
*algt
;
383 algt
= crypto_get_attr_type(tb
);
385 return ERR_CAST(algt
);
387 switch (algt
->type
& algt
->mask
& CRYPTO_ALG_TYPE_MASK
) {
388 case CRYPTO_ALG_TYPE_AEAD
:
389 return pcrypt_alloc_aead(tb
, algt
->type
, algt
->mask
);
392 return ERR_PTR(-EINVAL
);
395 static void pcrypt_free(struct crypto_instance
*inst
)
397 struct pcrypt_instance_ctx
*ctx
= crypto_instance_ctx(inst
);
399 crypto_drop_spawn(&ctx
->spawn
);
403 static int pcrypt_cpumask_change_notify(struct notifier_block
*self
,
404 unsigned long val
, void *data
)
406 struct pcrypt_instance
*pcrypt
;
407 struct pcrypt_cpumask
*new_mask
, *old_mask
;
409 if (!(val
& PADATA_CPU_SERIAL
))
412 pcrypt
= container_of(self
, struct pcrypt_instance
, nblock
);
413 new_mask
= kmalloc(sizeof(*new_mask
), GFP_KERNEL
);
416 if (!alloc_cpumask_var(&new_mask
->mask
, GFP_KERNEL
)) {
421 old_mask
= pcrypt
->cb_cpumask
;
423 padata_get_cpumask(pcrypt
->pinst
, PADATA_CPU_SERIAL
, new_mask
->mask
);
424 rcu_assign_pointer(pcrypt
->cb_cpumask
, new_mask
);
425 synchronize_rcu_bh();
427 free_cpumask_var(old_mask
->mask
);
432 static int __pcrypt_init_instance(struct pcrypt_instance
*pcrypt
,
436 struct pcrypt_cpumask
*mask
;
438 pcrypt
->wq
= create_workqueue(name
);
442 pcrypt
->pinst
= padata_alloc(pcrypt
->wq
);
444 goto err_destroy_workqueue
;
446 mask
= kmalloc(sizeof(*mask
), GFP_KERNEL
);
448 goto err_free_padata
;
449 if (!alloc_cpumask_var(&mask
->mask
, GFP_KERNEL
)) {
451 goto err_free_padata
;
454 padata_get_cpumask(pcrypt
->pinst
, PADATA_CPU_SERIAL
, mask
->mask
);
455 rcu_assign_pointer(pcrypt
->cb_cpumask
, mask
);
457 pcrypt
->nblock
.notifier_call
= pcrypt_cpumask_change_notify
;
458 ret
= padata_register_cpumask_notifier(pcrypt
->pinst
, &pcrypt
->nblock
);
460 goto err_free_cpumask
;
464 free_cpumask_var(mask
->mask
);
467 padata_free(pcrypt
->pinst
);
468 err_destroy_workqueue
:
469 destroy_workqueue(pcrypt
->wq
);
474 static void __pcrypt_deinit_instance(struct pcrypt_instance
*pcrypt
)
476 free_cpumask_var(pcrypt
->cb_cpumask
->mask
);
477 kfree(pcrypt
->cb_cpumask
);
479 padata_stop(pcrypt
->pinst
);
480 padata_unregister_cpumask_notifier(pcrypt
->pinst
, &pcrypt
->nblock
);
481 destroy_workqueue(pcrypt
->wq
);
482 padata_free(pcrypt
->pinst
);
485 static struct crypto_template pcrypt_tmpl
= {
487 .alloc
= pcrypt_alloc
,
489 .module
= THIS_MODULE
,
492 static int __init
pcrypt_init(void)
496 err
= __pcrypt_init_instance(&pencrypt
, "pencrypt");
500 err
= __pcrypt_init_instance(&pdecrypt
, "pdecrypt");
502 goto err_deinit_pencrypt
;
504 padata_start(pencrypt
.pinst
);
505 padata_start(pdecrypt
.pinst
);
507 return crypto_register_template(&pcrypt_tmpl
);
510 __pcrypt_deinit_instance(&pencrypt
);
515 static void __exit
pcrypt_exit(void)
517 __pcrypt_deinit_instance(&pencrypt
);
518 __pcrypt_deinit_instance(&pdecrypt
);
520 crypto_unregister_template(&pcrypt_tmpl
);
523 module_init(pcrypt_init
);
524 module_exit(pcrypt_exit
);
526 MODULE_LICENSE("GPL");
527 MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
528 MODULE_DESCRIPTION("Parallel crypto wrapper");