?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
crypto: serpent-sse2 - add lrw support
[linux-2.6/btrfs-unstable.git] / arch / x86 / crypto / serpent_sse2_glue.c
blobdb318e5cb240221e7bc613905d7948a7f99dcba1
1 /*
2 * Glue Code for SSE2 assembler versions of Serpent Cipher
3 *
4 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
5 *
6 * Glue code based on aesni-intel_glue.c by:
7 * Copyright (C) 2008, Intel Corp.
8 * Author: Huang Ying <ying.huang@intel.com>
9 *
10 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
11 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
12 * CTR part based on code (crypto/ctr.c) by:
13 * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
14 *
15 * This program is free software; you can redistribute it and/or modify
16 * it under the terms of the GNU General Public License as published by
17 * the Free Software Foundation; either version 2 of the License, or
18 * (at your option) any later version.
19 *
20 * This program is distributed in the hope that it will be useful,
21 * but WITHOUT ANY WARRANTY; without even the implied warranty of
22 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23 * GNU General Public License for more details.
24 *
25 * You should have received a copy of the GNU General Public License
26 * along with this program; if not, write to the Free Software
27 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
28 * USA
29 *
30 */
31
32 #include <linux/module.h>
33 #include <linux/hardirq.h>
34 #include <linux/types.h>
35 #include <linux/crypto.h>
36 #include <linux/err.h>
37 #include <crypto/algapi.h>
38 #include <crypto/serpent.h>
39 #include <crypto/cryptd.h>
40 #include <crypto/b128ops.h>
41 #include <crypto/ctr.h>
42 #include <crypto/lrw.h>
43 #include <asm/i387.h>
44 #include <asm/serpent.h>
45 #include <crypto/scatterwalk.h>
46 #include <linux/workqueue.h>
47 #include <linux/spinlock.h>
48
49 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE)
50 #define HAS_LRW
51 #endif
52
53 struct async_serpent_ctx {
54 struct cryptd_ablkcipher *cryptd_tfm;
55 };
56
57 static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
58 {
59 if (fpu_enabled)
60 return true;
61
62 /* SSE2 is only used when chunk to be processed is large enough, so
63 * do not enable FPU until it is necessary.
64 */
65 if (nbytes < SERPENT_BLOCK_SIZE * SERPENT_PARALLEL_BLOCKS)
66 return false;
67
68 kernel_fpu_begin();
69 return true;
70 }
71
72 static inline void serpent_fpu_end(bool fpu_enabled)
73 {
74 if (fpu_enabled)
75 kernel_fpu_end();
76 }
77
78 static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
79 bool enc)
80 {
81 bool fpu_enabled = false;
82 struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
83 const unsigned int bsize = SERPENT_BLOCK_SIZE;
84 unsigned int nbytes;
85 int err;
86
87 err = blkcipher_walk_virt(desc, walk);
88 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
89
90 while ((nbytes = walk->nbytes)) {
91 u8 *wsrc = walk->src.virt.addr;
92 u8 *wdst = walk->dst.virt.addr;
93
94 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
95
96 /* Process multi-block batch */
97 if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
98 do {
99 if (enc)
100 serpent_enc_blk_xway(ctx, wdst, wsrc);
101 else
102 serpent_dec_blk_xway(ctx, wdst, wsrc);
103
104 wsrc += bsize * SERPENT_PARALLEL_BLOCKS;
105 wdst += bsize * SERPENT_PARALLEL_BLOCKS;
106 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
107 } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
108
109 if (nbytes < bsize)
110 goto done;
111 }
112
113 /* Handle leftovers */
114 do {
115 if (enc)
116 __serpent_encrypt(ctx, wdst, wsrc);
117 else
118 __serpent_decrypt(ctx, wdst, wsrc);
119
120 wsrc += bsize;
121 wdst += bsize;
122 nbytes -= bsize;
123 } while (nbytes >= bsize);
124
125 done:
126 err = blkcipher_walk_done(desc, walk, nbytes);
127 }
128
129 serpent_fpu_end(fpu_enabled);
130 return err;
131 }
132
133 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
134 struct scatterlist *src, unsigned int nbytes)
135 {
136 struct blkcipher_walk walk;
137
138 blkcipher_walk_init(&walk, dst, src, nbytes);
139 return ecb_crypt(desc, &walk, true);
140 }
141
142 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
143 struct scatterlist *src, unsigned int nbytes)
144 {
145 struct blkcipher_walk walk;
146
147 blkcipher_walk_init(&walk, dst, src, nbytes);
148 return ecb_crypt(desc, &walk, false);
149 }
150
151 static struct crypto_alg blk_ecb_alg = {
152 .cra_name = "__ecb-serpent-sse2",
153 .cra_driver_name = "__driver-ecb-serpent-sse2",
154 .cra_priority = 0,
155 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
156 .cra_blocksize = SERPENT_BLOCK_SIZE,
157 .cra_ctxsize = sizeof(struct serpent_ctx),
158 .cra_alignmask = 0,
159 .cra_type = &crypto_blkcipher_type,
160 .cra_module = THIS_MODULE,
161 .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
162 .cra_u = {
163 .blkcipher = {
164 .min_keysize = SERPENT_MIN_KEY_SIZE,
165 .max_keysize = SERPENT_MAX_KEY_SIZE,
166 .setkey = serpent_setkey,
167 .encrypt = ecb_encrypt,
168 .decrypt = ecb_decrypt,
169 },
170 },
171 };
172
173 static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
174 struct blkcipher_walk *walk)
175 {
176 struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
177 const unsigned int bsize = SERPENT_BLOCK_SIZE;
178 unsigned int nbytes = walk->nbytes;
179 u128 *src = (u128 *)walk->src.virt.addr;
180 u128 *dst = (u128 *)walk->dst.virt.addr;
181 u128 *iv = (u128 *)walk->iv;
182
183 do {
184 u128_xor(dst, src, iv);
185 __serpent_encrypt(ctx, (u8 *)dst, (u8 *)dst);
186 iv = dst;
187
188 src += 1;
189 dst += 1;
190 nbytes -= bsize;
191 } while (nbytes >= bsize);
192
193 u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
194 return nbytes;
195 }
196
197 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
198 struct scatterlist *src, unsigned int nbytes)
199 {
200 struct blkcipher_walk walk;
201 int err;
202
203 blkcipher_walk_init(&walk, dst, src, nbytes);
204 err = blkcipher_walk_virt(desc, &walk);
205
206 while ((nbytes = walk.nbytes)) {
207 nbytes = __cbc_encrypt(desc, &walk);
208 err = blkcipher_walk_done(desc, &walk, nbytes);
209 }
210
211 return err;
212 }
213
214 static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
215 struct blkcipher_walk *walk)
216 {
217 struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
218 const unsigned int bsize = SERPENT_BLOCK_SIZE;
219 unsigned int nbytes = walk->nbytes;
220 u128 *src = (u128 *)walk->src.virt.addr;
221 u128 *dst = (u128 *)walk->dst.virt.addr;
222 u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
223 u128 last_iv;
224 int i;
225
226 /* Start of the last block. */
227 src += nbytes / bsize - 1;
228 dst += nbytes / bsize - 1;
229
230 last_iv = *src;
231
232 /* Process multi-block batch */
233 if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
234 do {
235 nbytes -= bsize * (SERPENT_PARALLEL_BLOCKS - 1);
236 src -= SERPENT_PARALLEL_BLOCKS - 1;
237 dst -= SERPENT_PARALLEL_BLOCKS - 1;
238
239 for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++)
240 ivs[i] = src[i];
241
242 serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
243
244 for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++)
245 u128_xor(dst + (i + 1), dst + (i + 1), ivs + i);
246
247 nbytes -= bsize;
248 if (nbytes < bsize)
249 goto done;
250
251 u128_xor(dst, dst, src - 1);
252 src -= 1;
253 dst -= 1;
254 } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
255
256 if (nbytes < bsize)
257 goto done;
258 }
259
260 /* Handle leftovers */
261 for (;;) {
262 __serpent_decrypt(ctx, (u8 *)dst, (u8 *)src);
263
264 nbytes -= bsize;
265 if (nbytes < bsize)
266 break;
267
268 u128_xor(dst, dst, src - 1);
269 src -= 1;
270 dst -= 1;
271 }
272
273 done:
274 u128_xor(dst, dst, (u128 *)walk->iv);
275 *(u128 *)walk->iv = last_iv;
276
277 return nbytes;
278 }
279
280 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
281 struct scatterlist *src, unsigned int nbytes)
282 {
283 bool fpu_enabled = false;
284 struct blkcipher_walk walk;
285 int err;
286
287 blkcipher_walk_init(&walk, dst, src, nbytes);
288 err = blkcipher_walk_virt(desc, &walk);
289 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
290
291 while ((nbytes = walk.nbytes)) {
292 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
293 nbytes = __cbc_decrypt(desc, &walk);
294 err = blkcipher_walk_done(desc, &walk, nbytes);
295 }
296
297 serpent_fpu_end(fpu_enabled);
298 return err;
299 }
300
301 static struct crypto_alg blk_cbc_alg = {
302 .cra_name = "__cbc-serpent-sse2",
303 .cra_driver_name = "__driver-cbc-serpent-sse2",
304 .cra_priority = 0,
305 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
306 .cra_blocksize = SERPENT_BLOCK_SIZE,
307 .cra_ctxsize = sizeof(struct serpent_ctx),
308 .cra_alignmask = 0,
309 .cra_type = &crypto_blkcipher_type,
310 .cra_module = THIS_MODULE,
311 .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
312 .cra_u = {
313 .blkcipher = {
314 .min_keysize = SERPENT_MIN_KEY_SIZE,
315 .max_keysize = SERPENT_MAX_KEY_SIZE,
316 .setkey = serpent_setkey,
317 .encrypt = cbc_encrypt,
318 .decrypt = cbc_decrypt,
319 },
320 },
321 };
322
323 static inline void u128_to_be128(be128 *dst, const u128 *src)
324 {
325 dst->a = cpu_to_be64(src->a);
326 dst->b = cpu_to_be64(src->b);
327 }
328
329 static inline void be128_to_u128(u128 *dst, const be128 *src)
330 {
331 dst->a = be64_to_cpu(src->a);
332 dst->b = be64_to_cpu(src->b);
333 }
334
335 static inline void u128_inc(u128 *i)
336 {
337 i->b++;
338 if (!i->b)
339 i->a++;
340 }
341
342 static void ctr_crypt_final(struct blkcipher_desc *desc,
343 struct blkcipher_walk *walk)
344 {
345 struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
346 u8 *ctrblk = walk->iv;
347 u8 keystream[SERPENT_BLOCK_SIZE];
348 u8 *src = walk->src.virt.addr;
349 u8 *dst = walk->dst.virt.addr;
350 unsigned int nbytes = walk->nbytes;
351
352 __serpent_encrypt(ctx, keystream, ctrblk);
353 crypto_xor(keystream, src, nbytes);
354 memcpy(dst, keystream, nbytes);
355
356 crypto_inc(ctrblk, SERPENT_BLOCK_SIZE);
357 }
358
359 static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
360 struct blkcipher_walk *walk)
361 {
362 struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
363 const unsigned int bsize = SERPENT_BLOCK_SIZE;
364 unsigned int nbytes = walk->nbytes;
365 u128 *src = (u128 *)walk->src.virt.addr;
366 u128 *dst = (u128 *)walk->dst.virt.addr;
367 u128 ctrblk;
368 be128 ctrblocks[SERPENT_PARALLEL_BLOCKS];
369 int i;
370
371 be128_to_u128(&ctrblk, (be128 *)walk->iv);
372
373 /* Process multi-block batch */
374 if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
375 do {
376 /* create ctrblks for parallel encrypt */
377 for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
378 if (dst != src)
379 dst[i] = src[i];
380
381 u128_to_be128(&ctrblocks[i], &ctrblk);
382 u128_inc(&ctrblk);
383 }
384
385 serpent_enc_blk_xway_xor(ctx, (u8 *)dst,
386 (u8 *)ctrblocks);
387
388 src += SERPENT_PARALLEL_BLOCKS;
389 dst += SERPENT_PARALLEL_BLOCKS;
390 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
391 } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
392
393 if (nbytes < bsize)
394 goto done;
395 }
396
397 /* Handle leftovers */
398 do {
399 if (dst != src)
400 *dst = *src;
401
402 u128_to_be128(&ctrblocks[0], &ctrblk);
403 u128_inc(&ctrblk);
404
405 __serpent_encrypt(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
406 u128_xor(dst, dst, (u128 *)ctrblocks);
407
408 src += 1;
409 dst += 1;
410 nbytes -= bsize;
411 } while (nbytes >= bsize);
412
413 done:
414 u128_to_be128((be128 *)walk->iv, &ctrblk);
415 return nbytes;
416 }
417
418 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
419 struct scatterlist *src, unsigned int nbytes)
420 {
421 bool fpu_enabled = false;
422 struct blkcipher_walk walk;
423 int err;
424
425 blkcipher_walk_init(&walk, dst, src, nbytes);
426 err = blkcipher_walk_virt_block(desc, &walk, SERPENT_BLOCK_SIZE);
427 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
428
429 while ((nbytes = walk.nbytes) >= SERPENT_BLOCK_SIZE) {
430 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
431 nbytes = __ctr_crypt(desc, &walk);
432 err = blkcipher_walk_done(desc, &walk, nbytes);
433 }
434
435 serpent_fpu_end(fpu_enabled);
436
437 if (walk.nbytes) {
438 ctr_crypt_final(desc, &walk);
439 err = blkcipher_walk_done(desc, &walk, 0);
440 }
441
442 return err;
443 }
444
445 static struct crypto_alg blk_ctr_alg = {
446 .cra_name = "__ctr-serpent-sse2",
447 .cra_driver_name = "__driver-ctr-serpent-sse2",
448 .cra_priority = 0,
449 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
450 .cra_blocksize = 1,
451 .cra_ctxsize = sizeof(struct serpent_ctx),
452 .cra_alignmask = 0,
453 .cra_type = &crypto_blkcipher_type,
454 .cra_module = THIS_MODULE,
455 .cra_list = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
456 .cra_u = {
457 .blkcipher = {
458 .min_keysize = SERPENT_MIN_KEY_SIZE,
459 .max_keysize = SERPENT_MAX_KEY_SIZE,
460 .ivsize = SERPENT_BLOCK_SIZE,
461 .setkey = serpent_setkey,
462 .encrypt = ctr_crypt,
463 .decrypt = ctr_crypt,
464 },
465 },
466 };
467
468 #ifdef HAS_LRW
469
470 struct crypt_priv {
471 struct serpent_ctx *ctx;
472 bool fpu_enabled;
473 };
474
475 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
476 {
477 const unsigned int bsize = SERPENT_BLOCK_SIZE;
478 struct crypt_priv *ctx = priv;
479 int i;
480
481 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
482
483 if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
484 serpent_enc_blk_xway(ctx->ctx, srcdst, srcdst);
485 return;
486 }
487
488 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
489 __serpent_encrypt(ctx->ctx, srcdst, srcdst);
490 }
491
492 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
493 {
494 const unsigned int bsize = SERPENT_BLOCK_SIZE;
495 struct crypt_priv *ctx = priv;
496 int i;
497
498 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
499
500 if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
501 serpent_dec_blk_xway(ctx->ctx, srcdst, srcdst);
502 return;
503 }
504
505 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
506 __serpent_decrypt(ctx->ctx, srcdst, srcdst);
507 }
508
509 struct serpent_lrw_ctx {
510 struct lrw_table_ctx lrw_table;
511 struct serpent_ctx serpent_ctx;
512 };
513
514 static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
515 unsigned int keylen)
516 {
517 struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
518 int err;
519
520 err = __serpent_setkey(&ctx->serpent_ctx, key, keylen -
521 SERPENT_BLOCK_SIZE);
522 if (err)
523 return err;
524
525 return lrw_init_table(&ctx->lrw_table, key + keylen -
526 SERPENT_BLOCK_SIZE);
527 }
528
529 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
530 struct scatterlist *src, unsigned int nbytes)
531 {
532 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
533 be128 buf[SERPENT_PARALLEL_BLOCKS];
534 struct crypt_priv crypt_ctx = {
535 .ctx = &ctx->serpent_ctx,
536 .fpu_enabled = false,
537 };
538 struct lrw_crypt_req req = {
539 .tbuf = buf,
540 .tbuflen = sizeof(buf),
541
542 .table_ctx = &ctx->lrw_table,
543 .crypt_ctx = &crypt_ctx,
544 .crypt_fn = encrypt_callback,
545 };
546 int ret;
547
548 ret = lrw_crypt(desc, dst, src, nbytes, &req);
549 serpent_fpu_end(crypt_ctx.fpu_enabled);
550
551 return ret;
552 }
553
554 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
555 struct scatterlist *src, unsigned int nbytes)
556 {
557 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
558 be128 buf[SERPENT_PARALLEL_BLOCKS];
559 struct crypt_priv crypt_ctx = {
560 .ctx = &ctx->serpent_ctx,
561 .fpu_enabled = false,
562 };
563 struct lrw_crypt_req req = {
564 .tbuf = buf,
565 .tbuflen = sizeof(buf),
566
567 .table_ctx = &ctx->lrw_table,
568 .crypt_ctx = &crypt_ctx,
569 .crypt_fn = decrypt_callback,
570 };
571 int ret;
572
573 ret = lrw_crypt(desc, dst, src, nbytes, &req);
574 serpent_fpu_end(crypt_ctx.fpu_enabled);
575
576 return ret;
577 }
578
579 static void lrw_exit_tfm(struct crypto_tfm *tfm)
580 {
581 struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
582
583 lrw_free_table(&ctx->lrw_table);
584 }
585
586 static struct crypto_alg blk_lrw_alg = {
587 .cra_name = "__lrw-serpent-sse2",
588 .cra_driver_name = "__driver-lrw-serpent-sse2",
589 .cra_priority = 0,
590 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
591 .cra_blocksize = SERPENT_BLOCK_SIZE,
592 .cra_ctxsize = sizeof(struct serpent_lrw_ctx),
593 .cra_alignmask = 0,
594 .cra_type = &crypto_blkcipher_type,
595 .cra_module = THIS_MODULE,
596 .cra_list = LIST_HEAD_INIT(blk_lrw_alg.cra_list),
597 .cra_exit = lrw_exit_tfm,
598 .cra_u = {
599 .blkcipher = {
600 .min_keysize = SERPENT_MIN_KEY_SIZE +
601 SERPENT_BLOCK_SIZE,
602 .max_keysize = SERPENT_MAX_KEY_SIZE +
603 SERPENT_BLOCK_SIZE,
604 .ivsize = SERPENT_BLOCK_SIZE,
605 .setkey = lrw_serpent_setkey,
606 .encrypt = lrw_encrypt,
607 .decrypt = lrw_decrypt,
608 },
609 },
610 };
611
612 #endif
613
614 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
615 unsigned int key_len)
616 {
617 struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
618 struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
619 int err;
620
621 crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
622 crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
623 & CRYPTO_TFM_REQ_MASK);
624 err = crypto_ablkcipher_setkey(child, key, key_len);
625 crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
626 & CRYPTO_TFM_RES_MASK);
627 return err;
628 }
629
630 static int __ablk_encrypt(struct ablkcipher_request *req)
631 {
632 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
633 struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
634 struct blkcipher_desc desc;
635
636 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
637 desc.info = req->info;
638 desc.flags = 0;
639
640 return crypto_blkcipher_crt(desc.tfm)->encrypt(
641 &desc, req->dst, req->src, req->nbytes);
642 }
643
644 static int ablk_encrypt(struct ablkcipher_request *req)
645 {
646 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
647 struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
648
649 if (!irq_fpu_usable()) {
650 struct ablkcipher_request *cryptd_req =
651 ablkcipher_request_ctx(req);
652
653 memcpy(cryptd_req, req, sizeof(*req));
654 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
655
656 return crypto_ablkcipher_encrypt(cryptd_req);
657 } else {
658 return __ablk_encrypt(req);
659 }
660 }
661
662 static int ablk_decrypt(struct ablkcipher_request *req)
663 {
664 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
665 struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
666
667 if (!irq_fpu_usable()) {
668 struct ablkcipher_request *cryptd_req =
669 ablkcipher_request_ctx(req);
670
671 memcpy(cryptd_req, req, sizeof(*req));
672 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
673
674 return crypto_ablkcipher_decrypt(cryptd_req);
675 } else {
676 struct blkcipher_desc desc;
677
678 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
679 desc.info = req->info;
680 desc.flags = 0;
681
682 return crypto_blkcipher_crt(desc.tfm)->decrypt(
683 &desc, req->dst, req->src, req->nbytes);
684 }
685 }
686
687 static void ablk_exit(struct crypto_tfm *tfm)
688 {
689 struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm);
690
691 cryptd_free_ablkcipher(ctx->cryptd_tfm);
692 }
693
694 static void ablk_init_common(struct crypto_tfm *tfm,
695 struct cryptd_ablkcipher *cryptd_tfm)
696 {
697 struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm);
698
699 ctx->cryptd_tfm = cryptd_tfm;
700 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
701 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
702 }
703
704 static int ablk_ecb_init(struct crypto_tfm *tfm)
705 {
706 struct cryptd_ablkcipher *cryptd_tfm;
707
708 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-serpent-sse2", 0, 0);
709 if (IS_ERR(cryptd_tfm))
710 return PTR_ERR(cryptd_tfm);
711 ablk_init_common(tfm, cryptd_tfm);
712 return 0;
713 }
714
715 static struct crypto_alg ablk_ecb_alg = {
716 .cra_name = "ecb(serpent)",
717 .cra_driver_name = "ecb-serpent-sse2",
718 .cra_priority = 400,
719 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
720 .cra_blocksize = SERPENT_BLOCK_SIZE,
721 .cra_ctxsize = sizeof(struct async_serpent_ctx),
722 .cra_alignmask = 0,
723 .cra_type = &crypto_ablkcipher_type,
724 .cra_module = THIS_MODULE,
725 .cra_list = LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
726 .cra_init = ablk_ecb_init,
727 .cra_exit = ablk_exit,
728 .cra_u = {
729 .ablkcipher = {
730 .min_keysize = SERPENT_MIN_KEY_SIZE,
731 .max_keysize = SERPENT_MAX_KEY_SIZE,
732 .setkey = ablk_set_key,
733 .encrypt = ablk_encrypt,
734 .decrypt = ablk_decrypt,
735 },
736 },
737 };
738
739 static int ablk_cbc_init(struct crypto_tfm *tfm)
740 {
741 struct cryptd_ablkcipher *cryptd_tfm;
742
743 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-serpent-sse2", 0, 0);
744 if (IS_ERR(cryptd_tfm))
745 return PTR_ERR(cryptd_tfm);
746 ablk_init_common(tfm, cryptd_tfm);
747 return 0;
748 }
749
750 static struct crypto_alg ablk_cbc_alg = {
751 .cra_name = "cbc(serpent)",
752 .cra_driver_name = "cbc-serpent-sse2",
753 .cra_priority = 400,
754 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
755 .cra_blocksize = SERPENT_BLOCK_SIZE,
756 .cra_ctxsize = sizeof(struct async_serpent_ctx),
757 .cra_alignmask = 0,
758 .cra_type = &crypto_ablkcipher_type,
759 .cra_module = THIS_MODULE,
760 .cra_list = LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
761 .cra_init = ablk_cbc_init,
762 .cra_exit = ablk_exit,
763 .cra_u = {
764 .ablkcipher = {
765 .min_keysize = SERPENT_MIN_KEY_SIZE,
766 .max_keysize = SERPENT_MAX_KEY_SIZE,
767 .ivsize = SERPENT_BLOCK_SIZE,
768 .setkey = ablk_set_key,
769 .encrypt = __ablk_encrypt,
770 .decrypt = ablk_decrypt,
771 },
772 },
773 };
774
775 static int ablk_ctr_init(struct crypto_tfm *tfm)
776 {
777 struct cryptd_ablkcipher *cryptd_tfm;
778
779 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-serpent-sse2", 0, 0);
780 if (IS_ERR(cryptd_tfm))
781 return PTR_ERR(cryptd_tfm);
782 ablk_init_common(tfm, cryptd_tfm);
783 return 0;
784 }
785
786 static struct crypto_alg ablk_ctr_alg = {
787 .cra_name = "ctr(serpent)",
788 .cra_driver_name = "ctr-serpent-sse2",
789 .cra_priority = 400,
790 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
791 .cra_blocksize = 1,
792 .cra_ctxsize = sizeof(struct async_serpent_ctx),
793 .cra_alignmask = 0,
794 .cra_type = &crypto_ablkcipher_type,
795 .cra_module = THIS_MODULE,
796 .cra_list = LIST_HEAD_INIT(ablk_ctr_alg.cra_list),
797 .cra_init = ablk_ctr_init,
798 .cra_exit = ablk_exit,
799 .cra_u = {
800 .ablkcipher = {
801 .min_keysize = SERPENT_MIN_KEY_SIZE,
802 .max_keysize = SERPENT_MAX_KEY_SIZE,
803 .ivsize = SERPENT_BLOCK_SIZE,
804 .setkey = ablk_set_key,
805 .encrypt = ablk_encrypt,
806 .decrypt = ablk_encrypt,
807 .geniv = "chainiv",
808 },
809 },
810 };
811
812 #ifdef HAS_LRW
813
814 static int ablk_lrw_init(struct crypto_tfm *tfm)
815 {
816 struct cryptd_ablkcipher *cryptd_tfm;
817
818 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-lrw-serpent-sse2", 0, 0);
819 if (IS_ERR(cryptd_tfm))
820 return PTR_ERR(cryptd_tfm);
821 ablk_init_common(tfm, cryptd_tfm);
822 return 0;
823 }
824
825 static struct crypto_alg ablk_lrw_alg = {
826 .cra_name = "lrw(serpent)",
827 .cra_driver_name = "lrw-serpent-sse2",
828 .cra_priority = 400,
829 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
830 .cra_blocksize = SERPENT_BLOCK_SIZE,
831 .cra_ctxsize = sizeof(struct async_serpent_ctx),
832 .cra_alignmask = 0,
833 .cra_type = &crypto_ablkcipher_type,
834 .cra_module = THIS_MODULE,
835 .cra_list = LIST_HEAD_INIT(ablk_lrw_alg.cra_list),
836 .cra_init = ablk_lrw_init,
837 .cra_exit = ablk_exit,
838 .cra_u = {
839 .ablkcipher = {
840 .min_keysize = SERPENT_MIN_KEY_SIZE +
841 SERPENT_BLOCK_SIZE,
842 .max_keysize = SERPENT_MAX_KEY_SIZE +
843 SERPENT_BLOCK_SIZE,
844 .ivsize = SERPENT_BLOCK_SIZE,
845 .setkey = ablk_set_key,
846 .encrypt = ablk_encrypt,
847 .decrypt = ablk_decrypt,
848 },
849 },
850 };
851
852 #endif
853
854 static int __init serpent_sse2_init(void)
855 {
856 int err;
857
858 if (!cpu_has_xmm2) {
859 printk(KERN_INFO "SSE2 instructions are not detected.\n");
860 return -ENODEV;
861 }
862
863 err = crypto_register_alg(&blk_ecb_alg);
864 if (err)
865 goto blk_ecb_err;
866 err = crypto_register_alg(&blk_cbc_alg);
867 if (err)
868 goto blk_cbc_err;
869 err = crypto_register_alg(&blk_ctr_alg);
870 if (err)
871 goto blk_ctr_err;
872 err = crypto_register_alg(&ablk_ecb_alg);
873 if (err)
874 goto ablk_ecb_err;
875 err = crypto_register_alg(&ablk_cbc_alg);
876 if (err)
877 goto ablk_cbc_err;
878 err = crypto_register_alg(&ablk_ctr_alg);
879 if (err)
880 goto ablk_ctr_err;
881 #ifdef HAS_LRW
882 err = crypto_register_alg(&blk_lrw_alg);
883 if (err)
884 goto blk_lrw_err;
885 err = crypto_register_alg(&ablk_lrw_alg);
886 if (err)
887 goto ablk_lrw_err;
888 #endif
889 return err;
890
891 #ifdef HAS_LRW
892 ablk_lrw_err:
893 crypto_unregister_alg(&blk_lrw_alg);
894 blk_lrw_err:
895 crypto_unregister_alg(&ablk_ctr_alg);
896 #endif
897 ablk_ctr_err:
898 crypto_unregister_alg(&ablk_cbc_alg);
899 ablk_cbc_err:
900 crypto_unregister_alg(&ablk_ecb_alg);
901 ablk_ecb_err:
902 crypto_unregister_alg(&blk_ctr_alg);
903 blk_ctr_err:
904 crypto_unregister_alg(&blk_cbc_alg);
905 blk_cbc_err:
906 crypto_unregister_alg(&blk_ecb_alg);
907 blk_ecb_err:
908 return err;
909 }
910
911 static void __exit serpent_sse2_exit(void)
912 {
913 #ifdef HAS_LRW
914 crypto_unregister_alg(&ablk_lrw_alg);
915 crypto_unregister_alg(&blk_lrw_alg);
916 #endif
917 crypto_unregister_alg(&ablk_ctr_alg);
918 crypto_unregister_alg(&ablk_cbc_alg);
919 crypto_unregister_alg(&ablk_ecb_alg);
920 crypto_unregister_alg(&blk_ctr_alg);
921 crypto_unregister_alg(&blk_cbc_alg);
922 crypto_unregister_alg(&blk_ecb_alg);
923 }
924
925 module_init(serpent_sse2_init);
926 module_exit(serpent_sse2_exit);
927
928 MODULE_DESCRIPTION("Serpent Cipher Algorithm, SSE2 optimized");
929 MODULE_LICENSE("GPL");
930 MODULE_ALIAS("serpent");
(deprecated) Btrfs devel tree