2 * Glue Code for AVX assembler version of Twofish Cipher
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
24 #include <linux/module.h>
25 #include <linux/hardirq.h>
26 #include <linux/types.h>
27 #include <linux/crypto.h>
28 #include <linux/err.h>
29 #include <crypto/algapi.h>
30 #include <crypto/twofish.h>
31 #include <crypto/cryptd.h>
32 #include <crypto/b128ops.h>
33 #include <crypto/ctr.h>
34 #include <crypto/lrw.h>
35 #include <crypto/xts.h>
38 #include <asm/xsave.h>
39 #include <asm/crypto/twofish.h>
40 #include <asm/crypto/ablk_helper.h>
41 #include <asm/crypto/glue_helper.h>
42 #include <crypto/scatterwalk.h>
43 #include <linux/workqueue.h>
44 #include <linux/spinlock.h>
46 #define TWOFISH_PARALLEL_BLOCKS 8
48 /* 8-way parallel cipher functions */
49 asmlinkage
void twofish_ecb_enc_8way(struct twofish_ctx
*ctx
, u8
*dst
,
51 asmlinkage
void twofish_ecb_dec_8way(struct twofish_ctx
*ctx
, u8
*dst
,
54 asmlinkage
void twofish_cbc_dec_8way(struct twofish_ctx
*ctx
, u8
*dst
,
56 asmlinkage
void twofish_ctr_8way(struct twofish_ctx
*ctx
, u8
*dst
,
57 const u8
*src
, le128
*iv
);
59 static inline void twofish_enc_blk_3way(struct twofish_ctx
*ctx
, u8
*dst
,
62 __twofish_enc_blk_3way(ctx
, dst
, src
, false);
66 static const struct common_glue_ctx twofish_enc
= {
68 .fpu_blocks_limit
= TWOFISH_PARALLEL_BLOCKS
,
71 .num_blocks
= TWOFISH_PARALLEL_BLOCKS
,
72 .fn_u
= { .ecb
= GLUE_FUNC_CAST(twofish_ecb_enc_8way
) }
75 .fn_u
= { .ecb
= GLUE_FUNC_CAST(twofish_enc_blk_3way
) }
78 .fn_u
= { .ecb
= GLUE_FUNC_CAST(twofish_enc_blk
) }
82 static const struct common_glue_ctx twofish_ctr
= {
84 .fpu_blocks_limit
= TWOFISH_PARALLEL_BLOCKS
,
87 .num_blocks
= TWOFISH_PARALLEL_BLOCKS
,
88 .fn_u
= { .ctr
= GLUE_CTR_FUNC_CAST(twofish_ctr_8way
) }
91 .fn_u
= { .ctr
= GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way
) }
94 .fn_u
= { .ctr
= GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr
) }
98 static const struct common_glue_ctx twofish_dec
= {
100 .fpu_blocks_limit
= TWOFISH_PARALLEL_BLOCKS
,
103 .num_blocks
= TWOFISH_PARALLEL_BLOCKS
,
104 .fn_u
= { .ecb
= GLUE_FUNC_CAST(twofish_ecb_dec_8way
) }
107 .fn_u
= { .ecb
= GLUE_FUNC_CAST(twofish_dec_blk_3way
) }
110 .fn_u
= { .ecb
= GLUE_FUNC_CAST(twofish_dec_blk
) }
114 static const struct common_glue_ctx twofish_dec_cbc
= {
116 .fpu_blocks_limit
= TWOFISH_PARALLEL_BLOCKS
,
119 .num_blocks
= TWOFISH_PARALLEL_BLOCKS
,
120 .fn_u
= { .cbc
= GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way
) }
123 .fn_u
= { .cbc
= GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way
) }
126 .fn_u
= { .cbc
= GLUE_CBC_FUNC_CAST(twofish_dec_blk
) }
130 static int ecb_encrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
131 struct scatterlist
*src
, unsigned int nbytes
)
133 return glue_ecb_crypt_128bit(&twofish_enc
, desc
, dst
, src
, nbytes
);
136 static int ecb_decrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
137 struct scatterlist
*src
, unsigned int nbytes
)
139 return glue_ecb_crypt_128bit(&twofish_dec
, desc
, dst
, src
, nbytes
);
142 static int cbc_encrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
143 struct scatterlist
*src
, unsigned int nbytes
)
145 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk
), desc
,
149 static int cbc_decrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
150 struct scatterlist
*src
, unsigned int nbytes
)
152 return glue_cbc_decrypt_128bit(&twofish_dec_cbc
, desc
, dst
, src
,
156 static int ctr_crypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
157 struct scatterlist
*src
, unsigned int nbytes
)
159 return glue_ctr_crypt_128bit(&twofish_ctr
, desc
, dst
, src
, nbytes
);
162 static inline bool twofish_fpu_begin(bool fpu_enabled
, unsigned int nbytes
)
164 return glue_fpu_begin(TF_BLOCK_SIZE
, TWOFISH_PARALLEL_BLOCKS
, NULL
,
165 fpu_enabled
, nbytes
);
168 static inline void twofish_fpu_end(bool fpu_enabled
)
170 glue_fpu_end(fpu_enabled
);
174 struct twofish_ctx
*ctx
;
178 static void encrypt_callback(void *priv
, u8
*srcdst
, unsigned int nbytes
)
180 const unsigned int bsize
= TF_BLOCK_SIZE
;
181 struct crypt_priv
*ctx
= priv
;
184 ctx
->fpu_enabled
= twofish_fpu_begin(ctx
->fpu_enabled
, nbytes
);
186 if (nbytes
== bsize
* TWOFISH_PARALLEL_BLOCKS
) {
187 twofish_ecb_enc_8way(ctx
->ctx
, srcdst
, srcdst
);
191 for (i
= 0; i
< nbytes
/ (bsize
* 3); i
++, srcdst
+= bsize
* 3)
192 twofish_enc_blk_3way(ctx
->ctx
, srcdst
, srcdst
);
196 for (i
= 0; i
< nbytes
/ bsize
; i
++, srcdst
+= bsize
)
197 twofish_enc_blk(ctx
->ctx
, srcdst
, srcdst
);
200 static void decrypt_callback(void *priv
, u8
*srcdst
, unsigned int nbytes
)
202 const unsigned int bsize
= TF_BLOCK_SIZE
;
203 struct crypt_priv
*ctx
= priv
;
206 ctx
->fpu_enabled
= twofish_fpu_begin(ctx
->fpu_enabled
, nbytes
);
208 if (nbytes
== bsize
* TWOFISH_PARALLEL_BLOCKS
) {
209 twofish_ecb_dec_8way(ctx
->ctx
, srcdst
, srcdst
);
213 for (i
= 0; i
< nbytes
/ (bsize
* 3); i
++, srcdst
+= bsize
* 3)
214 twofish_dec_blk_3way(ctx
->ctx
, srcdst
, srcdst
);
218 for (i
= 0; i
< nbytes
/ bsize
; i
++, srcdst
+= bsize
)
219 twofish_dec_blk(ctx
->ctx
, srcdst
, srcdst
);
222 static int lrw_encrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
223 struct scatterlist
*src
, unsigned int nbytes
)
225 struct twofish_lrw_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
226 be128 buf
[TWOFISH_PARALLEL_BLOCKS
];
227 struct crypt_priv crypt_ctx
= {
228 .ctx
= &ctx
->twofish_ctx
,
229 .fpu_enabled
= false,
231 struct lrw_crypt_req req
= {
233 .tbuflen
= sizeof(buf
),
235 .table_ctx
= &ctx
->lrw_table
,
236 .crypt_ctx
= &crypt_ctx
,
237 .crypt_fn
= encrypt_callback
,
241 desc
->flags
&= ~CRYPTO_TFM_REQ_MAY_SLEEP
;
242 ret
= lrw_crypt(desc
, dst
, src
, nbytes
, &req
);
243 twofish_fpu_end(crypt_ctx
.fpu_enabled
);
248 static int lrw_decrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
249 struct scatterlist
*src
, unsigned int nbytes
)
251 struct twofish_lrw_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
252 be128 buf
[TWOFISH_PARALLEL_BLOCKS
];
253 struct crypt_priv crypt_ctx
= {
254 .ctx
= &ctx
->twofish_ctx
,
255 .fpu_enabled
= false,
257 struct lrw_crypt_req req
= {
259 .tbuflen
= sizeof(buf
),
261 .table_ctx
= &ctx
->lrw_table
,
262 .crypt_ctx
= &crypt_ctx
,
263 .crypt_fn
= decrypt_callback
,
267 desc
->flags
&= ~CRYPTO_TFM_REQ_MAY_SLEEP
;
268 ret
= lrw_crypt(desc
, dst
, src
, nbytes
, &req
);
269 twofish_fpu_end(crypt_ctx
.fpu_enabled
);
274 static int xts_encrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
275 struct scatterlist
*src
, unsigned int nbytes
)
277 struct twofish_xts_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
278 be128 buf
[TWOFISH_PARALLEL_BLOCKS
];
279 struct crypt_priv crypt_ctx
= {
280 .ctx
= &ctx
->crypt_ctx
,
281 .fpu_enabled
= false,
283 struct xts_crypt_req req
= {
285 .tbuflen
= sizeof(buf
),
287 .tweak_ctx
= &ctx
->tweak_ctx
,
288 .tweak_fn
= XTS_TWEAK_CAST(twofish_enc_blk
),
289 .crypt_ctx
= &crypt_ctx
,
290 .crypt_fn
= encrypt_callback
,
294 desc
->flags
&= ~CRYPTO_TFM_REQ_MAY_SLEEP
;
295 ret
= xts_crypt(desc
, dst
, src
, nbytes
, &req
);
296 twofish_fpu_end(crypt_ctx
.fpu_enabled
);
301 static int xts_decrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
302 struct scatterlist
*src
, unsigned int nbytes
)
304 struct twofish_xts_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
305 be128 buf
[TWOFISH_PARALLEL_BLOCKS
];
306 struct crypt_priv crypt_ctx
= {
307 .ctx
= &ctx
->crypt_ctx
,
308 .fpu_enabled
= false,
310 struct xts_crypt_req req
= {
312 .tbuflen
= sizeof(buf
),
314 .tweak_ctx
= &ctx
->tweak_ctx
,
315 .tweak_fn
= XTS_TWEAK_CAST(twofish_enc_blk
),
316 .crypt_ctx
= &crypt_ctx
,
317 .crypt_fn
= decrypt_callback
,
321 desc
->flags
&= ~CRYPTO_TFM_REQ_MAY_SLEEP
;
322 ret
= xts_crypt(desc
, dst
, src
, nbytes
, &req
);
323 twofish_fpu_end(crypt_ctx
.fpu_enabled
);
328 static struct crypto_alg twofish_algs
[10] = { {
329 .cra_name
= "__ecb-twofish-avx",
330 .cra_driver_name
= "__driver-ecb-twofish-avx",
332 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
333 .cra_blocksize
= TF_BLOCK_SIZE
,
334 .cra_ctxsize
= sizeof(struct twofish_ctx
),
336 .cra_type
= &crypto_blkcipher_type
,
337 .cra_module
= THIS_MODULE
,
340 .min_keysize
= TF_MIN_KEY_SIZE
,
341 .max_keysize
= TF_MAX_KEY_SIZE
,
342 .setkey
= twofish_setkey
,
343 .encrypt
= ecb_encrypt
,
344 .decrypt
= ecb_decrypt
,
348 .cra_name
= "__cbc-twofish-avx",
349 .cra_driver_name
= "__driver-cbc-twofish-avx",
351 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
352 .cra_blocksize
= TF_BLOCK_SIZE
,
353 .cra_ctxsize
= sizeof(struct twofish_ctx
),
355 .cra_type
= &crypto_blkcipher_type
,
356 .cra_module
= THIS_MODULE
,
359 .min_keysize
= TF_MIN_KEY_SIZE
,
360 .max_keysize
= TF_MAX_KEY_SIZE
,
361 .setkey
= twofish_setkey
,
362 .encrypt
= cbc_encrypt
,
363 .decrypt
= cbc_decrypt
,
367 .cra_name
= "__ctr-twofish-avx",
368 .cra_driver_name
= "__driver-ctr-twofish-avx",
370 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
372 .cra_ctxsize
= sizeof(struct twofish_ctx
),
374 .cra_type
= &crypto_blkcipher_type
,
375 .cra_module
= THIS_MODULE
,
378 .min_keysize
= TF_MIN_KEY_SIZE
,
379 .max_keysize
= TF_MAX_KEY_SIZE
,
380 .ivsize
= TF_BLOCK_SIZE
,
381 .setkey
= twofish_setkey
,
382 .encrypt
= ctr_crypt
,
383 .decrypt
= ctr_crypt
,
387 .cra_name
= "__lrw-twofish-avx",
388 .cra_driver_name
= "__driver-lrw-twofish-avx",
390 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
391 .cra_blocksize
= TF_BLOCK_SIZE
,
392 .cra_ctxsize
= sizeof(struct twofish_lrw_ctx
),
394 .cra_type
= &crypto_blkcipher_type
,
395 .cra_module
= THIS_MODULE
,
396 .cra_exit
= lrw_twofish_exit_tfm
,
399 .min_keysize
= TF_MIN_KEY_SIZE
+
401 .max_keysize
= TF_MAX_KEY_SIZE
+
403 .ivsize
= TF_BLOCK_SIZE
,
404 .setkey
= lrw_twofish_setkey
,
405 .encrypt
= lrw_encrypt
,
406 .decrypt
= lrw_decrypt
,
410 .cra_name
= "__xts-twofish-avx",
411 .cra_driver_name
= "__driver-xts-twofish-avx",
413 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
414 .cra_blocksize
= TF_BLOCK_SIZE
,
415 .cra_ctxsize
= sizeof(struct twofish_xts_ctx
),
417 .cra_type
= &crypto_blkcipher_type
,
418 .cra_module
= THIS_MODULE
,
421 .min_keysize
= TF_MIN_KEY_SIZE
* 2,
422 .max_keysize
= TF_MAX_KEY_SIZE
* 2,
423 .ivsize
= TF_BLOCK_SIZE
,
424 .setkey
= xts_twofish_setkey
,
425 .encrypt
= xts_encrypt
,
426 .decrypt
= xts_decrypt
,
430 .cra_name
= "ecb(twofish)",
431 .cra_driver_name
= "ecb-twofish-avx",
433 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
| CRYPTO_ALG_ASYNC
,
434 .cra_blocksize
= TF_BLOCK_SIZE
,
435 .cra_ctxsize
= sizeof(struct async_helper_ctx
),
437 .cra_type
= &crypto_ablkcipher_type
,
438 .cra_module
= THIS_MODULE
,
439 .cra_init
= ablk_init
,
440 .cra_exit
= ablk_exit
,
443 .min_keysize
= TF_MIN_KEY_SIZE
,
444 .max_keysize
= TF_MAX_KEY_SIZE
,
445 .setkey
= ablk_set_key
,
446 .encrypt
= ablk_encrypt
,
447 .decrypt
= ablk_decrypt
,
451 .cra_name
= "cbc(twofish)",
452 .cra_driver_name
= "cbc-twofish-avx",
454 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
| CRYPTO_ALG_ASYNC
,
455 .cra_blocksize
= TF_BLOCK_SIZE
,
456 .cra_ctxsize
= sizeof(struct async_helper_ctx
),
458 .cra_type
= &crypto_ablkcipher_type
,
459 .cra_module
= THIS_MODULE
,
460 .cra_init
= ablk_init
,
461 .cra_exit
= ablk_exit
,
464 .min_keysize
= TF_MIN_KEY_SIZE
,
465 .max_keysize
= TF_MAX_KEY_SIZE
,
466 .ivsize
= TF_BLOCK_SIZE
,
467 .setkey
= ablk_set_key
,
468 .encrypt
= __ablk_encrypt
,
469 .decrypt
= ablk_decrypt
,
473 .cra_name
= "ctr(twofish)",
474 .cra_driver_name
= "ctr-twofish-avx",
476 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
| CRYPTO_ALG_ASYNC
,
478 .cra_ctxsize
= sizeof(struct async_helper_ctx
),
480 .cra_type
= &crypto_ablkcipher_type
,
481 .cra_module
= THIS_MODULE
,
482 .cra_init
= ablk_init
,
483 .cra_exit
= ablk_exit
,
486 .min_keysize
= TF_MIN_KEY_SIZE
,
487 .max_keysize
= TF_MAX_KEY_SIZE
,
488 .ivsize
= TF_BLOCK_SIZE
,
489 .setkey
= ablk_set_key
,
490 .encrypt
= ablk_encrypt
,
491 .decrypt
= ablk_encrypt
,
496 .cra_name
= "lrw(twofish)",
497 .cra_driver_name
= "lrw-twofish-avx",
499 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
| CRYPTO_ALG_ASYNC
,
500 .cra_blocksize
= TF_BLOCK_SIZE
,
501 .cra_ctxsize
= sizeof(struct async_helper_ctx
),
503 .cra_type
= &crypto_ablkcipher_type
,
504 .cra_module
= THIS_MODULE
,
505 .cra_init
= ablk_init
,
506 .cra_exit
= ablk_exit
,
509 .min_keysize
= TF_MIN_KEY_SIZE
+
511 .max_keysize
= TF_MAX_KEY_SIZE
+
513 .ivsize
= TF_BLOCK_SIZE
,
514 .setkey
= ablk_set_key
,
515 .encrypt
= ablk_encrypt
,
516 .decrypt
= ablk_decrypt
,
520 .cra_name
= "xts(twofish)",
521 .cra_driver_name
= "xts-twofish-avx",
523 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
| CRYPTO_ALG_ASYNC
,
524 .cra_blocksize
= TF_BLOCK_SIZE
,
525 .cra_ctxsize
= sizeof(struct async_helper_ctx
),
527 .cra_type
= &crypto_ablkcipher_type
,
528 .cra_module
= THIS_MODULE
,
529 .cra_init
= ablk_init
,
530 .cra_exit
= ablk_exit
,
533 .min_keysize
= TF_MIN_KEY_SIZE
* 2,
534 .max_keysize
= TF_MAX_KEY_SIZE
* 2,
535 .ivsize
= TF_BLOCK_SIZE
,
536 .setkey
= ablk_set_key
,
537 .encrypt
= ablk_encrypt
,
538 .decrypt
= ablk_decrypt
,
543 static int __init
twofish_init(void)
547 if (!cpu_has_avx
|| !cpu_has_osxsave
) {
548 printk(KERN_INFO
"AVX instructions are not detected.\n");
552 xcr0
= xgetbv(XCR_XFEATURE_ENABLED_MASK
);
553 if ((xcr0
& (XSTATE_SSE
| XSTATE_YMM
)) != (XSTATE_SSE
| XSTATE_YMM
)) {
554 printk(KERN_INFO
"AVX detected but unusable.\n");
558 return crypto_register_algs(twofish_algs
, ARRAY_SIZE(twofish_algs
));
561 static void __exit
twofish_exit(void)
563 crypto_unregister_algs(twofish_algs
, ARRAY_SIZE(twofish_algs
));
566 module_init(twofish_init
);
567 module_exit(twofish_exit
);
569 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
570 MODULE_LICENSE("GPL");
571 MODULE_ALIAS("twofish");