arch/x86/crypto/serpent_sse2_glue.c

   1 /*
   2  * Glue Code for SSE2 assembler versions of Serpent Cipher
   3  *
   4  * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   5  *
   6  * Glue code based on aesni-intel_glue.c by:
   7  *  Copyright (C) 2008, Intel Corp.
   8  *    Author: Huang Ying <ying.huang@intel.com>
   9  *
  10  * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
  11  *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  12  * CTR part based on code (crypto/ctr.c) by:
  13  *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
  14  *
  15  * This program is free software; you can redistribute it and/or modify
  16  * it under the terms of the GNU General Public License as published by
  17  * the Free Software Foundation; either version 2 of the License, or
  18  * (at your option) any later version.
  19  *
  20  * This program is distributed in the hope that it will be useful,
  21  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  22  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  23  * GNU General Public License for more details.
  24  *
  25  * You should have received a copy of the GNU General Public License
  26  * along with this program; if not, write to the Free Software
  27  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  28  * USA
  29  *
  30  */
  31
  32 #include <linux/module.h>
  33 #include <linux/hardirq.h>
  34 #include <linux/types.h>
  35 #include <linux/crypto.h>
  36 #include <linux/err.h>
  37 #include <crypto/algapi.h>
  38 #include <crypto/serpent.h>
  39 #include <crypto/cryptd.h>
  40 #include <crypto/b128ops.h>
  41 #include <crypto/ctr.h>
  42 #include <crypto/lrw.h>
  43 #include <crypto/xts.h>
  44 #include <asm/i387.h>
  45 #include <asm/serpent.h>
  46 #include <crypto/scatterwalk.h>
  47 #include <linux/workqueue.h>
  48 #include <linux/spinlock.h>
  49
  50 struct async_serpent_ctx {
  51         struct cryptd_ablkcipher *cryptd_tfm;
  52 };
  53
  54 static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
  55 {
  56         if (fpu_enabled)
  57                 return true;
  58
  59         /* SSE2 is only used when chunk to be processed is large enough, so
  60          * do not enable FPU until it is necessary.
  61          */
  62         if (nbytes < SERPENT_BLOCK_SIZE * SERPENT_PARALLEL_BLOCKS)
  63                 return false;
  64
  65         kernel_fpu_begin();
  66         return true;
  67 }
  68
  69 static inline void serpent_fpu_end(bool fpu_enabled)
  70 {
  71         if (fpu_enabled)
  72                 kernel_fpu_end();
  73 }
  74
  75 static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
  76                      bool enc)
  77 {
  78         bool fpu_enabled = false;
  79         struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  80         const unsigned int bsize = SERPENT_BLOCK_SIZE;
  81         unsigned int nbytes;
  82         int err;
  83
  84         err = blkcipher_walk_virt(desc, walk);
  85         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  86
  87         while ((nbytes = walk->nbytes)) {
  88                 u8 *wsrc = walk->src.virt.addr;
  89                 u8 *wdst = walk->dst.virt.addr;
  90
  91                 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
  92
  93                 /* Process multi-block batch */
  94                 if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
  95                         do {
  96                                 if (enc)
  97                                         serpent_enc_blk_xway(ctx, wdst, wsrc);
  98                                 else
  99                                         serpent_dec_blk_xway(ctx, wdst, wsrc);
 100
 101                                 wsrc += bsize * SERPENT_PARALLEL_BLOCKS;
 102                                 wdst += bsize * SERPENT_PARALLEL_BLOCKS;
 103                                 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
 104                         } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
 105
 106                         if (nbytes < bsize)
 107                                 goto done;
 108                 }
 109
 110                 /* Handle leftovers */
 111                 do {
 112                         if (enc)
 113                                 __serpent_encrypt(ctx, wdst, wsrc);
 114                         else
 115                                 __serpent_decrypt(ctx, wdst, wsrc);
 116
 117                         wsrc += bsize;
 118                         wdst += bsize;
 119                         nbytes -= bsize;
 120                 } while (nbytes >= bsize);
 121
 122 done:
 123                 err = blkcipher_walk_done(desc, walk, nbytes);
 124         }
 125
 126         serpent_fpu_end(fpu_enabled);
 127         return err;
 128 }
 129
 130 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 131                        struct scatterlist *src, unsigned int nbytes)
 132 {
 133         struct blkcipher_walk walk;
 134
 135         blkcipher_walk_init(&walk, dst, src, nbytes);
 136         return ecb_crypt(desc, &walk, true);
 137 }
 138
 139 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 140                        struct scatterlist *src, unsigned int nbytes)
 141 {
 142         struct blkcipher_walk walk;
 143
 144         blkcipher_walk_init(&walk, dst, src, nbytes);
 145         return ecb_crypt(desc, &walk, false);
 146 }
 147
 148 static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
 149                                   struct blkcipher_walk *walk)
 150 {
 151         struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 152         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 153         unsigned int nbytes = walk->nbytes;
 154         u128 *src = (u128 *)walk->src.virt.addr;
 155         u128 *dst = (u128 *)walk->dst.virt.addr;
 156         u128 *iv = (u128 *)walk->iv;
 157
 158         do {
 159                 u128_xor(dst, src, iv);
 160                 __serpent_encrypt(ctx, (u8 *)dst, (u8 *)dst);
 161                 iv = dst;
 162
 163                 src += 1;
 164                 dst += 1;
 165                 nbytes -= bsize;
 166         } while (nbytes >= bsize);
 167
 168         u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
 169         return nbytes;
 170 }
 171
 172 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 173                        struct scatterlist *src, unsigned int nbytes)
 174 {
 175         struct blkcipher_walk walk;
 176         int err;
 177
 178         blkcipher_walk_init(&walk, dst, src, nbytes);
 179         err = blkcipher_walk_virt(desc, &walk);
 180
 181         while ((nbytes = walk.nbytes)) {
 182                 nbytes = __cbc_encrypt(desc, &walk);
 183                 err = blkcipher_walk_done(desc, &walk, nbytes);
 184         }
 185
 186         return err;
 187 }
 188
 189 static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
 190                                   struct blkcipher_walk *walk)
 191 {
 192         struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 193         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 194         unsigned int nbytes = walk->nbytes;
 195         u128 *src = (u128 *)walk->src.virt.addr;
 196         u128 *dst = (u128 *)walk->dst.virt.addr;
 197         u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
 198         u128 last_iv;
 199         int i;
 200
 201         /* Start of the last block. */
 202         src += nbytes / bsize - 1;
 203         dst += nbytes / bsize - 1;
 204
 205         last_iv = *src;
 206
 207         /* Process multi-block batch */
 208         if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
 209                 do {
 210                         nbytes -= bsize * (SERPENT_PARALLEL_BLOCKS - 1);
 211                         src -= SERPENT_PARALLEL_BLOCKS - 1;
 212                         dst -= SERPENT_PARALLEL_BLOCKS - 1;
 213
 214                         for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++)
 215                                 ivs[i] = src[i];
 216
 217                         serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
 218
 219                         for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++)
 220                                 u128_xor(dst + (i + 1), dst + (i + 1), ivs + i);
 221
 222                         nbytes -= bsize;
 223                         if (nbytes < bsize)
 224                                 goto done;
 225
 226                         u128_xor(dst, dst, src - 1);
 227                         src -= 1;
 228                         dst -= 1;
 229                 } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
 230
 231                 if (nbytes < bsize)
 232                         goto done;
 233         }
 234
 235         /* Handle leftovers */
 236         for (;;) {
 237                 __serpent_decrypt(ctx, (u8 *)dst, (u8 *)src);
 238
 239                 nbytes -= bsize;
 240                 if (nbytes < bsize)
 241                         break;
 242
 243                 u128_xor(dst, dst, src - 1);
 244                 src -= 1;
 245                 dst -= 1;
 246         }
 247
 248 done:
 249         u128_xor(dst, dst, (u128 *)walk->iv);
 250         *(u128 *)walk->iv = last_iv;
 251
 252         return nbytes;
 253 }
 254
 255 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 256                        struct scatterlist *src, unsigned int nbytes)
 257 {
 258         bool fpu_enabled = false;
 259         struct blkcipher_walk walk;
 260         int err;
 261
 262         blkcipher_walk_init(&walk, dst, src, nbytes);
 263         err = blkcipher_walk_virt(desc, &walk);
 264         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 265
 266         while ((nbytes = walk.nbytes)) {
 267                 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
 268                 nbytes = __cbc_decrypt(desc, &walk);
 269                 err = blkcipher_walk_done(desc, &walk, nbytes);
 270         }
 271
 272         serpent_fpu_end(fpu_enabled);
 273         return err;
 274 }
 275
 276 static inline void u128_to_be128(be128 *dst, const u128 *src)
 277 {
 278         dst->a = cpu_to_be64(src->a);
 279         dst->b = cpu_to_be64(src->b);
 280 }
 281
 282 static inline void be128_to_u128(u128 *dst, const be128 *src)
 283 {
 284         dst->a = be64_to_cpu(src->a);
 285         dst->b = be64_to_cpu(src->b);
 286 }
 287
 288 static inline void u128_inc(u128 *i)
 289 {
 290         i->b++;
 291         if (!i->b)
 292                 i->a++;
 293 }
 294
 295 static void ctr_crypt_final(struct blkcipher_desc *desc,
 296                             struct blkcipher_walk *walk)
 297 {
 298         struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 299         u8 *ctrblk = walk->iv;
 300         u8 keystream[SERPENT_BLOCK_SIZE];
 301         u8 *src = walk->src.virt.addr;
 302         u8 *dst = walk->dst.virt.addr;
 303         unsigned int nbytes = walk->nbytes;
 304
 305         __serpent_encrypt(ctx, keystream, ctrblk);
 306         crypto_xor(keystream, src, nbytes);
 307         memcpy(dst, keystream, nbytes);
 308
 309         crypto_inc(ctrblk, SERPENT_BLOCK_SIZE);
 310 }
 311
 312 static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
 313                                 struct blkcipher_walk *walk)
 314 {
 315         struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 316         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 317         unsigned int nbytes = walk->nbytes;
 318         u128 *src = (u128 *)walk->src.virt.addr;
 319         u128 *dst = (u128 *)walk->dst.virt.addr;
 320         u128 ctrblk;
 321         be128 ctrblocks[SERPENT_PARALLEL_BLOCKS];
 322         int i;
 323
 324         be128_to_u128(&ctrblk, (be128 *)walk->iv);
 325
 326         /* Process multi-block batch */
 327         if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
 328                 do {
 329                         /* create ctrblks for parallel encrypt */
 330                         for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
 331                                 if (dst != src)
 332                                         dst[i] = src[i];
 333
 334                                 u128_to_be128(&ctrblocks[i], &ctrblk);
 335                                 u128_inc(&ctrblk);
 336                         }
 337
 338                         serpent_enc_blk_xway_xor(ctx, (u8 *)dst,
 339                                                  (u8 *)ctrblocks);
 340
 341                         src += SERPENT_PARALLEL_BLOCKS;
 342                         dst += SERPENT_PARALLEL_BLOCKS;
 343                         nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
 344                 } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
 345
 346                 if (nbytes < bsize)
 347                         goto done;
 348         }
 349
 350         /* Handle leftovers */
 351         do {
 352                 if (dst != src)
 353                         *dst = *src;
 354
 355                 u128_to_be128(&ctrblocks[0], &ctrblk);
 356                 u128_inc(&ctrblk);
 357
 358                 __serpent_encrypt(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
 359                 u128_xor(dst, dst, (u128 *)ctrblocks);
 360
 361                 src += 1;
 362                 dst += 1;
 363                 nbytes -= bsize;
 364         } while (nbytes >= bsize);
 365
 366 done:
 367         u128_to_be128((be128 *)walk->iv, &ctrblk);
 368         return nbytes;
 369 }
 370
 371 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 372                      struct scatterlist *src, unsigned int nbytes)
 373 {
 374         bool fpu_enabled = false;
 375         struct blkcipher_walk walk;
 376         int err;
 377
 378         blkcipher_walk_init(&walk, dst, src, nbytes);
 379         err = blkcipher_walk_virt_block(desc, &walk, SERPENT_BLOCK_SIZE);
 380         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 381
 382         while ((nbytes = walk.nbytes) >= SERPENT_BLOCK_SIZE) {
 383                 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
 384                 nbytes = __ctr_crypt(desc, &walk);
 385                 err = blkcipher_walk_done(desc, &walk, nbytes);
 386         }
 387
 388         serpent_fpu_end(fpu_enabled);
 389
 390         if (walk.nbytes) {
 391                 ctr_crypt_final(desc, &walk);
 392                 err = blkcipher_walk_done(desc, &walk, 0);
 393         }
 394
 395         return err;
 396 }
 397
 398 struct crypt_priv {
 399         struct serpent_ctx *ctx;
 400         bool fpu_enabled;
 401 };
 402
 403 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 404 {
 405         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 406         struct crypt_priv *ctx = priv;
 407         int i;
 408
 409         ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
 410
 411         if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
 412                 serpent_enc_blk_xway(ctx->ctx, srcdst, srcdst);
 413                 return;
 414         }
 415
 416         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 417                 __serpent_encrypt(ctx->ctx, srcdst, srcdst);
 418 }
 419
 420 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 421 {
 422         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 423         struct crypt_priv *ctx = priv;
 424         int i;
 425
 426         ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
 427
 428         if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
 429                 serpent_dec_blk_xway(ctx->ctx, srcdst, srcdst);
 430                 return;
 431         }
 432
 433         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 434                 __serpent_decrypt(ctx->ctx, srcdst, srcdst);
 435 }
 436
 437 struct serpent_lrw_ctx {
 438         struct lrw_table_ctx lrw_table;
 439         struct serpent_ctx serpent_ctx;
 440 };
 441
 442 static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
 443                               unsigned int keylen)
 444 {
 445         struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
 446         int err;
 447
 448         err = __serpent_setkey(&ctx->serpent_ctx, key, keylen -
 449                                                         SERPENT_BLOCK_SIZE);
 450         if (err)
 451                 return err;
 452
 453         return lrw_init_table(&ctx->lrw_table, key + keylen -
 454                                                 SERPENT_BLOCK_SIZE);
 455 }
 456
 457 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 458                        struct scatterlist *src, unsigned int nbytes)
 459 {
 460         struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 461         be128 buf[SERPENT_PARALLEL_BLOCKS];
 462         struct crypt_priv crypt_ctx = {
 463                 .ctx = &ctx->serpent_ctx,
 464                 .fpu_enabled = false,
 465         };
 466         struct lrw_crypt_req req = {
 467                 .tbuf = buf,
 468                 .tbuflen = sizeof(buf),
 469
 470                 .table_ctx = &ctx->lrw_table,
 471                 .crypt_ctx = &crypt_ctx,
 472                 .crypt_fn = encrypt_callback,
 473         };
 474         int ret;
 475
 476         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 477         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 478         serpent_fpu_end(crypt_ctx.fpu_enabled);
 479
 480         return ret;
 481 }
 482
 483 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 484                        struct scatterlist *src, unsigned int nbytes)
 485 {
 486         struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 487         be128 buf[SERPENT_PARALLEL_BLOCKS];
 488         struct crypt_priv crypt_ctx = {
 489                 .ctx = &ctx->serpent_ctx,
 490                 .fpu_enabled = false,
 491         };
 492         struct lrw_crypt_req req = {
 493                 .tbuf = buf,
 494                 .tbuflen = sizeof(buf),
 495
 496                 .table_ctx = &ctx->lrw_table,
 497                 .crypt_ctx = &crypt_ctx,
 498                 .crypt_fn = decrypt_callback,
 499         };
 500         int ret;
 501
 502         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 503         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 504         serpent_fpu_end(crypt_ctx.fpu_enabled);
 505
 506         return ret;
 507 }
 508
 509 static void lrw_exit_tfm(struct crypto_tfm *tfm)
 510 {
 511         struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
 512
 513         lrw_free_table(&ctx->lrw_table);
 514 }
 515
 516 struct serpent_xts_ctx {
 517         struct serpent_ctx tweak_ctx;
 518         struct serpent_ctx crypt_ctx;
 519 };
 520
 521 static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
 522                               unsigned int keylen)
 523 {
 524         struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm);
 525         u32 *flags = &tfm->crt_flags;
 526         int err;
 527
 528         /* key consists of keys of equal size concatenated, therefore
 529          * the length must be even
 530          */
 531         if (keylen % 2) {
 532                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 533                 return -EINVAL;
 534         }
 535
 536         /* first half of xts-key is for crypt */
 537         err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2);
 538         if (err)
 539                 return err;
 540
 541         /* second half of xts-key is for tweak */
 542         return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
 543 }
 544
 545 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 546                        struct scatterlist *src, unsigned int nbytes)
 547 {
 548         struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 549         be128 buf[SERPENT_PARALLEL_BLOCKS];
 550         struct crypt_priv crypt_ctx = {
 551                 .ctx = &ctx->crypt_ctx,
 552                 .fpu_enabled = false,
 553         };
 554         struct xts_crypt_req req = {
 555                 .tbuf = buf,
 556                 .tbuflen = sizeof(buf),
 557
 558                 .tweak_ctx = &ctx->tweak_ctx,
 559                 .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
 560                 .crypt_ctx = &crypt_ctx,
 561                 .crypt_fn = encrypt_callback,
 562         };
 563         int ret;
 564
 565         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 566         ret = xts_crypt(desc, dst, src, nbytes, &req);
 567         serpent_fpu_end(crypt_ctx.fpu_enabled);
 568
 569         return ret;
 570 }
 571
 572 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 573                        struct scatterlist *src, unsigned int nbytes)
 574 {
 575         struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 576         be128 buf[SERPENT_PARALLEL_BLOCKS];
 577         struct crypt_priv crypt_ctx = {
 578                 .ctx = &ctx->crypt_ctx,
 579                 .fpu_enabled = false,
 580         };
 581         struct xts_crypt_req req = {
 582                 .tbuf = buf,
 583                 .tbuflen = sizeof(buf),
 584
 585                 .tweak_ctx = &ctx->tweak_ctx,
 586                 .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
 587                 .crypt_ctx = &crypt_ctx,
 588                 .crypt_fn = decrypt_callback,
 589         };
 590         int ret;
 591
 592         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 593         ret = xts_crypt(desc, dst, src, nbytes, &req);
 594         serpent_fpu_end(crypt_ctx.fpu_enabled);
 595
 596         return ret;
 597 }
 598
 599 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
 600                         unsigned int key_len)
 601 {
 602         struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 603         struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
 604         int err;
 605
 606         crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 607         crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
 608                                     & CRYPTO_TFM_REQ_MASK);
 609         err = crypto_ablkcipher_setkey(child, key, key_len);
 610         crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
 611                                     & CRYPTO_TFM_RES_MASK);
 612         return err;
 613 }
 614
 615 static int __ablk_encrypt(struct ablkcipher_request *req)
 616 {
 617         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 618         struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 619         struct blkcipher_desc desc;
 620
 621         desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
 622         desc.info = req->info;
 623         desc.flags = 0;
 624
 625         return crypto_blkcipher_crt(desc.tfm)->encrypt(
 626                 &desc, req->dst, req->src, req->nbytes);
 627 }
 628
 629 static int ablk_encrypt(struct ablkcipher_request *req)
 630 {
 631         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 632         struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 633
 634         if (!irq_fpu_usable()) {
 635                 struct ablkcipher_request *cryptd_req =
 636                         ablkcipher_request_ctx(req);
 637
 638                 memcpy(cryptd_req, req, sizeof(*req));
 639                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 640
 641                 return crypto_ablkcipher_encrypt(cryptd_req);
 642         } else {
 643                 return __ablk_encrypt(req);
 644         }
 645 }
 646
 647 static int ablk_decrypt(struct ablkcipher_request *req)
 648 {
 649         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 650         struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 651
 652         if (!irq_fpu_usable()) {
 653                 struct ablkcipher_request *cryptd_req =
 654                         ablkcipher_request_ctx(req);
 655
 656                 memcpy(cryptd_req, req, sizeof(*req));
 657                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 658
 659                 return crypto_ablkcipher_decrypt(cryptd_req);
 660         } else {
 661                 struct blkcipher_desc desc;
 662
 663                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
 664                 desc.info = req->info;
 665                 desc.flags = 0;
 666
 667                 return crypto_blkcipher_crt(desc.tfm)->decrypt(
 668                         &desc, req->dst, req->src, req->nbytes);
 669         }
 670 }
 671
 672 static void ablk_exit(struct crypto_tfm *tfm)
 673 {
 674         struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm);
 675
 676         cryptd_free_ablkcipher(ctx->cryptd_tfm);
 677 }
 678
 679 static int ablk_init(struct crypto_tfm *tfm)
 680 {
 681         struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm);
 682         struct cryptd_ablkcipher *cryptd_tfm;
 683         char drv_name[CRYPTO_MAX_ALG_NAME];
 684
 685         snprintf(drv_name, sizeof(drv_name), "__driver-%s",
 686                                         crypto_tfm_alg_driver_name(tfm));
 687
 688         cryptd_tfm = cryptd_alloc_ablkcipher(drv_name, 0, 0);
 689         if (IS_ERR(cryptd_tfm))
 690                 return PTR_ERR(cryptd_tfm);
 691
 692         ctx->cryptd_tfm = cryptd_tfm;
 693         tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
 694                 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
 695
 696         return 0;
 697 }
 698
 699 static struct crypto_alg serpent_algs[10] = { {
 700         .cra_name               = "__ecb-serpent-sse2",
 701         .cra_driver_name        = "__driver-ecb-serpent-sse2",
 702         .cra_priority           = 0,
 703         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 704         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 705         .cra_ctxsize            = sizeof(struct serpent_ctx),
 706         .cra_alignmask          = 0,
 707         .cra_type               = &crypto_blkcipher_type,
 708         .cra_module             = THIS_MODULE,
 709         .cra_list               = LIST_HEAD_INIT(serpent_algs[0].cra_list),
 710         .cra_u = {
 711                 .blkcipher = {
 712                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 713                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 714                         .setkey         = serpent_setkey,
 715                         .encrypt        = ecb_encrypt,
 716                         .decrypt        = ecb_decrypt,
 717                 },
 718         },
 719 }, {
 720         .cra_name               = "__cbc-serpent-sse2",
 721         .cra_driver_name        = "__driver-cbc-serpent-sse2",
 722         .cra_priority           = 0,
 723         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 724         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 725         .cra_ctxsize            = sizeof(struct serpent_ctx),
 726         .cra_alignmask          = 0,
 727         .cra_type               = &crypto_blkcipher_type,
 728         .cra_module             = THIS_MODULE,
 729         .cra_list               = LIST_HEAD_INIT(serpent_algs[1].cra_list),
 730         .cra_u = {
 731                 .blkcipher = {
 732                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 733                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 734                         .setkey         = serpent_setkey,
 735                         .encrypt        = cbc_encrypt,
 736                         .decrypt        = cbc_decrypt,
 737                 },
 738         },
 739 }, {
 740         .cra_name               = "__ctr-serpent-sse2",
 741         .cra_driver_name        = "__driver-ctr-serpent-sse2",
 742         .cra_priority           = 0,
 743         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 744         .cra_blocksize          = 1,
 745         .cra_ctxsize            = sizeof(struct serpent_ctx),
 746         .cra_alignmask          = 0,
 747         .cra_type               = &crypto_blkcipher_type,
 748         .cra_module             = THIS_MODULE,
 749         .cra_list               = LIST_HEAD_INIT(serpent_algs[2].cra_list),
 750         .cra_u = {
 751                 .blkcipher = {
 752                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 753                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 754                         .ivsize         = SERPENT_BLOCK_SIZE,
 755                         .setkey         = serpent_setkey,
 756                         .encrypt        = ctr_crypt,
 757                         .decrypt        = ctr_crypt,
 758                 },
 759         },
 760 }, {
 761         .cra_name               = "__lrw-serpent-sse2",
 762         .cra_driver_name        = "__driver-lrw-serpent-sse2",
 763         .cra_priority           = 0,
 764         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 765         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 766         .cra_ctxsize            = sizeof(struct serpent_lrw_ctx),
 767         .cra_alignmask          = 0,
 768         .cra_type               = &crypto_blkcipher_type,
 769         .cra_module             = THIS_MODULE,
 770         .cra_list               = LIST_HEAD_INIT(serpent_algs[3].cra_list),
 771         .cra_exit               = lrw_exit_tfm,
 772         .cra_u = {
 773                 .blkcipher = {
 774                         .min_keysize    = SERPENT_MIN_KEY_SIZE +
 775                                           SERPENT_BLOCK_SIZE,
 776                         .max_keysize    = SERPENT_MAX_KEY_SIZE +
 777                                           SERPENT_BLOCK_SIZE,
 778                         .ivsize         = SERPENT_BLOCK_SIZE,
 779                         .setkey         = lrw_serpent_setkey,
 780                         .encrypt        = lrw_encrypt,
 781                         .decrypt        = lrw_decrypt,
 782                 },
 783         },
 784 }, {
 785         .cra_name               = "__xts-serpent-sse2",
 786         .cra_driver_name        = "__driver-xts-serpent-sse2",
 787         .cra_priority           = 0,
 788         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 789         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 790         .cra_ctxsize            = sizeof(struct serpent_xts_ctx),
 791         .cra_alignmask          = 0,
 792         .cra_type               = &crypto_blkcipher_type,
 793         .cra_module             = THIS_MODULE,
 794         .cra_list               = LIST_HEAD_INIT(serpent_algs[4].cra_list),
 795         .cra_u = {
 796                 .blkcipher = {
 797                         .min_keysize    = SERPENT_MIN_KEY_SIZE * 2,
 798                         .max_keysize    = SERPENT_MAX_KEY_SIZE * 2,
 799                         .ivsize         = SERPENT_BLOCK_SIZE,
 800                         .setkey         = xts_serpent_setkey,
 801                         .encrypt        = xts_encrypt,
 802                         .decrypt        = xts_decrypt,
 803                 },
 804         },
 805 }, {
 806         .cra_name               = "ecb(serpent)",
 807         .cra_driver_name        = "ecb-serpent-sse2",
 808         .cra_priority           = 400,
 809         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 810         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 811         .cra_ctxsize            = sizeof(struct async_serpent_ctx),
 812         .cra_alignmask          = 0,
 813         .cra_type               = &crypto_ablkcipher_type,
 814         .cra_module             = THIS_MODULE,
 815         .cra_list               = LIST_HEAD_INIT(serpent_algs[5].cra_list),
 816         .cra_init               = ablk_init,
 817         .cra_exit               = ablk_exit,
 818         .cra_u = {
 819                 .ablkcipher = {
 820                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 821                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 822                         .setkey         = ablk_set_key,
 823                         .encrypt        = ablk_encrypt,
 824                         .decrypt        = ablk_decrypt,
 825                 },
 826         },
 827 }, {
 828         .cra_name               = "cbc(serpent)",
 829         .cra_driver_name        = "cbc-serpent-sse2",
 830         .cra_priority           = 400,
 831         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 832         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 833         .cra_ctxsize            = sizeof(struct async_serpent_ctx),
 834         .cra_alignmask          = 0,
 835         .cra_type               = &crypto_ablkcipher_type,
 836         .cra_module             = THIS_MODULE,
 837         .cra_list               = LIST_HEAD_INIT(serpent_algs[6].cra_list),
 838         .cra_init               = ablk_init,
 839         .cra_exit               = ablk_exit,
 840         .cra_u = {
 841                 .ablkcipher = {
 842                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 843                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 844                         .ivsize         = SERPENT_BLOCK_SIZE,
 845                         .setkey         = ablk_set_key,
 846                         .encrypt        = __ablk_encrypt,
 847                         .decrypt        = ablk_decrypt,
 848                 },
 849         },
 850 }, {
 851         .cra_name               = "ctr(serpent)",
 852         .cra_driver_name        = "ctr-serpent-sse2",
 853         .cra_priority           = 400,
 854         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 855         .cra_blocksize          = 1,
 856         .cra_ctxsize            = sizeof(struct async_serpent_ctx),
 857         .cra_alignmask          = 0,
 858         .cra_type               = &crypto_ablkcipher_type,
 859         .cra_module             = THIS_MODULE,
 860         .cra_list               = LIST_HEAD_INIT(serpent_algs[7].cra_list),
 861         .cra_init               = ablk_init,
 862         .cra_exit               = ablk_exit,
 863         .cra_u = {
 864                 .ablkcipher = {
 865                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 866                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 867                         .ivsize         = SERPENT_BLOCK_SIZE,
 868                         .setkey         = ablk_set_key,
 869                         .encrypt        = ablk_encrypt,
 870                         .decrypt        = ablk_encrypt,
 871                         .geniv          = "chainiv",
 872                 },
 873         },
 874 }, {
 875         .cra_name               = "lrw(serpent)",
 876         .cra_driver_name        = "lrw-serpent-sse2",
 877         .cra_priority           = 400,
 878         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 879         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 880         .cra_ctxsize            = sizeof(struct async_serpent_ctx),
 881         .cra_alignmask          = 0,
 882         .cra_type               = &crypto_ablkcipher_type,
 883         .cra_module             = THIS_MODULE,
 884         .cra_list               = LIST_HEAD_INIT(serpent_algs[8].cra_list),
 885         .cra_init               = ablk_init,
 886         .cra_exit               = ablk_exit,
 887         .cra_u = {
 888                 .ablkcipher = {
 889                         .min_keysize    = SERPENT_MIN_KEY_SIZE +
 890                                           SERPENT_BLOCK_SIZE,
 891                         .max_keysize    = SERPENT_MAX_KEY_SIZE +
 892                                           SERPENT_BLOCK_SIZE,
 893                         .ivsize         = SERPENT_BLOCK_SIZE,
 894                         .setkey         = ablk_set_key,
 895                         .encrypt        = ablk_encrypt,
 896                         .decrypt        = ablk_decrypt,
 897                 },
 898         },
 899 }, {
 900         .cra_name               = "xts(serpent)",
 901         .cra_driver_name        = "xts-serpent-sse2",
 902         .cra_priority           = 400,
 903         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 904         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 905         .cra_ctxsize            = sizeof(struct async_serpent_ctx),
 906         .cra_alignmask          = 0,
 907         .cra_type               = &crypto_ablkcipher_type,
 908         .cra_module             = THIS_MODULE,
 909         .cra_list               = LIST_HEAD_INIT(serpent_algs[9].cra_list),
 910         .cra_init               = ablk_init,
 911         .cra_exit               = ablk_exit,
 912         .cra_u = {
 913                 .ablkcipher = {
 914                         .min_keysize    = SERPENT_MIN_KEY_SIZE * 2,
 915                         .max_keysize    = SERPENT_MAX_KEY_SIZE * 2,
 916                         .ivsize         = SERPENT_BLOCK_SIZE,
 917                         .setkey         = ablk_set_key,
 918                         .encrypt        = ablk_encrypt,
 919                         .decrypt        = ablk_decrypt,
 920                 },
 921         },
 922 } };
 923
 924 static int __init serpent_sse2_init(void)
 925 {
 926         if (!cpu_has_xmm2) {
 927                 printk(KERN_INFO "SSE2 instructions are not detected.\n");
 928                 return -ENODEV;
 929         }
 930
 931         return crypto_register_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
 932 }
 933
 934 static void __exit serpent_sse2_exit(void)
 935 {
 936         crypto_unregister_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
 937 }
 938
 939 module_init(serpent_sse2_init);
 940 module_exit(serpent_sse2_exit);
 941
 942 MODULE_DESCRIPTION("Serpent Cipher Algorithm, SSE2 optimized");
 943 MODULE_LICENSE("GPL");
 944 MODULE_ALIAS("serpent");