drivers/crypto/padlock-aes.c

   1 /*
   2  * Cryptographic API.
   3  *
   4  * Support for VIA PadLock hardware crypto engine.
   5  *
   6  * Copyright (c) 2004  Michal Ludvig <michal@logix.cz>
   7  *
   8  */
   9
  10 #include <crypto/algapi.h>
  11 #include <crypto/aes.h>
  12 #include <linux/module.h>
  13 #include <linux/init.h>
  14 #include <linux/types.h>
  15 #include <linux/errno.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/kernel.h>
  18 #include <linux/percpu.h>
  19 #include <linux/smp.h>
  20 #include <asm/byteorder.h>
  21 #include <asm/i387.h>
  22 #include "padlock.h"
  23
  24 /* Control word. */
  25 struct cword {
  26         unsigned int __attribute__ ((__packed__))
  27                 rounds:4,
  28                 algo:3,
  29                 keygen:1,
  30                 interm:1,
  31                 encdec:1,
  32                 ksize:2;
  33 } __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
  34
  35 /* Whenever making any changes to the following
  36  * structure *make sure* you keep E, d_data
  37  * and cword aligned on 16 Bytes boundaries and
  38  * the Hardware can access 16 * 16 bytes of E and d_data
  39  * (only the first 15 * 16 bytes matter but the HW reads
  40  * more).
  41  */
  42 struct aes_ctx {
  43         u32 E[AES_MAX_KEYLENGTH_U32]
  44                 __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
  45         u32 d_data[AES_MAX_KEYLENGTH_U32]
  46                 __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
  47         struct {
  48                 struct cword encrypt;
  49                 struct cword decrypt;
  50         } cword;
  51         u32 *D;
  52 };
  53
  54 static DEFINE_PER_CPU(struct cword *, last_cword);
  55
  56 /* Tells whether the ACE is capable to generate
  57    the extended key for a given key_len. */
  58 static inline int
  59 aes_hw_extkey_available(uint8_t key_len)
  60 {
  61         /* TODO: We should check the actual CPU model/stepping
  62                  as it's possible that the capability will be
  63                  added in the next CPU revisions. */
  64         if (key_len == 16)
  65                 return 1;
  66         return 0;
  67 }
  68
  69 static inline struct aes_ctx *aes_ctx_common(void *ctx)
  70 {
  71         unsigned long addr = (unsigned long)ctx;
  72         unsigned long align = PADLOCK_ALIGNMENT;
  73
  74         if (align <= crypto_tfm_ctx_alignment())
  75                 align = 1;
  76         return (struct aes_ctx *)ALIGN(addr, align);
  77 }
  78
  79 static inline struct aes_ctx *aes_ctx(struct crypto_tfm *tfm)
  80 {
  81         return aes_ctx_common(crypto_tfm_ctx(tfm));
  82 }
  83
  84 static inline struct aes_ctx *blk_aes_ctx(struct crypto_blkcipher *tfm)
  85 {
  86         return aes_ctx_common(crypto_blkcipher_ctx(tfm));
  87 }
  88
  89 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
  90                        unsigned int key_len)
  91 {
  92         struct aes_ctx *ctx = aes_ctx(tfm);
  93         const __le32 *key = (const __le32 *)in_key;
  94         u32 *flags = &tfm->crt_flags;
  95         struct crypto_aes_ctx gen_aes;
  96         int cpu;
  97
  98         if (key_len % 8) {
  99                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 100                 return -EINVAL;
 101         }
 102
 103         /*
 104          * If the hardware is capable of generating the extended key
 105          * itself we must supply the plain key for both encryption
 106          * and decryption.
 107          */
 108         ctx->D = ctx->E;
 109
 110         ctx->E[0] = le32_to_cpu(key[0]);
 111         ctx->E[1] = le32_to_cpu(key[1]);
 112         ctx->E[2] = le32_to_cpu(key[2]);
 113         ctx->E[3] = le32_to_cpu(key[3]);
 114
 115         /* Prepare control words. */
 116         memset(&ctx->cword, 0, sizeof(ctx->cword));
 117
 118         ctx->cword.decrypt.encdec = 1;
 119         ctx->cword.encrypt.rounds = 10 + (key_len - 16) / 4;
 120         ctx->cword.decrypt.rounds = ctx->cword.encrypt.rounds;
 121         ctx->cword.encrypt.ksize = (key_len - 16) / 8;
 122         ctx->cword.decrypt.ksize = ctx->cword.encrypt.ksize;
 123
 124         /* Don't generate extended keys if the hardware can do it. */
 125         if (aes_hw_extkey_available(key_len))
 126                 goto ok;
 127
 128         ctx->D = ctx->d_data;
 129         ctx->cword.encrypt.keygen = 1;
 130         ctx->cword.decrypt.keygen = 1;
 131
 132         if (crypto_aes_expand_key(&gen_aes, in_key, key_len)) {
 133                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 134                 return -EINVAL;
 135         }
 136
 137         memcpy(ctx->E, gen_aes.key_enc, AES_MAX_KEYLENGTH);
 138         memcpy(ctx->D, gen_aes.key_dec, AES_MAX_KEYLENGTH);
 139
 140 ok:
 141         for_each_online_cpu(cpu)
 142                 if (&ctx->cword.encrypt == per_cpu(last_cword, cpu) ||
 143                     &ctx->cword.decrypt == per_cpu(last_cword, cpu))
 144                         per_cpu(last_cword, cpu) = NULL;
 145
 146         return 0;
 147 }
 148
 149 /* ====== Encryption/decryption routines ====== */
 150
 151 /* These are the real call to PadLock. */
 152 static inline void padlock_reset_key(struct cword *cword)
 153 {
 154         int cpu = raw_smp_processor_id();
 155
 156         if (cword != per_cpu(last_cword, cpu))
 157                 asm volatile ("pushfl; popfl");
 158 }
 159
 160 static inline void padlock_store_cword(struct cword *cword)
 161 {
 162         per_cpu(last_cword, raw_smp_processor_id()) = cword;
 163 }
 164
 165 /*
 166  * While the padlock instructions don't use FP/SSE registers, they
 167  * generate a spurious DNA fault when cr0.ts is '1'. These instructions
 168  * should be used only inside the irq_ts_save/restore() context
 169  */
 170
 171 static inline void padlock_xcrypt(const u8 *input, u8 *output, void *key,
 172                                   struct cword *control_word)
 173 {
 174         asm volatile (".byte 0xf3,0x0f,0xa7,0xc8"       /* rep xcryptecb */
 175                       : "+S"(input), "+D"(output)
 176                       : "d"(control_word), "b"(key), "c"(1));
 177 }
 178
 179 static void aes_crypt_copy(const u8 *in, u8 *out, u32 *key, struct cword *cword)
 180 {
 181         u8 buf[AES_BLOCK_SIZE * 2 + PADLOCK_ALIGNMENT - 1];
 182         u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
 183
 184         memcpy(tmp, in, AES_BLOCK_SIZE);
 185         padlock_xcrypt(tmp, out, key, cword);
 186 }
 187
 188 static inline void aes_crypt(const u8 *in, u8 *out, u32 *key,
 189                              struct cword *cword)
 190 {
 191         /* padlock_xcrypt requires at least two blocks of data. */
 192         if (unlikely(!(((unsigned long)in ^ (PAGE_SIZE - AES_BLOCK_SIZE)) &
 193                        (PAGE_SIZE - 1)))) {
 194                 aes_crypt_copy(in, out, key, cword);
 195                 return;
 196         }
 197
 198         padlock_xcrypt(in, out, key, cword);
 199 }
 200
 201 static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key,
 202                                       void *control_word, u32 count)
 203 {
 204         if (count == 1) {
 205                 aes_crypt(input, output, key, control_word);
 206                 return;
 207         }
 208
 209         asm volatile ("test $1, %%cl;"
 210                       "je 1f;"
 211                       "lea -1(%%ecx), %%eax;"
 212                       "mov $1, %%ecx;"
 213                       ".byte 0xf3,0x0f,0xa7,0xc8;"      /* rep xcryptecb */
 214                       "mov %%eax, %%ecx;"
 215                       "1:"
 216                       ".byte 0xf3,0x0f,0xa7,0xc8"       /* rep xcryptecb */
 217                       : "+S"(input), "+D"(output)
 218                       : "d"(control_word), "b"(key), "c"(count)
 219                       : "ax");
 220 }
 221
 222 static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key,
 223                                      u8 *iv, void *control_word, u32 count)
 224 {
 225         /* rep xcryptcbc */
 226         asm volatile (".byte 0xf3,0x0f,0xa7,0xd0"
 227                       : "+S" (input), "+D" (output), "+a" (iv)
 228                       : "d" (control_word), "b" (key), "c" (count));
 229         return iv;
 230 }
 231
 232 static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
 233 {
 234         struct aes_ctx *ctx = aes_ctx(tfm);
 235         int ts_state;
 236
 237         padlock_reset_key(&ctx->cword.encrypt);
 238         ts_state = irq_ts_save();
 239         aes_crypt(in, out, ctx->E, &ctx->cword.encrypt);
 240         irq_ts_restore(ts_state);
 241         padlock_store_cword(&ctx->cword.encrypt);
 242 }
 243
 244 static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
 245 {
 246         struct aes_ctx *ctx = aes_ctx(tfm);
 247         int ts_state;
 248
 249         padlock_reset_key(&ctx->cword.encrypt);
 250         ts_state = irq_ts_save();
 251         aes_crypt(in, out, ctx->D, &ctx->cword.decrypt);
 252         irq_ts_restore(ts_state);
 253         padlock_store_cword(&ctx->cword.encrypt);
 254 }
 255
 256 static struct crypto_alg aes_alg = {
 257         .cra_name               =       "aes",
 258         .cra_driver_name        =       "aes-padlock",
 259         .cra_priority           =       PADLOCK_CRA_PRIORITY,
 260         .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER,
 261         .cra_blocksize          =       AES_BLOCK_SIZE,
 262         .cra_ctxsize            =       sizeof(struct aes_ctx),
 263         .cra_alignmask          =       PADLOCK_ALIGNMENT - 1,
 264         .cra_module             =       THIS_MODULE,
 265         .cra_list               =       LIST_HEAD_INIT(aes_alg.cra_list),
 266         .cra_u                  =       {
 267                 .cipher = {
 268                         .cia_min_keysize        =       AES_MIN_KEY_SIZE,
 269                         .cia_max_keysize        =       AES_MAX_KEY_SIZE,
 270                         .cia_setkey             =       aes_set_key,
 271                         .cia_encrypt            =       aes_encrypt,
 272                         .cia_decrypt            =       aes_decrypt,
 273                 }
 274         }
 275 };
 276
 277 static int ecb_aes_encrypt(struct blkcipher_desc *desc,
 278                            struct scatterlist *dst, struct scatterlist *src,
 279                            unsigned int nbytes)
 280 {
 281         struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
 282         struct blkcipher_walk walk;
 283         int err;
 284         int ts_state;
 285
 286         padlock_reset_key(&ctx->cword.encrypt);
 287
 288         blkcipher_walk_init(&walk, dst, src, nbytes);
 289         err = blkcipher_walk_virt(desc, &walk);
 290
 291         ts_state = irq_ts_save();
 292         while ((nbytes = walk.nbytes)) {
 293                 padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
 294                                    ctx->E, &ctx->cword.encrypt,
 295                                    nbytes / AES_BLOCK_SIZE);
 296                 nbytes &= AES_BLOCK_SIZE - 1;
 297                 err = blkcipher_walk_done(desc, &walk, nbytes);
 298         }
 299         irq_ts_restore(ts_state);
 300
 301         padlock_store_cword(&ctx->cword.encrypt);
 302
 303         return err;
 304 }
 305
 306 static int ecb_aes_decrypt(struct blkcipher_desc *desc,
 307                            struct scatterlist *dst, struct scatterlist *src,
 308                            unsigned int nbytes)
 309 {
 310         struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
 311         struct blkcipher_walk walk;
 312         int err;
 313         int ts_state;
 314
 315         padlock_reset_key(&ctx->cword.decrypt);
 316
 317         blkcipher_walk_init(&walk, dst, src, nbytes);
 318         err = blkcipher_walk_virt(desc, &walk);
 319
 320         ts_state = irq_ts_save();
 321         while ((nbytes = walk.nbytes)) {
 322                 padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
 323                                    ctx->D, &ctx->cword.decrypt,
 324                                    nbytes / AES_BLOCK_SIZE);
 325                 nbytes &= AES_BLOCK_SIZE - 1;
 326                 err = blkcipher_walk_done(desc, &walk, nbytes);
 327         }
 328         irq_ts_restore(ts_state);
 329
 330         padlock_store_cword(&ctx->cword.encrypt);
 331
 332         return err;
 333 }
 334
 335 static struct crypto_alg ecb_aes_alg = {
 336         .cra_name               =       "ecb(aes)",
 337         .cra_driver_name        =       "ecb-aes-padlock",
 338         .cra_priority           =       PADLOCK_COMPOSITE_PRIORITY,
 339         .cra_flags              =       CRYPTO_ALG_TYPE_BLKCIPHER,
 340         .cra_blocksize          =       AES_BLOCK_SIZE,
 341         .cra_ctxsize            =       sizeof(struct aes_ctx),
 342         .cra_alignmask          =       PADLOCK_ALIGNMENT - 1,
 343         .cra_type               =       &crypto_blkcipher_type,
 344         .cra_module             =       THIS_MODULE,
 345         .cra_list               =       LIST_HEAD_INIT(ecb_aes_alg.cra_list),
 346         .cra_u                  =       {
 347                 .blkcipher = {
 348                         .min_keysize            =       AES_MIN_KEY_SIZE,
 349                         .max_keysize            =       AES_MAX_KEY_SIZE,
 350                         .setkey                 =       aes_set_key,
 351                         .encrypt                =       ecb_aes_encrypt,
 352                         .decrypt                =       ecb_aes_decrypt,
 353                 }
 354         }
 355 };
 356
 357 static int cbc_aes_encrypt(struct blkcipher_desc *desc,
 358                            struct scatterlist *dst, struct scatterlist *src,
 359                            unsigned int nbytes)
 360 {
 361         struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
 362         struct blkcipher_walk walk;
 363         int err;
 364         int ts_state;
 365
 366         padlock_reset_key(&ctx->cword.encrypt);
 367
 368         blkcipher_walk_init(&walk, dst, src, nbytes);
 369         err = blkcipher_walk_virt(desc, &walk);
 370
 371         ts_state = irq_ts_save();
 372         while ((nbytes = walk.nbytes)) {
 373                 u8 *iv = padlock_xcrypt_cbc(walk.src.virt.addr,
 374                                             walk.dst.virt.addr, ctx->E,
 375                                             walk.iv, &ctx->cword.encrypt,
 376                                             nbytes / AES_BLOCK_SIZE);
 377                 memcpy(walk.iv, iv, AES_BLOCK_SIZE);
 378                 nbytes &= AES_BLOCK_SIZE - 1;
 379                 err = blkcipher_walk_done(desc, &walk, nbytes);
 380         }
 381         irq_ts_restore(ts_state);
 382
 383         padlock_store_cword(&ctx->cword.decrypt);
 384
 385         return err;
 386 }
 387
 388 static int cbc_aes_decrypt(struct blkcipher_desc *desc,
 389                            struct scatterlist *dst, struct scatterlist *src,
 390                            unsigned int nbytes)
 391 {
 392         struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
 393         struct blkcipher_walk walk;
 394         int err;
 395         int ts_state;
 396
 397         padlock_reset_key(&ctx->cword.encrypt);
 398
 399         blkcipher_walk_init(&walk, dst, src, nbytes);
 400         err = blkcipher_walk_virt(desc, &walk);
 401
 402         ts_state = irq_ts_save();
 403         while ((nbytes = walk.nbytes)) {
 404                 padlock_xcrypt_cbc(walk.src.virt.addr, walk.dst.virt.addr,
 405                                    ctx->D, walk.iv, &ctx->cword.decrypt,
 406                                    nbytes / AES_BLOCK_SIZE);
 407                 nbytes &= AES_BLOCK_SIZE - 1;
 408                 err = blkcipher_walk_done(desc, &walk, nbytes);
 409         }
 410
 411         irq_ts_restore(ts_state);
 412
 413         padlock_store_cword(&ctx->cword.encrypt);
 414
 415         return err;
 416 }
 417
 418 static struct crypto_alg cbc_aes_alg = {
 419         .cra_name               =       "cbc(aes)",
 420         .cra_driver_name        =       "cbc-aes-padlock",
 421         .cra_priority           =       PADLOCK_COMPOSITE_PRIORITY,
 422         .cra_flags              =       CRYPTO_ALG_TYPE_BLKCIPHER,
 423         .cra_blocksize          =       AES_BLOCK_SIZE,
 424         .cra_ctxsize            =       sizeof(struct aes_ctx),
 425         .cra_alignmask          =       PADLOCK_ALIGNMENT - 1,
 426         .cra_type               =       &crypto_blkcipher_type,
 427         .cra_module             =       THIS_MODULE,
 428         .cra_list               =       LIST_HEAD_INIT(cbc_aes_alg.cra_list),
 429         .cra_u                  =       {
 430                 .blkcipher = {
 431                         .min_keysize            =       AES_MIN_KEY_SIZE,
 432                         .max_keysize            =       AES_MAX_KEY_SIZE,
 433                         .ivsize                 =       AES_BLOCK_SIZE,
 434                         .setkey                 =       aes_set_key,
 435                         .encrypt                =       cbc_aes_encrypt,
 436                         .decrypt                =       cbc_aes_decrypt,
 437                 }
 438         }
 439 };
 440
 441 static int __init padlock_init(void)
 442 {
 443         int ret;
 444
 445         if (!cpu_has_xcrypt) {
 446                 printk(KERN_NOTICE PFX "VIA PadLock not detected.\n");
 447                 return -ENODEV;
 448         }
 449
 450         if (!cpu_has_xcrypt_enabled) {
 451                 printk(KERN_NOTICE PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
 452                 return -ENODEV;
 453         }
 454
 455         if ((ret = crypto_register_alg(&aes_alg)))
 456                 goto aes_err;
 457
 458         if ((ret = crypto_register_alg(&ecb_aes_alg)))
 459                 goto ecb_aes_err;
 460
 461         if ((ret = crypto_register_alg(&cbc_aes_alg)))
 462                 goto cbc_aes_err;
 463
 464         printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
 465
 466 out:
 467         return ret;
 468
 469 cbc_aes_err:
 470         crypto_unregister_alg(&ecb_aes_alg);
 471 ecb_aes_err:
 472         crypto_unregister_alg(&aes_alg);
 473 aes_err:
 474         printk(KERN_ERR PFX "VIA PadLock AES initialization failed.\n");
 475         goto out;
 476 }
 477
 478 static void __exit padlock_fini(void)
 479 {
 480         crypto_unregister_alg(&cbc_aes_alg);
 481         crypto_unregister_alg(&ecb_aes_alg);
 482         crypto_unregister_alg(&aes_alg);
 483 }
 484
 485 module_init(padlock_init);
 486 module_exit(padlock_fini);
 487
 488 MODULE_DESCRIPTION("VIA PadLock AES algorithm support");
 489 MODULE_LICENSE("GPL");
 490 MODULE_AUTHOR("Michal Ludvig");
 491
 492 MODULE_ALIAS("aes-all");