lib/libcrypto/engine/eng_aesni.c

   1 /* $OpenBSD: eng_aesni.c,v 1.10 2017/01/29 17:49:23 beck Exp $ */
   2 /*
   3  * Support for Intel AES-NI intruction set
   4  *   Author: Huang Ying <ying.huang@intel.com>
   5  *
   6  * Intel AES-NI is a new set of Single Instruction Multiple Data
   7  * (SIMD) instructions that are going to be introduced in the next
   8  * generation of Intel processor, as of 2009. These instructions
   9  * enable fast and secure data encryption and decryption, using the
  10  * Advanced Encryption Standard (AES), defined by FIPS Publication
  11  * number 197.  The architecture introduces six instructions that
  12  * offer full hardware support for AES. Four of them support high
  13  * performance data encryption and decryption, and the other two
  14  * instructions support the AES key expansion procedure.
  15  *
  16  * The white paper can be downloaded from:
  17  *   http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf
  18  *
  19  * This file is based on engines/e_padlock.c
  20  */
  21
  22 /* ====================================================================
  23  * Copyright (c) 1999-2001 The OpenSSL Project.  All rights reserved.
  24  *
  25  * Redistribution and use in source and binary forms, with or without
  26  * modification, are permitted provided that the following conditions
  27  * are met:
  28  *
  29  * 1. Redistributions of source code must retain the above copyright
  30  *    notice, this list of conditions and the following disclaimer.
  31  *
  32  * 2. Redistributions in binary form must reproduce the above copyright
  33  *    notice, this list of conditions and the following disclaimer in
  34  *    the documentation and/or other materials provided with the
  35  *    distribution.
  36  *
  37  * 3. All advertising materials mentioning features or use of this
  38  *    software must display the following acknowledgment:
  39  *    "This product includes software developed by the OpenSSL Project
  40  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
  41  *
  42  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
  43  *    endorse or promote products derived from this software without
  44  *    prior written permission. For written permission, please contact
  45  *    licensing@OpenSSL.org.
  46  *
  47  * 5. Products derived from this software may not be called "OpenSSL"
  48  *    nor may "OpenSSL" appear in their names without prior written
  49  *    permission of the OpenSSL Project.
  50  *
  51  * 6. Redistributions of any form whatsoever must retain the following
  52  *    acknowledgment:
  53  *    "This product includes software developed by the OpenSSL Project
  54  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
  55  *
  56  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
  57  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  58  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  59  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
  60  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  61  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  62  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  63  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  64  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  65  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  66  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
  67  * OF THE POSSIBILITY OF SUCH DAMAGE.
  68  * ====================================================================
  69  *
  70  * This product includes cryptographic software written by Eric Young
  71  * (eay@cryptsoft.com).  This product includes software written by Tim
  72  * Hudson (tjh@cryptsoft.com).
  73  *
  74  */
  75
  76 #include <stdio.h>
  77
  78 #include <openssl/opensslconf.h>
  79
  80 #if !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_AES_NI) && !defined(OPENSSL_NO_AES)
  81
  82 #include <openssl/aes.h>
  83 #include <openssl/dso.h>
  84 #include <openssl/engine.h>
  85 #include <openssl/err.h>
  86 #include <openssl/evp.h>
  87
  88 /* AES-NI is available *ONLY* on some x86 CPUs.  Not only that it
  89    doesn't exist elsewhere, but it even can't be compiled on other
  90    platforms! */
  91 #undef COMPILE_HW_AESNI
  92 #if (defined(__x86_64) || defined(__x86_64__) || \
  93      defined(_M_AMD64) || defined(_M_X64) || \
  94      defined(OPENSSL_IA32_SSE2)) && !defined(OPENSSL_NO_ASM) && !defined(__i386__)
  95 #define COMPILE_HW_AESNI
  96 #include "x86_arch.h"
  97 #endif
  98 static ENGINE *ENGINE_aesni(void);
  99
 100 void ENGINE_load_aesni(void)
 101 {
 102 /* On non-x86 CPUs it just returns. */
 103 #ifdef COMPILE_HW_AESNI
 104         ENGINE *toadd = ENGINE_aesni();
 105         if (!toadd)
 106                 return;
 107         ENGINE_add (toadd);
 108         ENGINE_register_complete (toadd);
 109         ENGINE_free (toadd);
 110         ERR_clear_error ();
 111 #endif
 112 }
 113
 114 #ifdef COMPILE_HW_AESNI
 115 int aesni_set_encrypt_key(const unsigned char *userKey, int bits,
 116     AES_KEY *key);
 117 int aesni_set_decrypt_key(const unsigned char *userKey, int bits,
 118     AES_KEY *key);
 119
 120 void aesni_encrypt(const unsigned char *in, unsigned char *out,
 121     const AES_KEY *key);
 122 void aesni_decrypt(const unsigned char *in, unsigned char *out,
 123     const AES_KEY *key);
 124
 125 void aesni_ecb_encrypt(const unsigned char *in, unsigned char *out,
 126     size_t length, const AES_KEY *key, int enc);
 127 void aesni_cbc_encrypt(const unsigned char *in, unsigned char *out,
 128     size_t length, const AES_KEY *key, unsigned char *ivec, int enc);
 129
 130 /* Function for ENGINE detection and control */
 131 static int aesni_init(ENGINE *e);
 132
 133 /* Cipher Stuff */
 134 static int aesni_ciphers(ENGINE *e, const EVP_CIPHER **cipher,
 135     const int **nids, int nid);
 136
 137 #define AESNI_MIN_ALIGN 16
 138 #define AESNI_ALIGN(x) \
 139         ((void *)(((unsigned long)(x)+AESNI_MIN_ALIGN-1)&~(AESNI_MIN_ALIGN-1)))
 140
 141 /* Engine names */
 142 static const char   aesni_id[] = "aesni",
 143     aesni_name[] = "Intel AES-NI engine",
 144     no_aesni_name[] = "Intel AES-NI engine (no-aesni)";
 145
 146
 147 /* The input and output encrypted as though 128bit cfb mode is being
 148  * used.  The extra state information to record how much of the
 149  * 128bit block we have used is contained in *num;
 150  */
 151 static void
 152 aesni_cfb128_encrypt(const unsigned char *in, unsigned char *out,
 153     unsigned int len, const void *key, unsigned char ivec[16], int *num,
 154     int enc)
 155 {
 156         unsigned int n;
 157         size_t l = 0;
 158
 159         n = *num;
 160
 161         if (enc) {
 162 #if !defined(OPENSSL_SMALL_FOOTPRINT)
 163                         if (16%sizeof(size_t) == 0) do {        /* always true actually */
 164                         while (n && len) {
 165                                 *(out++) = ivec[n] ^= *(in++);
 166                                 --len;
 167                                 n = (n + 1) % 16;
 168                         }
 169                         while (len >= 16) {
 170                                 aesni_encrypt(ivec, ivec, key);
 171                                 for (n = 0; n < 16; n += sizeof(size_t)) {
 172                                         *(size_t*)(out + n) =
 173                                             *(size_t*)(ivec + n) ^= *(size_t*)(in + n);
 174                                 }
 175                                 len -= 16;
 176                                 out += 16;
 177                                 in += 16;
 178                         }
 179                         n = 0;
 180                         if (len) {
 181                                 aesni_encrypt(ivec, ivec, key);
 182                                 while (len--) {
 183                                         out[n] = ivec[n] ^= in[n];
 184                                         ++n;
 185                                 }
 186                         }
 187                         *num = n;
 188                         return;
 189                 } while (0);
 190                 /* the rest would be commonly eliminated by x86* compiler */
 191 #endif
 192                 while (l < len) {
 193                         if (n == 0) {
 194                                 aesni_encrypt(ivec, ivec, key);
 195                         }
 196                         out[l] = ivec[n] ^= in[l];
 197                         ++l;
 198                         n = (n + 1) % 16;
 199                 }
 200                 *num = n;
 201         } else {
 202 #if !defined(OPENSSL_SMALL_FOOTPRINT)
 203                         if (16%sizeof(size_t) == 0) do {        /* always true actually */
 204                         while (n && len) {
 205                                 unsigned char c;
 206                                 *(out++) = ivec[n] ^ (c = *(in++));
 207                                 ivec[n] = c;
 208                                 --len;
 209                                 n = (n + 1) % 16;
 210                         }
 211                         while (len >= 16) {
 212                                 aesni_encrypt(ivec, ivec, key);
 213                                 for (n = 0; n < 16; n += sizeof(size_t)) {
 214                                         size_t t = *(size_t*)(in + n);
 215                                         *(size_t*)(out + n) = *(size_t*)(ivec + n) ^ t;
 216                                         *(size_t*)(ivec + n) = t;
 217                                 }
 218                                 len -= 16;
 219                                 out += 16;
 220                                 in += 16;
 221                         }
 222                         n = 0;
 223                         if (len) {
 224                                 aesni_encrypt(ivec, ivec, key);
 225                                 while (len--) {
 226                                         unsigned char c;
 227                                         out[n] = ivec[n] ^ (c = in[n]);
 228                                         ivec[n] = c;
 229                                         ++n;
 230                                 }
 231                         }
 232                         *num = n;
 233                         return;
 234                 } while (0);
 235                 /* the rest would be commonly eliminated by x86* compiler */
 236 #endif
 237                 while (l < len) {
 238                         unsigned char c;
 239                         if (n == 0) {
 240                                 aesni_encrypt(ivec, ivec, key);
 241                         }
 242                         out[l] = ivec[n] ^ (c = in[l]);
 243                         ivec[n] = c;
 244                         ++l;
 245                         n = (n + 1) % 16;
 246                 }
 247                 *num = n;
 248         }
 249 }
 250
 251 /* The input and output encrypted as though 128bit ofb mode is being
 252  * used.  The extra state information to record how much of the
 253  * 128bit block we have used is contained in *num;
 254  */
 255 static void
 256 aesni_ofb128_encrypt(const unsigned char *in, unsigned char *out,
 257     unsigned int len, const void *key, unsigned char ivec[16], int *num)
 258 {
 259         unsigned int n;
 260         size_t l = 0;
 261
 262         n = *num;
 263
 264 #if !defined(OPENSSL_SMALL_FOOTPRINT)
 265                 if (16%sizeof(size_t) == 0) do { /* always true actually */
 266                 while (n && len) {
 267                         *(out++) = *(in++) ^ ivec[n];
 268                         --len;
 269                         n = (n + 1) % 16;
 270                 }
 271                 while (len >= 16) {
 272                         aesni_encrypt(ivec, ivec, key);
 273                         for (n = 0; n < 16; n += sizeof(size_t))
 274                                 *(size_t*)(out + n) =
 275                                     *(size_t*)(in + n) ^ *(size_t*)(ivec + n);
 276                         len -= 16;
 277                         out += 16;
 278                         in += 16;
 279                 }
 280                 n = 0;
 281                 if (len) {
 282                         aesni_encrypt(ivec, ivec, key);
 283                         while (len--) {
 284                                 out[n] = in[n] ^ ivec[n];
 285                                 ++n;
 286                         }
 287                 }
 288                 *num = n;
 289                 return;
 290         } while (0);
 291         /* the rest would be commonly eliminated by x86* compiler */
 292 #endif
 293         while (l < len) {
 294                 if (n == 0) {
 295                         aesni_encrypt(ivec, ivec, key);
 296                 }
 297                 out[l] = in[l] ^ ivec[n];
 298                 ++l;
 299                 n = (n + 1) % 16;
 300         }
 301
 302         *num = n;
 303 }
 304 /* ===== Engine "management" functions ===== */
 305
 306 /* Prepare the ENGINE structure for registration */
 307 static int
 308 aesni_bind_helper(ENGINE *e)
 309 {
 310         int engage;
 311
 312         engage = (OPENSSL_cpu_caps() & CPUCAP_MASK_AESNI) != 0;
 313
 314         /* Register everything or return with an error */
 315         if (!ENGINE_set_id(e, aesni_id) ||
 316             !ENGINE_set_name(e, engage ? aesni_name : no_aesni_name) ||
 317             !ENGINE_set_init_function(e, aesni_init) ||
 318             (engage && !ENGINE_set_ciphers (e, aesni_ciphers)))
 319                 return 0;
 320
 321         /* Everything looks good */
 322         return 1;
 323 }
 324
 325 /* Constructor */
 326 static ENGINE *
 327 ENGINE_aesni(void)
 328 {
 329         ENGINE *eng = ENGINE_new();
 330
 331         if (!eng) {
 332                 return NULL;
 333         }
 334
 335         if (!aesni_bind_helper(eng)) {
 336                 ENGINE_free(eng);
 337                 return NULL;
 338         }
 339
 340         return eng;
 341 }
 342
 343 /* Check availability of the engine */
 344 static int
 345 aesni_init(ENGINE *e)
 346 {
 347         return 1;
 348 }
 349
 350 #if defined(NID_aes_128_cfb128) && ! defined (NID_aes_128_cfb)
 351 #define NID_aes_128_cfb NID_aes_128_cfb128
 352 #endif
 353
 354 #if defined(NID_aes_128_ofb128) && ! defined (NID_aes_128_ofb)
 355 #define NID_aes_128_ofb NID_aes_128_ofb128
 356 #endif
 357
 358 #if defined(NID_aes_192_cfb128) && ! defined (NID_aes_192_cfb)
 359 #define NID_aes_192_cfb NID_aes_192_cfb128
 360 #endif
 361
 362 #if defined(NID_aes_192_ofb128) && ! defined (NID_aes_192_ofb)
 363 #define NID_aes_192_ofb NID_aes_192_ofb128
 364 #endif
 365
 366 #if defined(NID_aes_256_cfb128) && ! defined (NID_aes_256_cfb)
 367 #define NID_aes_256_cfb NID_aes_256_cfb128
 368 #endif
 369
 370 #if defined(NID_aes_256_ofb128) && ! defined (NID_aes_256_ofb)
 371 #define NID_aes_256_ofb NID_aes_256_ofb128
 372 #endif
 373
 374 /* List of supported ciphers. */
 375 static int aesni_cipher_nids[] = {
 376         NID_aes_128_ecb,
 377         NID_aes_128_cbc,
 378         NID_aes_128_cfb,
 379         NID_aes_128_ofb,
 380
 381         NID_aes_192_ecb,
 382         NID_aes_192_cbc,
 383         NID_aes_192_cfb,
 384         NID_aes_192_ofb,
 385
 386         NID_aes_256_ecb,
 387         NID_aes_256_cbc,
 388         NID_aes_256_cfb,
 389         NID_aes_256_ofb,
 390 };
 391 static int aesni_cipher_nids_num =
 392     (sizeof(aesni_cipher_nids) / sizeof(aesni_cipher_nids[0]));
 393
 394 typedef struct {
 395         AES_KEY ks;
 396         unsigned int _pad1[3];
 397 } AESNI_KEY;
 398
 399 static int
 400 aesni_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *user_key,
 401     const unsigned char *iv, int enc)
 402 {
 403         int ret;
 404         AES_KEY *key = AESNI_ALIGN(ctx->cipher_data);
 405
 406         if ((ctx->cipher->flags & EVP_CIPH_MODE) == EVP_CIPH_CFB_MODE ||
 407             (ctx->cipher->flags & EVP_CIPH_MODE) == EVP_CIPH_OFB_MODE ||
 408             enc)
 409                 ret = aesni_set_encrypt_key(user_key, ctx->key_len * 8, key);
 410         else
 411                 ret = aesni_set_decrypt_key(user_key, ctx->key_len * 8, key);
 412
 413         if (ret < 0) {
 414                 EVPerror(EVP_R_AES_KEY_SETUP_FAILED);
 415                 return 0;
 416         }
 417
 418         return 1;
 419 }
 420
 421 static int
 422 aesni_cipher_ecb(EVP_CIPHER_CTX *ctx, unsigned char *out,
 423     const unsigned char *in, size_t inl)
 424 {
 425         AES_KEY *key = AESNI_ALIGN(ctx->cipher_data);
 426
 427         aesni_ecb_encrypt(in, out, inl, key, ctx->encrypt);
 428         return 1;
 429 }
 430
 431 static int
 432 aesni_cipher_cbc(EVP_CIPHER_CTX *ctx, unsigned char *out,
 433     const unsigned char *in, size_t inl)
 434 {
 435         AES_KEY *key = AESNI_ALIGN(ctx->cipher_data);
 436
 437         aesni_cbc_encrypt(in, out, inl, key, ctx->iv, ctx->encrypt);
 438         return 1;
 439 }
 440
 441 static int
 442 aesni_cipher_cfb(EVP_CIPHER_CTX *ctx, unsigned char *out,
 443     const unsigned char *in, size_t inl)
 444 {
 445         AES_KEY *key = AESNI_ALIGN(ctx->cipher_data);
 446
 447         aesni_cfb128_encrypt(in, out, inl, key, ctx->iv, &ctx->num,
 448             ctx->encrypt);
 449         return 1;
 450 }
 451
 452 static int
 453 aesni_cipher_ofb(EVP_CIPHER_CTX *ctx, unsigned char *out,
 454     const unsigned char *in, size_t inl)
 455 {
 456         AES_KEY *key = AESNI_ALIGN(ctx->cipher_data);
 457
 458         aesni_ofb128_encrypt(in, out, inl, key, ctx->iv, &ctx->num);
 459         return 1;
 460 }
 461
 462 #define AES_BLOCK_SIZE          16
 463
 464 #define EVP_CIPHER_block_size_ECB       AES_BLOCK_SIZE
 465 #define EVP_CIPHER_block_size_CBC       AES_BLOCK_SIZE
 466 #define EVP_CIPHER_block_size_OFB       1
 467 #define EVP_CIPHER_block_size_CFB       1
 468
 469 /* Declaring so many ciphers by hand would be a pain.
 470    Instead introduce a bit of preprocessor magic :-) */
 471 #define DECLARE_AES_EVP(ksize,lmode,umode)      \
 472 static const EVP_CIPHER aesni_##ksize##_##lmode = {     \
 473         NID_aes_##ksize##_##lmode,                      \
 474         EVP_CIPHER_block_size_##umode,                  \
 475         ksize / 8,                                      \
 476         AES_BLOCK_SIZE,                                 \
 477         0 | EVP_CIPH_##umode##_MODE,                    \
 478         aesni_init_key,                         \
 479         aesni_cipher_##lmode,                           \
 480         NULL,                                           \
 481         sizeof(AESNI_KEY),                              \
 482         EVP_CIPHER_set_asn1_iv,                         \
 483         EVP_CIPHER_get_asn1_iv,                         \
 484         NULL,                                           \
 485         NULL                                            \
 486 }
 487
 488 DECLARE_AES_EVP(128, ecb, ECB);
 489 DECLARE_AES_EVP(128, cbc, CBC);
 490 DECLARE_AES_EVP(128, cfb, CFB);
 491 DECLARE_AES_EVP(128, ofb, OFB);
 492
 493 DECLARE_AES_EVP(192, ecb, ECB);
 494 DECLARE_AES_EVP(192, cbc, CBC);
 495 DECLARE_AES_EVP(192, cfb, CFB);
 496 DECLARE_AES_EVP(192, ofb, OFB);
 497
 498 DECLARE_AES_EVP(256, ecb, ECB);
 499 DECLARE_AES_EVP(256, cbc, CBC);
 500 DECLARE_AES_EVP(256, cfb, CFB);
 501 DECLARE_AES_EVP(256, ofb, OFB);
 502
 503 static int
 504 aesni_ciphers(ENGINE *e, const EVP_CIPHER **cipher, const int **nids, int nid)
 505 {
 506         /* No specific cipher => return a list of supported nids ... */
 507         if (!cipher) {
 508                 *nids = aesni_cipher_nids;
 509                 return aesni_cipher_nids_num;
 510         }
 511
 512         /* ... or the requested "cipher" otherwise */
 513         switch (nid) {
 514         case NID_aes_128_ecb:
 515                 *cipher = &aesni_128_ecb;
 516                 break;
 517         case NID_aes_128_cbc:
 518                 *cipher = &aesni_128_cbc;
 519                 break;
 520         case NID_aes_128_cfb:
 521                 *cipher = &aesni_128_cfb;
 522                 break;
 523         case NID_aes_128_ofb:
 524                 *cipher = &aesni_128_ofb;
 525                 break;
 526
 527         case NID_aes_192_ecb:
 528                 *cipher = &aesni_192_ecb;
 529                 break;
 530         case NID_aes_192_cbc:
 531                 *cipher = &aesni_192_cbc;
 532                 break;
 533         case NID_aes_192_cfb:
 534                 *cipher = &aesni_192_cfb;
 535                 break;
 536         case NID_aes_192_ofb:
 537                 *cipher = &aesni_192_ofb;
 538                 break;
 539
 540         case NID_aes_256_ecb:
 541                 *cipher = &aesni_256_ecb;
 542                 break;
 543         case NID_aes_256_cbc:
 544                 *cipher = &aesni_256_cbc;
 545                 break;
 546         case NID_aes_256_cfb:
 547                 *cipher = &aesni_256_cfb;
 548                 break;
 549         case NID_aes_256_ofb:
 550                 *cipher = &aesni_256_ofb;
 551                 break;
 552
 553         default:
 554                 /* Sorry, we don't support this NID */
 555                 *cipher = NULL;
 556                 return 0;
 557         }
 558         return 1;
 559 }
 560
 561 #endif /* COMPILE_HW_AESNI */
 562 #endif /* !defined(OPENSSL_NO_HW) && !defined(OPENSSL_NO_HW_AESNI) && !defined(OPENSSL_NO_AES) */