src/common/aes.c

   1 /* Copyright (c) 2001 Matej Pfajfar.
   2  * Copyright (c) 2001-2004, Roger Dingledine.
   3  * Copyright (c) 2004-2007, Roger Dingledine, Nick Mathewson. */
   4 /* See LICENSE for licensing information */
   5 /* $Id$ */
   6 const char aes_c_id[] = "$Id$";
   7
   8 /**
   9  * \file aes.c
  10  * \brief Implements the AES cipher (with 128-bit keys and blocks),
  11  * and a counter-mode stream cipher on top of AES.  This code is
  12  * taken from the main Rijndael distribution.  (We include this
  13  * because many people are running older versions of OpenSSL without
  14  * AES support.)
  15  **/
  16
  17 #include "orconfig.h"
  18 #include <openssl/opensslv.h>
  19 #include <assert.h>
  20 #include <stdlib.h>
  21 #include <string.h>
  22 #include "compat.h"
  23 #include "aes.h"
  24 #include "util.h"
  25 #include "log.h"
  26
  27 /* We have 3 strategies for getting AES: Via OpenSSL's AES_encrypt function,
  28  * via OpenSSL's EVP_EncryptUpdate function, or via the built-in AES
  29  * implementation below. */
  30 #undef USE_OPENSSL_AES
  31 #undef USE_OPENSSL_EVP
  32 #undef USE_BUILTIN_AES
  33
  34 /* Figure out our CPU type.  We use this to pick an AES implementation.
  35  * Macros are as listed at http://predef.sourceforge.net/prearch.html
  36  */
  37 #if (defined(i386) || defined(__i386__) || defined(__i386) || defined(_X86_) \
  38      || defined(_M_IX86) || defined(__THW_INTEL__) || defined(__I86__))
  39 # define CPU_IS_X86
  40 #elif (defined(__amd64__) || defined(__amd64) || \
  41        defined(__x86_64__) || defined(__x86_64) || \
  42        defined(_M_X64))
  43 # define CPU_IS_X86_64
  44 #elif (defined(__ia64__) || defined(__ia64) || defined(_IA64) || \
  45        defined(_M_IA64))
  46 # define CPU_IS_IA64
  47 #elif (defined(__sparc__) || defined(__sparc))
  48 # define CPU_IS_SPARC
  49 #elif (defined(__arm__) || defined (__TARGET_ARCH_ARM))
  50 # define CPU_IS_ARM
  51 #endif
  52
  53 /* Here we pick which to use, if none is force-defined.  See
  54  *      http://archives.seul.org/or/dev/Feb-2007/msg00045.html
  55  * for a summary of the most recent benchmarking results that led to this
  56  * nutty decision tree.
  57 */
  58 #if (!defined(USE_BUILTIN_AES) &&               \
  59      !defined(USE_OPENSSL_AES) &&               \
  60      !defined(USE_OPENSSL_EVP))
  61
  62 /* OpenSSL 0.9.7 was the first to support AES.  It was slower than our
  63  *    builtin implementation.
  64  * OpenSSL 0.9.8 added assembly implementations for i386 and ia64.
  65  *    Either the i386 stuff isn't used for x86-64, or it isn't faster.
  66  * OpenSSL 0.9.9 (not yet out) has added assembly implementations for
  67  *    x86_64 (aka amd64), sparc9, and arm
  68  *
  69  * Note: the "f" at the end of openssl version numbers below means
  70  * "release". */
  71 # if defined(CPU_IS_X86) || defined(CPU_IS_IA64)
  72 #  if OPENSSL_VERSION_NUMBER >= 0x0090800fL
  73 #   define USE_OPENSSL_AES
  74 #  endif
  75 # endif
  76
  77 # if defined(CPU_IS_X86_64) || defined(CPU_IS_ARM) || defined(CPU_IS_SPARC)
  78 #  if OPENSSL_VERSION_NUMBER >= 0x0090900fL
  79 #   define USE_OPENSSL_AES
  80 #  endif
  81 # endif
  82
  83 /* Otherwise, use the builtin implementation below. */
  84 # ifndef USE_OPENSSL_AES
  85 #  define USE_BUILTIN_AES
  86 # endif
  87 #endif /* endif need to pick a method */
  88
  89 /* Include OpenSSL headers as needed. */
  90 #ifdef USE_OPENSSL_AES
  91 # include <openssl/aes.h>
  92 #endif
  93 #ifdef USE_OPENSSL_EVP
  94 # include <openssl/evp.h>
  95 #endif
  96
  97 /* Figure out which AES optimizations to use. */
  98 #ifdef USE_BUILTIN_AES
  99 # define USE_RIJNDAEL_COUNTER_OPTIMIZATION
 100 # if 0 && (defined(__powerpc__) || defined(__powerpc64__))
 101 /* XXXX do more experimentation before concluding this is actually
 102  * a good idea. */
 103 #  define FULL_UNROLL
 104 # endif
 105 #endif
 106
 107 /*======================================================================*/
 108 /* From rijndael-alg-fst.h */
 109
 110 typedef uint64_t u64;
 111 typedef uint32_t u32;
 112 typedef uint8_t u8;
 113
 114 #ifdef USE_BUILTIN_AES
 115 #define MAXNR   14
 116
 117 static int rijndaelKeySetupEnc(u32 rk[/*4*(Nr + 1)*/],
 118                                const u8 cipherKey[], int keyBits);
 119 #ifdef USE_RIJNDAEL_COUNTER_OPTIMIZATION
 120 static void rijndaelEncrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr,
 121                             u32 ctr1, u32 ctr0, u8 ct[16]);
 122 #else
 123 static void rijndaelEncrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr,
 124                             const u8 pt[16], u8 ct[16]);
 125 #endif
 126 #endif
 127
 128 /*======================================================================*/
 129 /* Interface to AES code, and counter implementation */
 130
 131 struct aes_cnt_cipher {
 132 #if defined(USE_OPENSSL_EVP)
 133   EVP_CIPHER_CTX key;
 134 #elif defined(USE_OPENSSL_AES)
 135   AES_KEY key;
 136 #else
 137   u32 rk[4*(MAXNR+1)];
 138   int nr;
 139 #endif
 140   u32 counter1;
 141   u32 counter0;
 142   u8 buf[16];
 143   u8 pos;
 144 };
 145
 146 /**
 147  * Helper function: set <b>cipher</b>'s internal buffer to the encrypted
 148  * value of the current counter.
 149  */
 150 static INLINE void
 151 _aes_fill_buf(aes_cnt_cipher_t *cipher)
 152 {
 153   /* We don't currently use OpenSSL's counter mode implementation because:
 154    *  1) some versions have known bugs
 155    *  2) its attitude towards IVs is not our own
 156    *  3) changing the counter position was not trivial, last time I looked.
 157    * None of these issues are insurmountable in principle.
 158    */
 159 #if defined(USE_BUILTIN_AES) && defined(USE_RIJNDAEL_COUNTER_OPTIMIZATION)
 160   rijndaelEncrypt(cipher->rk, cipher->nr,
 161                   cipher->counter1, cipher->counter0, cipher->buf);
 162 #else
 163   u32 counter0 = cipher->counter0;
 164   u32 counter1 = cipher->counter1;
 165   u8 buf[16];
 166   memset(buf, 0, 8);
 167   buf[15] = (counter0 >> 0)  & 0xff;
 168   buf[14] = (counter0 >> 8)  & 0xff;
 169   buf[13] = (counter0 >> 16) & 0xff;
 170   buf[12] = (counter0 >> 24) & 0xff;
 171   buf[11] = (counter1 >> 0)  & 0xff;
 172   buf[10] = (counter1 >> 8)  & 0xff;
 173   buf[ 9] = (counter1 >> 16) & 0xff;
 174   buf[ 8] = (counter1 >> 24) & 0xff;
 175
 176 #if defined(USE_OPENSSL_EVP)
 177   {
 178     int outl=16, inl=16;
 179     EVP_EncryptUpdate(&cipher->key, cipher->buf, &outl, buf, inl);
 180   }
 181 #elif defined(USE_OPENSSL_AES)
 182   AES_encrypt(buf, cipher->buf, &cipher->key);
 183 #else
 184   rijndaelEncrypt(cipher->rk, cipher->nr, buf, cipher->buf);
 185 #endif
 186 #endif
 187 }
 188
 189 /**
 190  * Return a newly allocated counter-mode AES128 cipher implementation.
 191  */
 192 aes_cnt_cipher_t*
 193 aes_new_cipher(void)
 194 {
 195   aes_cnt_cipher_t* result = tor_malloc_zero(sizeof(aes_cnt_cipher_t));
 196
 197   return result;
 198 }
 199
 200 /** Set the key of <b>cipher</b> to <b>key</b>, which is
 201  * <b>key_bits</b> bits long (must be 128, 192, or 256).  Also resets
 202  * the counter to 0.
 203  */
 204 void
 205 aes_set_key(aes_cnt_cipher_t *cipher, const char *key, int key_bits)
 206 {
 207 #if defined(USE_OPENSSL_EVP)
 208   const EVP_CIPHER *c;
 209   switch (key_bits) {
 210     case 128: c = EVP_aes_128_ecb(); break;
 211     case 192: c = EVP_aes_192_ecb(); break;
 212     case 256: c = EVP_aes_256_ecb(); break;
 213     default: tor_assert(0);
 214   }
 215   EVP_EncryptInit(&cipher->key, c, (const unsigned char*)key, NULL);
 216 #elif defined(USE_OPENSSL_AES)
 217   AES_set_encrypt_key((const unsigned char *)key, key_bits, &(cipher->key));
 218 #else
 219   cipher->nr = rijndaelKeySetupEnc(cipher->rk, (const unsigned char*)key,
 220                                    key_bits);
 221 #endif
 222   cipher->counter0 = 0;
 223   cipher->counter1 = 0;
 224   cipher->pos = 0;
 225   _aes_fill_buf(cipher);
 226 }
 227
 228 /** Release storage held by <b>cipher</b>
 229  */
 230 void
 231 aes_free_cipher(aes_cnt_cipher_t *cipher)
 232 {
 233   assert(cipher);
 234 #ifdef USE_OPENSSL_EVP
 235   EVP_CIPHER_CTX_cleanup(&cipher->key);
 236 #endif
 237   memset(cipher, 0, sizeof(cipher));
 238   tor_free(cipher);
 239 }
 240
 241 /** Encrypt <b>len</b> bytes from <b>input</b>, storing the result in
 242  * <b>output</b>.  Uses the key in <b>cipher</b>, and advances the counter
 243  * by <b>len</b> bytes as it encrypts.
 244  */
 245 void
 246 aes_crypt(aes_cnt_cipher_t *cipher, const char *input, size_t len,
 247           char *output)
 248 {
 249   /* XXXX This function is up to 5% of our runtime in some profiles;
 250    * we should look into unrolling some of the loops; taking advantage
 251    * of alignmement, using a bigger buffer, and so on. Not till after 0.1.2.x,
 252    * though. */
 253   int c = cipher->pos;
 254   if (!len) return;
 255
 256   while (1) {
 257     do {
 258       if (len-- == 0) { cipher->pos = c; return; }
 259       *(output++) = *(input++) ^ cipher->buf[c];
 260     } while (++c != 16);
 261     cipher->pos = c = 0;
 262     if (! ++cipher->counter0)
 263       ++cipher->counter1;
 264     _aes_fill_buf(cipher);
 265   }
 266 }
 267
 268 /** Return the current value of <b>cipher</b>'s counter. */
 269 u64
 270 aes_get_counter(aes_cnt_cipher_t *cipher)
 271 {
 272   u64 counter = cipher->pos;
 273   counter |= ((u64)cipher->counter0) << 4;
 274   counter |= ((u64)cipher->counter1) << 36;
 275   return counter;
 276 }
 277
 278 /** Set <b>cipher</b>'s counter to <b>counter</b>. */
 279 void
 280 aes_set_counter(aes_cnt_cipher_t *cipher, u64 counter)
 281 {
 282   cipher->pos = (u8)(counter & 0x0f);
 283   cipher->counter0 = (u32) ((counter >> 4) & 0xffffffff);
 284   cipher->counter1 = (u32) (counter >> 36);
 285   _aes_fill_buf(cipher);
 286 }
 287
 288 /** Increment <b>cipher</b>'s counter by <b>delta</b>. */
 289 void
 290 aes_adjust_counter(aes_cnt_cipher_t *cipher, long delta)
 291 {
 292   u64 counter = aes_get_counter(cipher);
 293   counter += delta;
 294   aes_set_counter(cipher, counter);
 295 }
 296
 297 #ifdef USE_BUILTIN_AES
 298 /*======================================================================*/
 299 /* From rijndael-alg-fst.c */
 300
 301 /**
 302  * rijndael-alg-fst.c
 303  *
 304  * @version 3.0 (December 2000)
 305  *
 306  * Optimised ANSI C code for the Rijndael cipher (now AES)
 307  *
 308  * @author Vincent Rijmen <vincent.rijmen@esat.kuleuven.ac.be>
 309  * @author Antoon Bosselaers <antoon.bosselaers@esat.kuleuven.ac.be>
 310  * @author Paulo Barreto <paulo.barreto@terra.com.br>
 311  *
 312  * This code is hereby placed in the public domain.
 313  *
 314  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS
 315  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 316  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 317  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
 318  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 319  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 320  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 321  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 322  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
 323  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 324  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 325  */
 326
 327 /*
 328 Te0[x] = S [x].[02, 01, 01, 03];
 329 Te1[x] = S [x].[03, 02, 01, 01];
 330 Te2[x] = S [x].[01, 03, 02, 01];
 331 Te3[x] = S [x].[01, 01, 03, 02];
 332 Te4[x] = S [x].[01, 01, 01, 01];
 333
 334 Td0[x] = Si[x].[0e, 09, 0d, 0b];
 335 Td1[x] = Si[x].[0b, 0e, 09, 0d];
 336 Td2[x] = Si[x].[0d, 0b, 0e, 09];
 337 Td3[x] = Si[x].[09, 0d, 0b, 0e];
 338 Td4[x] = Si[x].[01, 01, 01, 01];
 339 */
 340
 341 static const u32 Te0[256] = {
 342     0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU,
 343     0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U,
 344     0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU,
 345     0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU,
 346     0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U,
 347     0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU,
 348     0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU,
 349     0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU,
 350     0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU,
 351     0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU,
 352     0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U,
 353     0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU,
 354     0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU,
 355     0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U,
 356     0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU,
 357     0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU,
 358     0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU,
 359     0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU,
 360     0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU,
 361     0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U,
 362     0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU,
 363     0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU,
 364     0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU,
 365     0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU,
 366     0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U,
 367     0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U,
 368     0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U,
 369     0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U,
 370     0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU,
 371     0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U,
 372     0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U,
 373     0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU,
 374     0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU,
 375     0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U,
 376     0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U,
 377     0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U,
 378     0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU,
 379     0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U,
 380     0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU,
 381     0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U,
 382     0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU,
 383     0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U,
 384     0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U,
 385     0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU,
 386     0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U,
 387     0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U,
 388     0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U,
 389     0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U,
 390     0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U,
 391     0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U,
 392     0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U,
 393     0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U,
 394     0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU,
 395     0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U,
 396     0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U,
 397     0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U,
 398     0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U,
 399     0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U,
 400     0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U,
 401     0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU,
 402     0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U,
 403     0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U,
 404     0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U,
 405     0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU,
 406 };
 407 static const u32 Te1[256] = {
 408     0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU,
 409     0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U,
 410     0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU,
 411     0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U,
 412     0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU,
 413     0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U,
 414     0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU,
 415     0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U,
 416     0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U,
 417     0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU,
 418     0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U,
 419     0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U,
 420     0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U,
 421     0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU,
 422     0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U,
 423     0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U,
 424     0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU,
 425     0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U,
 426     0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U,
 427     0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U,
 428     0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU,
 429     0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU,
 430     0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U,
 431     0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU,
 432     0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU,
 433     0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U,
 434     0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU,
 435     0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U,
 436     0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU,
 437     0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U,
 438     0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U,
 439     0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U,
 440     0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU,
 441     0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U,
 442     0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU,
 443     0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U,
 444     0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU,
 445     0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U,
 446     0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U,
 447     0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU,
 448     0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU,
 449     0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU,
 450     0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U,
 451     0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U,
 452     0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU,
 453     0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U,
 454     0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU,
 455     0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U,
 456     0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU,
 457     0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U,
 458     0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU,
 459     0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU,
 460     0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U,
 461     0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU,
 462     0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U,
 463     0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU,
 464     0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U,
 465     0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U,
 466     0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U,
 467     0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU,
 468     0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU,
 469     0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U,
 470     0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU,
 471     0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U,
 472 };
 473 static const u32 Te2[256] = {
 474     0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU,
 475     0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U,
 476     0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU,
 477     0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U,
 478     0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU,
 479     0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U,
 480     0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU,
 481     0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U,
 482     0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U,
 483     0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU,
 484     0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U,
 485     0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U,
 486     0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U,
 487     0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU,
 488     0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U,
 489     0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U,
 490     0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU,
 491     0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U,
 492     0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U,
 493     0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U,
 494     0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU,
 495     0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU,
 496     0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U,
 497     0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU,
 498     0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU,
 499     0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U,
 500     0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU,
 501     0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U,
 502     0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU,
 503     0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U,
 504     0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U,
 505     0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U,
 506     0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU,
 507     0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U,
 508     0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU,
 509     0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U,
 510     0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU,
 511     0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U,
 512     0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U,
 513     0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU,
 514     0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU,
 515     0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU,
 516     0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U,
 517     0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U,
 518     0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU,
 519     0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U,
 520     0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU,
 521     0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U,
 522     0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU,
 523     0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U,
 524     0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU,
 525     0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU,
 526     0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U,
 527     0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU,
 528     0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U,
 529     0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU,
 530     0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U,
 531     0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U,
 532     0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U,
 533     0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU,
 534     0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU,
 535     0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U,
 536     0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU,
 537     0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U,
 538 };
 539 static const u32 Te3[256] = {
 540
 541     0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U,
 542     0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U,
 543     0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U,
 544     0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU,
 545     0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU,
 546     0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU,
 547     0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U,
 548     0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU,
 549     0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU,
 550     0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U,
 551     0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U,
 552     0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU,
 553     0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU,
 554     0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU,
 555     0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU,
 556     0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU,
 557     0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U,
 558     0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU,
 559     0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU,
 560     0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U,
 561     0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U,
 562     0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U,
 563     0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U,
 564     0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U,
 565     0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU,
 566     0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U,
 567     0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU,
 568     0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU,
 569     0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U,
 570     0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U,
 571     0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U,
 572     0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU,
 573     0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U,
 574     0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU,
 575     0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU,
 576     0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U,
 577     0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U,
 578     0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU,
 579     0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U,
 580     0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU,
 581     0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U,
 582     0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U,
 583     0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U,
 584     0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U,
 585     0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU,
 586     0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U,
 587     0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU,
 588     0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U,
 589     0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU,
 590     0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U,
 591     0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU,
 592     0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU,
 593     0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU,
 594     0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU,
 595     0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U,
 596     0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U,
 597     0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U,
 598     0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U,
 599     0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U,
 600     0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U,
 601     0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU,
 602     0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U,
 603     0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU,
 604     0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU,
 605 };
 606 static const u32 Te4[256] = {
 607     0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU,
 608     0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U,
 609     0x30303030U, 0x01010101U, 0x67676767U, 0x2b2b2b2bU,
 610     0xfefefefeU, 0xd7d7d7d7U, 0xababababU, 0x76767676U,
 611     0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU,
 612     0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U,
 613     0xadadadadU, 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU,
 614     0x9c9c9c9cU, 0xa4a4a4a4U, 0x72727272U, 0xc0c0c0c0U,
 615     0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, 0x26262626U,
 616     0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU,
 617     0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U,
 618     0x71717171U, 0xd8d8d8d8U, 0x31313131U, 0x15151515U,
 619     0x04040404U, 0xc7c7c7c7U, 0x23232323U, 0xc3c3c3c3U,
 620     0x18181818U, 0x96969696U, 0x05050505U, 0x9a9a9a9aU,
 621     0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U,
 622     0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U,
 623     0x09090909U, 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU,
 624     0x1b1b1b1bU, 0x6e6e6e6eU, 0x5a5a5a5aU, 0xa0a0a0a0U,
 625     0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, 0xb3b3b3b3U,
 626     0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U,
 627     0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU,
 628     0x20202020U, 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU,
 629     0x6a6a6a6aU, 0xcbcbcbcbU, 0xbebebebeU, 0x39393939U,
 630     0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, 0xcfcfcfcfU,
 631     0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU,
 632     0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U,
 633     0x45454545U, 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU,
 634     0x50505050U, 0x3c3c3c3cU, 0x9f9f9f9fU, 0xa8a8a8a8U,
 635     0x51515151U, 0xa3a3a3a3U, 0x40404040U, 0x8f8f8f8fU,
 636     0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U,
 637     0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U,
 638     0x10101010U, 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U,
 639     0xcdcdcdcdU, 0x0c0c0c0cU, 0x13131313U, 0xececececU,
 640     0x5f5f5f5fU, 0x97979797U, 0x44444444U, 0x17171717U,
 641     0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU,
 642     0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U,
 643     0x60606060U, 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU,
 644     0x22222222U, 0x2a2a2a2aU, 0x90909090U, 0x88888888U,
 645     0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, 0x14141414U,
 646     0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU,
 647     0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU,
 648     0x49494949U, 0x06060606U, 0x24242424U, 0x5c5c5c5cU,
 649     0xc2c2c2c2U, 0xd3d3d3d3U, 0xacacacacU, 0x62626262U,
 650     0x91919191U, 0x95959595U, 0xe4e4e4e4U, 0x79797979U,
 651     0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU,
 652     0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U,
 653     0x6c6c6c6cU, 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU,
 654     0x65656565U, 0x7a7a7a7aU, 0xaeaeaeaeU, 0x08080808U,
 655     0xbabababaU, 0x78787878U, 0x25252525U, 0x2e2e2e2eU,
 656     0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U,
 657     0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU,
 658     0x4b4b4b4bU, 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU,
 659     0x70707070U, 0x3e3e3e3eU, 0xb5b5b5b5U, 0x66666666U,
 660     0x48484848U, 0x03030303U, 0xf6f6f6f6U, 0x0e0e0e0eU,
 661     0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U,
 662     0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU,
 663     0xe1e1e1e1U, 0xf8f8f8f8U, 0x98989898U, 0x11111111U,
 664     0x69696969U, 0xd9d9d9d9U, 0x8e8e8e8eU, 0x94949494U,
 665     0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, 0xe9e9e9e9U,
 666     0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU,
 667     0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU,
 668     0xbfbfbfbfU, 0xe6e6e6e6U, 0x42424242U, 0x68686868U,
 669     0x41414141U, 0x99999999U, 0x2d2d2d2dU, 0x0f0f0f0fU,
 670     0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, 0x16161616U,
 671 };
 672
 673 static const u32 rcon[] = {
 674         0x01000000, 0x02000000, 0x04000000, 0x08000000,
 675         0x10000000, 0x20000000, 0x40000000, 0x80000000,
 676         0x1B000000, 0x36000000, /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
 677 };
 678
 679 #define SWAP(x) (_lrotl(x, 8) & 0x00ff00ff | _lrotr(x, 8) & 0xff00ff00)
 680
 681 #ifdef _MSC_VER
 682 #define GETU32(p) SWAP(*((u32 *)(p)))
 683 #define PUTU32(ct, st) { *((u32 *)(ct)) = SWAP((st)); }
 684 #else
 685 #define GETU32(pt) (((u32)(pt)[0] << 24) ^ ((u32)(pt)[1] << 16) ^ ((u32)(pt)[2] <<  8) ^ ((u32)(pt)[3]))
 686 #define PUTU32(ct, st) { (ct)[0] = (u8)((st) >> 24); (ct)[1] = (u8)((st) >> 16); (ct)[2] = (u8)((st) >>  8); (ct)[3] = (u8)(st); }
 687 #endif
 688
 689 /**
 690  * Expand the cipher key into the encryption key schedule.
 691  *
 692  * @return      the number of rounds for the given cipher key size.
 693  */
 694 static int
 695 rijndaelKeySetupEnc(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], int keyBits)
 696 {
 697         int i = 0;
 698         u32 temp;
 699
 700         rk[0] = GETU32(cipherKey     );
 701         rk[1] = GETU32(cipherKey +  4);
 702         rk[2] = GETU32(cipherKey +  8);
 703         rk[3] = GETU32(cipherKey + 12);
 704         if (keyBits == 128) {
 705                 for (;;) {
 706                         temp  = rk[3];
 707                         rk[4] = rk[0] ^
 708                                 (Te4[(temp >> 16) & 0xff] & 0xff000000) ^
 709                                 (Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
 710                                 (Te4[(temp      ) & 0xff] & 0x0000ff00) ^
 711                                 (Te4[(temp >> 24)       ] & 0x000000ff) ^
 712                                 rcon[i];
 713                         rk[5] = rk[1] ^ rk[4];
 714                         rk[6] = rk[2] ^ rk[5];
 715                         rk[7] = rk[3] ^ rk[6];
 716                         if (++i == 10) {
 717                                 return 10;
 718                         }
 719                         rk += 4;
 720                 }
 721         }
 722         rk[4] = GETU32(cipherKey + 16);
 723         rk[5] = GETU32(cipherKey + 20);
 724         if (keyBits == 192) {
 725                 for (;;) {
 726                         temp = rk[ 5];
 727                         rk[ 6] = rk[ 0] ^
 728                                 (Te4[(temp >> 16) & 0xff] & 0xff000000) ^
 729                                 (Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
 730                                 (Te4[(temp      ) & 0xff] & 0x0000ff00) ^
 731                                 (Te4[(temp >> 24)       ] & 0x000000ff) ^
 732                                 rcon[i];
 733                         rk[ 7] = rk[ 1] ^ rk[ 6];
 734                         rk[ 8] = rk[ 2] ^ rk[ 7];
 735                         rk[ 9] = rk[ 3] ^ rk[ 8];
 736                         if (++i == 8) {
 737                                 return 12;
 738                         }
 739                         rk[10] = rk[ 4] ^ rk[ 9];
 740                         rk[11] = rk[ 5] ^ rk[10];
 741                         rk += 6;
 742                 }
 743         }
 744         rk[6] = GETU32(cipherKey + 24);
 745         rk[7] = GETU32(cipherKey + 28);
 746         if (keyBits == 256) {
 747         for (;;) {
 748                 temp = rk[ 7];
 749                 rk[ 8] = rk[ 0] ^
 750                         (Te4[(temp >> 16) & 0xff] & 0xff000000) ^
 751                         (Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
 752                         (Te4[(temp      ) & 0xff] & 0x0000ff00) ^
 753                         (Te4[(temp >> 24)       ] & 0x000000ff) ^
 754                         rcon[i];
 755                 rk[ 9] = rk[ 1] ^ rk[ 8];
 756                 rk[10] = rk[ 2] ^ rk[ 9];
 757                 rk[11] = rk[ 3] ^ rk[10];
 758                         if (++i == 7) {
 759                                 return 14;
 760                         }
 761                 temp = rk[11];
 762                 rk[12] = rk[ 4] ^
 763                         (Te4[(temp >> 24)       ] & 0xff000000) ^
 764                         (Te4[(temp >> 16) & 0xff] & 0x00ff0000) ^
 765                         (Te4[(temp >>  8) & 0xff] & 0x0000ff00) ^
 766                         (Te4[(temp      ) & 0xff] & 0x000000ff);
 767                 rk[13] = rk[ 5] ^ rk[12];
 768                 rk[14] = rk[ 6] ^ rk[13];
 769                 rk[15] = rk[ 7] ^ rk[14];
 770
 771                         rk += 8;
 772         }
 773         }
 774         return 0;
 775 }
 776
 777 #ifdef USE_RIJNDAEL_COUNTER_OPTIMIZATION
 778 static void
 779 rijndaelEncrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, u32 ctr1, u32 ctr0, u8 ct[16])
 780 #else
 781 static void
 782 rijndaelEncrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 pt[16], u8 ct[16])
 783 #endif
 784 {
 785         u32 s0, s1, s2, s3, t0, t1, t2, t3;
 786 #ifndef FULL_UNROLL
 787     int r;
 788 #endif /* ?FULL_UNROLL */
 789
 790     /*
 791          * map byte array block to cipher state
 792          * and add initial round key:
 793          */
 794 #ifdef USE_RIJNDAEL_COUNTER_OPTIMIZATION
 795         s0 = rk[0];
 796         s1 = rk[1];
 797         s2 = ctr1 ^ rk[2];
 798         s3 = ctr0 ^ rk[3];
 799 #else
 800         s0 = GETU32(pt     ) ^ rk[0];
 801         s1 = GETU32(pt +  4) ^ rk[1];
 802         s2 = GETU32(pt +  8) ^ rk[2];
 803         s3 = GETU32(pt + 12) ^ rk[3];
 804 #endif
 805
 806 #ifdef FULL_UNROLL
 807     /* round 1: */
 808         t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[ 4];
 809         t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[ 5];
 810         t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[ 6];
 811         t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[ 7];
 812         /* round 2: */
 813         s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[ 8];
 814         s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[ 9];
 815         s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[10];
 816         s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[11];
 817     /* round 3: */
 818         t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[12];
 819         t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[13];
 820         t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[14];
 821         t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[15];
 822         /* round 4: */
 823         s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[16];
 824         s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[17];
 825         s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[18];
 826         s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[19];
 827     /* round 5: */
 828         t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[20];
 829         t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[21];
 830         t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[22];
 831         t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[23];
 832         /* round 6: */
 833         s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[24];
 834         s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[25];
 835         s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[26];
 836         s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[27];
 837     /* round 7: */
 838         t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[28];
 839         t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[29];
 840         t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[30];
 841         t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[31];
 842         /* round 8: */
 843         s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[32];
 844         s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[33];
 845         s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[34];
 846         s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[35];
 847     /* round 9: */
 848         t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[36];
 849         t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[37];
 850         t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[38];
 851         t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[39];
 852     if (Nr > 10) {
 853         /* round 10: */
 854         s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[40];
 855         s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[41];
 856         s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[42];
 857         s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[43];
 858         /* round 11: */
 859         t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[44];
 860         t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[45];
 861         t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[46];
 862         t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[47];
 863         if (Nr > 12) {
 864             /* round 12: */
 865             s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[48];
 866             s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[49];
 867             s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[50];
 868             s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[51];
 869             /* round 13: */
 870             t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[52];
 871             t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[53];
 872             t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[54];
 873             t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[55];
 874         }
 875     }
 876     rk += Nr << 2;
 877 #else  /* !FULL_UNROLL */
 878     /*
 879          * Nr - 1 full rounds:
 880          */
 881     r = Nr >> 1;
 882     for (;;) {
 883         t0 =
 884             Te0[(s0 >> 24)       ] ^
 885             Te1[(s1 >> 16) & 0xff] ^
 886             Te2[(s2 >>  8) & 0xff] ^
 887             Te3[(s3      ) & 0xff] ^
 888             rk[4];
 889         t1 =
 890             Te0[(s1 >> 24)       ] ^
 891             Te1[(s2 >> 16) & 0xff] ^
 892             Te2[(s3 >>  8) & 0xff] ^
 893             Te3[(s0      ) & 0xff] ^
 894             rk[5];
 895         t2 =
 896             Te0[(s2 >> 24)       ] ^
 897             Te1[(s3 >> 16) & 0xff] ^
 898             Te2[(s0 >>  8) & 0xff] ^
 899             Te3[(s1      ) & 0xff] ^
 900             rk[6];
 901         t3 =
 902             Te0[(s3 >> 24)       ] ^
 903             Te1[(s0 >> 16) & 0xff] ^
 904             Te2[(s1 >>  8) & 0xff] ^
 905             Te3[(s2      ) & 0xff] ^
 906             rk[7];
 907
 908         rk += 8;
 909         if (--r == 0) {
 910             break;
 911         }
 912
 913         s0 =
 914             Te0[(t0 >> 24)       ] ^
 915             Te1[(t1 >> 16) & 0xff] ^
 916             Te2[(t2 >>  8) & 0xff] ^
 917             Te3[(t3      ) & 0xff] ^
 918             rk[0];
 919         s1 =
 920             Te0[(t1 >> 24)       ] ^
 921             Te1[(t2 >> 16) & 0xff] ^
 922             Te2[(t3 >>  8) & 0xff] ^
 923             Te3[(t0      ) & 0xff] ^
 924             rk[1];
 925         s2 =
 926             Te0[(t2 >> 24)       ] ^
 927             Te1[(t3 >> 16) & 0xff] ^
 928             Te2[(t0 >>  8) & 0xff] ^
 929             Te3[(t1      ) & 0xff] ^
 930             rk[2];
 931         s3 =
 932             Te0[(t3 >> 24)       ] ^
 933             Te1[(t0 >> 16) & 0xff] ^
 934             Te2[(t1 >>  8) & 0xff] ^
 935             Te3[(t2      ) & 0xff] ^
 936             rk[3];
 937     }
 938 #endif /* ?FULL_UNROLL */
 939     /*
 940          * apply last round and
 941          * map cipher state to byte array block:
 942          */
 943         s0 =
 944                 (Te4[(t0 >> 24)       ] & 0xff000000) ^
 945                 (Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^
 946                 (Te4[(t2 >>  8) & 0xff] & 0x0000ff00) ^
 947                 (Te4[(t3      ) & 0xff] & 0x000000ff) ^
 948                 rk[0];
 949         PUTU32(ct     , s0);
 950         s1 =
 951                 (Te4[(t1 >> 24)       ] & 0xff000000) ^
 952                 (Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^
 953                 (Te4[(t3 >>  8) & 0xff] & 0x0000ff00) ^
 954                 (Te4[(t0      ) & 0xff] & 0x000000ff) ^
 955                 rk[1];
 956         PUTU32(ct +  4, s1);
 957         s2 =
 958                 (Te4[(t2 >> 24)       ] & 0xff000000) ^
 959                 (Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^
 960                 (Te4[(t0 >>  8) & 0xff] & 0x0000ff00) ^
 961                 (Te4[(t1      ) & 0xff] & 0x000000ff) ^
 962                 rk[2];
 963         PUTU32(ct +  8, s2);
 964         s3 =
 965                 (Te4[(t3 >> 24)       ] & 0xff000000) ^
 966                 (Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^
 967                 (Te4[(t1 >>  8) & 0xff] & 0x0000ff00) ^
 968                 (Te4[(t2      ) & 0xff] & 0x000000ff) ^
 969                 rk[3];
 970         PUTU32(ct + 12, s3);
 971 }
 972 #endif
 973
 974 #ifdef AES_BENCHMARK
 975 int
 976 main(int c, char **v)
 977 {
 978   int i;
 979   char blob[509]; /* the size of a cell payload. */
 980   char blob_out[509];
 981   aes_cnt_cipher_t *cipher = aes_new_cipher();
 982   aes_set_key(cipher, "aesbenchmarkkey!", 128);
 983   memset(blob, 'z', sizeof(blob));
 984
 985   for (i=0;i<1000000; ++i) {
 986     aes_crypt(cipher, blob, sizeof(blob), blob_out);
 987   }
 988   return 0;
 989 }
 990 #endif