target/arm/crypto_helper.c

   1 /*
   2  * crypto_helper.c - emulate v8 Crypto Extensions instructions
   3  *
   4  * Copyright (C) 2013 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  */
  11
  12 #include "qemu/osdep.h"
  13
  14 #include "cpu.h"
  15 #include "exec/helper-proto.h"
  16 #include "tcg/tcg-gvec-desc.h"
  17 #include "crypto/aes.h"
  18 #include "crypto/sm4.h"
  19 #include "vec_internal.h"
  20
  21 union CRYPTO_STATE {
  22     uint8_t    bytes[16];
  23     uint32_t   words[4];
  24     uint64_t   l[2];
  25 };
  26
  27 #if HOST_BIG_ENDIAN
  28 #define CR_ST_BYTE(state, i)   ((state).bytes[(15 - (i)) ^ 8])
  29 #define CR_ST_WORD(state, i)   ((state).words[(3 - (i)) ^ 2])
  30 #else
  31 #define CR_ST_BYTE(state, i)   ((state).bytes[i])
  32 #define CR_ST_WORD(state, i)   ((state).words[i])
  33 #endif
  34
  35 /*
  36  * The caller has not been converted to full gvec, and so only
  37  * modifies the low 16 bytes of the vector register.
  38  */
  39 static void clear_tail_16(void *vd, uint32_t desc)
  40 {
  41     int opr_sz = simd_oprsz(desc);
  42     int max_sz = simd_maxsz(desc);
  43
  44     assert(opr_sz == 16);
  45     clear_tail(vd, opr_sz, max_sz);
  46 }
  47
  48 static void do_crypto_aese(uint64_t *rd, uint64_t *rn,
  49                            uint64_t *rm, bool decrypt)
  50 {
  51     static uint8_t const * const sbox[2] = { AES_sbox, AES_isbox };
  52     static uint8_t const * const shift[2] = { AES_shifts, AES_ishifts };
  53     union CRYPTO_STATE rk = { .l = { rm[0], rm[1] } };
  54     union CRYPTO_STATE st = { .l = { rn[0], rn[1] } };
  55     int i;
  56
  57     /* xor state vector with round key */
  58     rk.l[0] ^= st.l[0];
  59     rk.l[1] ^= st.l[1];
  60
  61     /* combine ShiftRows operation and sbox substitution */
  62     for (i = 0; i < 16; i++) {
  63         CR_ST_BYTE(st, i) = sbox[decrypt][CR_ST_BYTE(rk, shift[decrypt][i])];
  64     }
  65
  66     rd[0] = st.l[0];
  67     rd[1] = st.l[1];
  68 }
  69
  70 void HELPER(crypto_aese)(void *vd, void *vn, void *vm, uint32_t desc)
  71 {
  72     intptr_t i, opr_sz = simd_oprsz(desc);
  73     bool decrypt = simd_data(desc);
  74
  75     for (i = 0; i < opr_sz; i += 16) {
  76         do_crypto_aese(vd + i, vn + i, vm + i, decrypt);
  77     }
  78     clear_tail(vd, opr_sz, simd_maxsz(desc));
  79 }
  80
  81 static void do_crypto_aesmc(uint64_t *rd, uint64_t *rm, bool decrypt)
  82 {
  83     static uint32_t const mc[][256] = { {
  84         /* MixColumns lookup table */
  85         0x00000000, 0x03010102, 0x06020204, 0x05030306,
  86         0x0c040408, 0x0f05050a, 0x0a06060c, 0x0907070e,
  87         0x18080810, 0x1b090912, 0x1e0a0a14, 0x1d0b0b16,
  88         0x140c0c18, 0x170d0d1a, 0x120e0e1c, 0x110f0f1e,
  89         0x30101020, 0x33111122, 0x36121224, 0x35131326,
  90         0x3c141428, 0x3f15152a, 0x3a16162c, 0x3917172e,
  91         0x28181830, 0x2b191932, 0x2e1a1a34, 0x2d1b1b36,
  92         0x241c1c38, 0x271d1d3a, 0x221e1e3c, 0x211f1f3e,
  93         0x60202040, 0x63212142, 0x66222244, 0x65232346,
  94         0x6c242448, 0x6f25254a, 0x6a26264c, 0x6927274e,
  95         0x78282850, 0x7b292952, 0x7e2a2a54, 0x7d2b2b56,
  96         0x742c2c58, 0x772d2d5a, 0x722e2e5c, 0x712f2f5e,
  97         0x50303060, 0x53313162, 0x56323264, 0x55333366,
  98         0x5c343468, 0x5f35356a, 0x5a36366c, 0x5937376e,
  99         0x48383870, 0x4b393972, 0x4e3a3a74, 0x4d3b3b76,
 100         0x443c3c78, 0x473d3d7a, 0x423e3e7c, 0x413f3f7e,
 101         0xc0404080, 0xc3414182, 0xc6424284, 0xc5434386,
 102         0xcc444488, 0xcf45458a, 0xca46468c, 0xc947478e,
 103         0xd8484890, 0xdb494992, 0xde4a4a94, 0xdd4b4b96,
 104         0xd44c4c98, 0xd74d4d9a, 0xd24e4e9c, 0xd14f4f9e,
 105         0xf05050a0, 0xf35151a2, 0xf65252a4, 0xf55353a6,
 106         0xfc5454a8, 0xff5555aa, 0xfa5656ac, 0xf95757ae,
 107         0xe85858b0, 0xeb5959b2, 0xee5a5ab4, 0xed5b5bb6,
 108         0xe45c5cb8, 0xe75d5dba, 0xe25e5ebc, 0xe15f5fbe,
 109         0xa06060c0, 0xa36161c2, 0xa66262c4, 0xa56363c6,
 110         0xac6464c8, 0xaf6565ca, 0xaa6666cc, 0xa96767ce,
 111         0xb86868d0, 0xbb6969d2, 0xbe6a6ad4, 0xbd6b6bd6,
 112         0xb46c6cd8, 0xb76d6dda, 0xb26e6edc, 0xb16f6fde,
 113         0x907070e0, 0x937171e2, 0x967272e4, 0x957373e6,
 114         0x9c7474e8, 0x9f7575ea, 0x9a7676ec, 0x997777ee,
 115         0x887878f0, 0x8b7979f2, 0x8e7a7af4, 0x8d7b7bf6,
 116         0x847c7cf8, 0x877d7dfa, 0x827e7efc, 0x817f7ffe,
 117         0x9b80801b, 0x98818119, 0x9d82821f, 0x9e83831d,
 118         0x97848413, 0x94858511, 0x91868617, 0x92878715,
 119         0x8388880b, 0x80898909, 0x858a8a0f, 0x868b8b0d,
 120         0x8f8c8c03, 0x8c8d8d01, 0x898e8e07, 0x8a8f8f05,
 121         0xab90903b, 0xa8919139, 0xad92923f, 0xae93933d,
 122         0xa7949433, 0xa4959531, 0xa1969637, 0xa2979735,
 123         0xb398982b, 0xb0999929, 0xb59a9a2f, 0xb69b9b2d,
 124         0xbf9c9c23, 0xbc9d9d21, 0xb99e9e27, 0xba9f9f25,
 125         0xfba0a05b, 0xf8a1a159, 0xfda2a25f, 0xfea3a35d,
 126         0xf7a4a453, 0xf4a5a551, 0xf1a6a657, 0xf2a7a755,
 127         0xe3a8a84b, 0xe0a9a949, 0xe5aaaa4f, 0xe6abab4d,
 128         0xefacac43, 0xecadad41, 0xe9aeae47, 0xeaafaf45,
 129         0xcbb0b07b, 0xc8b1b179, 0xcdb2b27f, 0xceb3b37d,
 130         0xc7b4b473, 0xc4b5b571, 0xc1b6b677, 0xc2b7b775,
 131         0xd3b8b86b, 0xd0b9b969, 0xd5baba6f, 0xd6bbbb6d,
 132         0xdfbcbc63, 0xdcbdbd61, 0xd9bebe67, 0xdabfbf65,
 133         0x5bc0c09b, 0x58c1c199, 0x5dc2c29f, 0x5ec3c39d,
 134         0x57c4c493, 0x54c5c591, 0x51c6c697, 0x52c7c795,
 135         0x43c8c88b, 0x40c9c989, 0x45caca8f, 0x46cbcb8d,
 136         0x4fcccc83, 0x4ccdcd81, 0x49cece87, 0x4acfcf85,
 137         0x6bd0d0bb, 0x68d1d1b9, 0x6dd2d2bf, 0x6ed3d3bd,
 138         0x67d4d4b3, 0x64d5d5b1, 0x61d6d6b7, 0x62d7d7b5,
 139         0x73d8d8ab, 0x70d9d9a9, 0x75dadaaf, 0x76dbdbad,
 140         0x7fdcdca3, 0x7cdddda1, 0x79dedea7, 0x7adfdfa5,
 141         0x3be0e0db, 0x38e1e1d9, 0x3de2e2df, 0x3ee3e3dd,
 142         0x37e4e4d3, 0x34e5e5d1, 0x31e6e6d7, 0x32e7e7d5,
 143         0x23e8e8cb, 0x20e9e9c9, 0x25eaeacf, 0x26ebebcd,
 144         0x2fececc3, 0x2cededc1, 0x29eeeec7, 0x2aefefc5,
 145         0x0bf0f0fb, 0x08f1f1f9, 0x0df2f2ff, 0x0ef3f3fd,
 146         0x07f4f4f3, 0x04f5f5f1, 0x01f6f6f7, 0x02f7f7f5,
 147         0x13f8f8eb, 0x10f9f9e9, 0x15fafaef, 0x16fbfbed,
 148         0x1ffcfce3, 0x1cfdfde1, 0x19fefee7, 0x1affffe5,
 149     }, {
 150         /* Inverse MixColumns lookup table */
 151         0x00000000, 0x0b0d090e, 0x161a121c, 0x1d171b12,
 152         0x2c342438, 0x27392d36, 0x3a2e3624, 0x31233f2a,
 153         0x58684870, 0x5365417e, 0x4e725a6c, 0x457f5362,
 154         0x745c6c48, 0x7f516546, 0x62467e54, 0x694b775a,
 155         0xb0d090e0, 0xbbdd99ee, 0xa6ca82fc, 0xadc78bf2,
 156         0x9ce4b4d8, 0x97e9bdd6, 0x8afea6c4, 0x81f3afca,
 157         0xe8b8d890, 0xe3b5d19e, 0xfea2ca8c, 0xf5afc382,
 158         0xc48cfca8, 0xcf81f5a6, 0xd296eeb4, 0xd99be7ba,
 159         0x7bbb3bdb, 0x70b632d5, 0x6da129c7, 0x66ac20c9,
 160         0x578f1fe3, 0x5c8216ed, 0x41950dff, 0x4a9804f1,
 161         0x23d373ab, 0x28de7aa5, 0x35c961b7, 0x3ec468b9,
 162         0x0fe75793, 0x04ea5e9d, 0x19fd458f, 0x12f04c81,
 163         0xcb6bab3b, 0xc066a235, 0xdd71b927, 0xd67cb029,
 164         0xe75f8f03, 0xec52860d, 0xf1459d1f, 0xfa489411,
 165         0x9303e34b, 0x980eea45, 0x8519f157, 0x8e14f859,
 166         0xbf37c773, 0xb43ace7d, 0xa92dd56f, 0xa220dc61,
 167         0xf66d76ad, 0xfd607fa3, 0xe07764b1, 0xeb7a6dbf,
 168         0xda595295, 0xd1545b9b, 0xcc434089, 0xc74e4987,
 169         0xae053edd, 0xa50837d3, 0xb81f2cc1, 0xb31225cf,
 170         0x82311ae5, 0x893c13eb, 0x942b08f9, 0x9f2601f7,
 171         0x46bde64d, 0x4db0ef43, 0x50a7f451, 0x5baafd5f,
 172         0x6a89c275, 0x6184cb7b, 0x7c93d069, 0x779ed967,
 173         0x1ed5ae3d, 0x15d8a733, 0x08cfbc21, 0x03c2b52f,
 174         0x32e18a05, 0x39ec830b, 0x24fb9819, 0x2ff69117,
 175         0x8dd64d76, 0x86db4478, 0x9bcc5f6a, 0x90c15664,
 176         0xa1e2694e, 0xaaef6040, 0xb7f87b52, 0xbcf5725c,
 177         0xd5be0506, 0xdeb30c08, 0xc3a4171a, 0xc8a91e14,
 178         0xf98a213e, 0xf2872830, 0xef903322, 0xe49d3a2c,
 179         0x3d06dd96, 0x360bd498, 0x2b1ccf8a, 0x2011c684,
 180         0x1132f9ae, 0x1a3ff0a0, 0x0728ebb2, 0x0c25e2bc,
 181         0x656e95e6, 0x6e639ce8, 0x737487fa, 0x78798ef4,
 182         0x495ab1de, 0x4257b8d0, 0x5f40a3c2, 0x544daacc,
 183         0xf7daec41, 0xfcd7e54f, 0xe1c0fe5d, 0xeacdf753,
 184         0xdbeec879, 0xd0e3c177, 0xcdf4da65, 0xc6f9d36b,
 185         0xafb2a431, 0xa4bfad3f, 0xb9a8b62d, 0xb2a5bf23,
 186         0x83868009, 0x888b8907, 0x959c9215, 0x9e919b1b,
 187         0x470a7ca1, 0x4c0775af, 0x51106ebd, 0x5a1d67b3,
 188         0x6b3e5899, 0x60335197, 0x7d244a85, 0x7629438b,
 189         0x1f6234d1, 0x146f3ddf, 0x097826cd, 0x02752fc3,
 190         0x335610e9, 0x385b19e7, 0x254c02f5, 0x2e410bfb,
 191         0x8c61d79a, 0x876cde94, 0x9a7bc586, 0x9176cc88,
 192         0xa055f3a2, 0xab58faac, 0xb64fe1be, 0xbd42e8b0,
 193         0xd4099fea, 0xdf0496e4, 0xc2138df6, 0xc91e84f8,
 194         0xf83dbbd2, 0xf330b2dc, 0xee27a9ce, 0xe52aa0c0,
 195         0x3cb1477a, 0x37bc4e74, 0x2aab5566, 0x21a65c68,
 196         0x10856342, 0x1b886a4c, 0x069f715e, 0x0d927850,
 197         0x64d90f0a, 0x6fd40604, 0x72c31d16, 0x79ce1418,
 198         0x48ed2b32, 0x43e0223c, 0x5ef7392e, 0x55fa3020,
 199         0x01b79aec, 0x0aba93e2, 0x17ad88f0, 0x1ca081fe,
 200         0x2d83bed4, 0x268eb7da, 0x3b99acc8, 0x3094a5c6,
 201         0x59dfd29c, 0x52d2db92, 0x4fc5c080, 0x44c8c98e,
 202         0x75ebf6a4, 0x7ee6ffaa, 0x63f1e4b8, 0x68fcedb6,
 203         0xb1670a0c, 0xba6a0302, 0xa77d1810, 0xac70111e,
 204         0x9d532e34, 0x965e273a, 0x8b493c28, 0x80443526,
 205         0xe90f427c, 0xe2024b72, 0xff155060, 0xf418596e,
 206         0xc53b6644, 0xce366f4a, 0xd3217458, 0xd82c7d56,
 207         0x7a0ca137, 0x7101a839, 0x6c16b32b, 0x671bba25,
 208         0x5638850f, 0x5d358c01, 0x40229713, 0x4b2f9e1d,
 209         0x2264e947, 0x2969e049, 0x347efb5b, 0x3f73f255,
 210         0x0e50cd7f, 0x055dc471, 0x184adf63, 0x1347d66d,
 211         0xcadc31d7, 0xc1d138d9, 0xdcc623cb, 0xd7cb2ac5,
 212         0xe6e815ef, 0xede51ce1, 0xf0f207f3, 0xfbff0efd,
 213         0x92b479a7, 0x99b970a9, 0x84ae6bbb, 0x8fa362b5,
 214         0xbe805d9f, 0xb58d5491, 0xa89a4f83, 0xa397468d,
 215     } };
 216
 217     union CRYPTO_STATE st = { .l = { rm[0], rm[1] } };
 218     int i;
 219
 220     for (i = 0; i < 16; i += 4) {
 221         CR_ST_WORD(st, i >> 2) =
 222             mc[decrypt][CR_ST_BYTE(st, i)] ^
 223             rol32(mc[decrypt][CR_ST_BYTE(st, i + 1)], 8) ^
 224             rol32(mc[decrypt][CR_ST_BYTE(st, i + 2)], 16) ^
 225             rol32(mc[decrypt][CR_ST_BYTE(st, i + 3)], 24);
 226     }
 227
 228     rd[0] = st.l[0];
 229     rd[1] = st.l[1];
 230 }
 231
 232 void HELPER(crypto_aesmc)(void *vd, void *vm, uint32_t desc)
 233 {
 234     intptr_t i, opr_sz = simd_oprsz(desc);
 235     bool decrypt = simd_data(desc);
 236
 237     for (i = 0; i < opr_sz; i += 16) {
 238         do_crypto_aesmc(vd + i, vm + i, decrypt);
 239     }
 240     clear_tail(vd, opr_sz, simd_maxsz(desc));
 241 }
 242
 243 /*
 244  * SHA-1 logical functions
 245  */
 246
 247 static uint32_t cho(uint32_t x, uint32_t y, uint32_t z)
 248 {
 249     return (x & (y ^ z)) ^ z;
 250 }
 251
 252 static uint32_t par(uint32_t x, uint32_t y, uint32_t z)
 253 {
 254     return x ^ y ^ z;
 255 }
 256
 257 static uint32_t maj(uint32_t x, uint32_t y, uint32_t z)
 258 {
 259     return (x & y) | ((x | y) & z);
 260 }
 261
 262 void HELPER(crypto_sha1su0)(void *vd, void *vn, void *vm, uint32_t desc)
 263 {
 264     uint64_t *d = vd, *n = vn, *m = vm;
 265     uint64_t d0, d1;
 266
 267     d0 = d[1] ^ d[0] ^ m[0];
 268     d1 = n[0] ^ d[1] ^ m[1];
 269     d[0] = d0;
 270     d[1] = d1;
 271
 272     clear_tail_16(vd, desc);
 273 }
 274
 275 static inline void crypto_sha1_3reg(uint64_t *rd, uint64_t *rn,
 276                                     uint64_t *rm, uint32_t desc,
 277                                     uint32_t (*fn)(union CRYPTO_STATE *d))
 278 {
 279     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 280     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 281     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 282     int i;
 283
 284     for (i = 0; i < 4; i++) {
 285         uint32_t t = fn(&d);
 286
 287         t += rol32(CR_ST_WORD(d, 0), 5) + CR_ST_WORD(n, 0)
 288              + CR_ST_WORD(m, i);
 289
 290         CR_ST_WORD(n, 0) = CR_ST_WORD(d, 3);
 291         CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
 292         CR_ST_WORD(d, 2) = ror32(CR_ST_WORD(d, 1), 2);
 293         CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
 294         CR_ST_WORD(d, 0) = t;
 295     }
 296     rd[0] = d.l[0];
 297     rd[1] = d.l[1];
 298
 299     clear_tail_16(rd, desc);
 300 }
 301
 302 static uint32_t do_sha1c(union CRYPTO_STATE *d)
 303 {
 304     return cho(CR_ST_WORD(*d, 1), CR_ST_WORD(*d, 2), CR_ST_WORD(*d, 3));
 305 }
 306
 307 void HELPER(crypto_sha1c)(void *vd, void *vn, void *vm, uint32_t desc)
 308 {
 309     crypto_sha1_3reg(vd, vn, vm, desc, do_sha1c);
 310 }
 311
 312 static uint32_t do_sha1p(union CRYPTO_STATE *d)
 313 {
 314     return par(CR_ST_WORD(*d, 1), CR_ST_WORD(*d, 2), CR_ST_WORD(*d, 3));
 315 }
 316
 317 void HELPER(crypto_sha1p)(void *vd, void *vn, void *vm, uint32_t desc)
 318 {
 319     crypto_sha1_3reg(vd, vn, vm, desc, do_sha1p);
 320 }
 321
 322 static uint32_t do_sha1m(union CRYPTO_STATE *d)
 323 {
 324     return maj(CR_ST_WORD(*d, 1), CR_ST_WORD(*d, 2), CR_ST_WORD(*d, 3));
 325 }
 326
 327 void HELPER(crypto_sha1m)(void *vd, void *vn, void *vm, uint32_t desc)
 328 {
 329     crypto_sha1_3reg(vd, vn, vm, desc, do_sha1m);
 330 }
 331
 332 void HELPER(crypto_sha1h)(void *vd, void *vm, uint32_t desc)
 333 {
 334     uint64_t *rd = vd;
 335     uint64_t *rm = vm;
 336     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 337
 338     CR_ST_WORD(m, 0) = ror32(CR_ST_WORD(m, 0), 2);
 339     CR_ST_WORD(m, 1) = CR_ST_WORD(m, 2) = CR_ST_WORD(m, 3) = 0;
 340
 341     rd[0] = m.l[0];
 342     rd[1] = m.l[1];
 343
 344     clear_tail_16(vd, desc);
 345 }
 346
 347 void HELPER(crypto_sha1su1)(void *vd, void *vm, uint32_t desc)
 348 {
 349     uint64_t *rd = vd;
 350     uint64_t *rm = vm;
 351     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 352     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 353
 354     CR_ST_WORD(d, 0) = rol32(CR_ST_WORD(d, 0) ^ CR_ST_WORD(m, 1), 1);
 355     CR_ST_WORD(d, 1) = rol32(CR_ST_WORD(d, 1) ^ CR_ST_WORD(m, 2), 1);
 356     CR_ST_WORD(d, 2) = rol32(CR_ST_WORD(d, 2) ^ CR_ST_WORD(m, 3), 1);
 357     CR_ST_WORD(d, 3) = rol32(CR_ST_WORD(d, 3) ^ CR_ST_WORD(d, 0), 1);
 358
 359     rd[0] = d.l[0];
 360     rd[1] = d.l[1];
 361
 362     clear_tail_16(vd, desc);
 363 }
 364
 365 /*
 366  * The SHA-256 logical functions, according to
 367  * http://csrc.nist.gov/groups/STM/cavp/documents/shs/sha256-384-512.pdf
 368  */
 369
 370 static uint32_t S0(uint32_t x)
 371 {
 372     return ror32(x, 2) ^ ror32(x, 13) ^ ror32(x, 22);
 373 }
 374
 375 static uint32_t S1(uint32_t x)
 376 {
 377     return ror32(x, 6) ^ ror32(x, 11) ^ ror32(x, 25);
 378 }
 379
 380 static uint32_t s0(uint32_t x)
 381 {
 382     return ror32(x, 7) ^ ror32(x, 18) ^ (x >> 3);
 383 }
 384
 385 static uint32_t s1(uint32_t x)
 386 {
 387     return ror32(x, 17) ^ ror32(x, 19) ^ (x >> 10);
 388 }
 389
 390 void HELPER(crypto_sha256h)(void *vd, void *vn, void *vm, uint32_t desc)
 391 {
 392     uint64_t *rd = vd;
 393     uint64_t *rn = vn;
 394     uint64_t *rm = vm;
 395     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 396     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 397     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 398     int i;
 399
 400     for (i = 0; i < 4; i++) {
 401         uint32_t t = cho(CR_ST_WORD(n, 0), CR_ST_WORD(n, 1), CR_ST_WORD(n, 2))
 402                      + CR_ST_WORD(n, 3) + S1(CR_ST_WORD(n, 0))
 403                      + CR_ST_WORD(m, i);
 404
 405         CR_ST_WORD(n, 3) = CR_ST_WORD(n, 2);
 406         CR_ST_WORD(n, 2) = CR_ST_WORD(n, 1);
 407         CR_ST_WORD(n, 1) = CR_ST_WORD(n, 0);
 408         CR_ST_WORD(n, 0) = CR_ST_WORD(d, 3) + t;
 409
 410         t += maj(CR_ST_WORD(d, 0), CR_ST_WORD(d, 1), CR_ST_WORD(d, 2))
 411              + S0(CR_ST_WORD(d, 0));
 412
 413         CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
 414         CR_ST_WORD(d, 2) = CR_ST_WORD(d, 1);
 415         CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
 416         CR_ST_WORD(d, 0) = t;
 417     }
 418
 419     rd[0] = d.l[0];
 420     rd[1] = d.l[1];
 421
 422     clear_tail_16(vd, desc);
 423 }
 424
 425 void HELPER(crypto_sha256h2)(void *vd, void *vn, void *vm, uint32_t desc)
 426 {
 427     uint64_t *rd = vd;
 428     uint64_t *rn = vn;
 429     uint64_t *rm = vm;
 430     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 431     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 432     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 433     int i;
 434
 435     for (i = 0; i < 4; i++) {
 436         uint32_t t = cho(CR_ST_WORD(d, 0), CR_ST_WORD(d, 1), CR_ST_WORD(d, 2))
 437                      + CR_ST_WORD(d, 3) + S1(CR_ST_WORD(d, 0))
 438                      + CR_ST_WORD(m, i);
 439
 440         CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
 441         CR_ST_WORD(d, 2) = CR_ST_WORD(d, 1);
 442         CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
 443         CR_ST_WORD(d, 0) = CR_ST_WORD(n, 3 - i) + t;
 444     }
 445
 446     rd[0] = d.l[0];
 447     rd[1] = d.l[1];
 448
 449     clear_tail_16(vd, desc);
 450 }
 451
 452 void HELPER(crypto_sha256su0)(void *vd, void *vm, uint32_t desc)
 453 {
 454     uint64_t *rd = vd;
 455     uint64_t *rm = vm;
 456     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 457     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 458
 459     CR_ST_WORD(d, 0) += s0(CR_ST_WORD(d, 1));
 460     CR_ST_WORD(d, 1) += s0(CR_ST_WORD(d, 2));
 461     CR_ST_WORD(d, 2) += s0(CR_ST_WORD(d, 3));
 462     CR_ST_WORD(d, 3) += s0(CR_ST_WORD(m, 0));
 463
 464     rd[0] = d.l[0];
 465     rd[1] = d.l[1];
 466
 467     clear_tail_16(vd, desc);
 468 }
 469
 470 void HELPER(crypto_sha256su1)(void *vd, void *vn, void *vm, uint32_t desc)
 471 {
 472     uint64_t *rd = vd;
 473     uint64_t *rn = vn;
 474     uint64_t *rm = vm;
 475     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 476     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 477     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 478
 479     CR_ST_WORD(d, 0) += s1(CR_ST_WORD(m, 2)) + CR_ST_WORD(n, 1);
 480     CR_ST_WORD(d, 1) += s1(CR_ST_WORD(m, 3)) + CR_ST_WORD(n, 2);
 481     CR_ST_WORD(d, 2) += s1(CR_ST_WORD(d, 0)) + CR_ST_WORD(n, 3);
 482     CR_ST_WORD(d, 3) += s1(CR_ST_WORD(d, 1)) + CR_ST_WORD(m, 0);
 483
 484     rd[0] = d.l[0];
 485     rd[1] = d.l[1];
 486
 487     clear_tail_16(vd, desc);
 488 }
 489
 490 /*
 491  * The SHA-512 logical functions (same as above but using 64-bit operands)
 492  */
 493
 494 static uint64_t cho512(uint64_t x, uint64_t y, uint64_t z)
 495 {
 496     return (x & (y ^ z)) ^ z;
 497 }
 498
 499 static uint64_t maj512(uint64_t x, uint64_t y, uint64_t z)
 500 {
 501     return (x & y) | ((x | y) & z);
 502 }
 503
 504 static uint64_t S0_512(uint64_t x)
 505 {
 506     return ror64(x, 28) ^ ror64(x, 34) ^ ror64(x, 39);
 507 }
 508
 509 static uint64_t S1_512(uint64_t x)
 510 {
 511     return ror64(x, 14) ^ ror64(x, 18) ^ ror64(x, 41);
 512 }
 513
 514 static uint64_t s0_512(uint64_t x)
 515 {
 516     return ror64(x, 1) ^ ror64(x, 8) ^ (x >> 7);
 517 }
 518
 519 static uint64_t s1_512(uint64_t x)
 520 {
 521     return ror64(x, 19) ^ ror64(x, 61) ^ (x >> 6);
 522 }
 523
 524 void HELPER(crypto_sha512h)(void *vd, void *vn, void *vm, uint32_t desc)
 525 {
 526     uint64_t *rd = vd;
 527     uint64_t *rn = vn;
 528     uint64_t *rm = vm;
 529     uint64_t d0 = rd[0];
 530     uint64_t d1 = rd[1];
 531
 532     d1 += S1_512(rm[1]) + cho512(rm[1], rn[0], rn[1]);
 533     d0 += S1_512(d1 + rm[0]) + cho512(d1 + rm[0], rm[1], rn[0]);
 534
 535     rd[0] = d0;
 536     rd[1] = d1;
 537
 538     clear_tail_16(vd, desc);
 539 }
 540
 541 void HELPER(crypto_sha512h2)(void *vd, void *vn, void *vm, uint32_t desc)
 542 {
 543     uint64_t *rd = vd;
 544     uint64_t *rn = vn;
 545     uint64_t *rm = vm;
 546     uint64_t d0 = rd[0];
 547     uint64_t d1 = rd[1];
 548
 549     d1 += S0_512(rm[0]) + maj512(rn[0], rm[1], rm[0]);
 550     d0 += S0_512(d1) + maj512(d1, rm[0], rm[1]);
 551
 552     rd[0] = d0;
 553     rd[1] = d1;
 554
 555     clear_tail_16(vd, desc);
 556 }
 557
 558 void HELPER(crypto_sha512su0)(void *vd, void *vn, uint32_t desc)
 559 {
 560     uint64_t *rd = vd;
 561     uint64_t *rn = vn;
 562     uint64_t d0 = rd[0];
 563     uint64_t d1 = rd[1];
 564
 565     d0 += s0_512(rd[1]);
 566     d1 += s0_512(rn[0]);
 567
 568     rd[0] = d0;
 569     rd[1] = d1;
 570
 571     clear_tail_16(vd, desc);
 572 }
 573
 574 void HELPER(crypto_sha512su1)(void *vd, void *vn, void *vm, uint32_t desc)
 575 {
 576     uint64_t *rd = vd;
 577     uint64_t *rn = vn;
 578     uint64_t *rm = vm;
 579
 580     rd[0] += s1_512(rn[0]) + rm[0];
 581     rd[1] += s1_512(rn[1]) + rm[1];
 582
 583     clear_tail_16(vd, desc);
 584 }
 585
 586 void HELPER(crypto_sm3partw1)(void *vd, void *vn, void *vm, uint32_t desc)
 587 {
 588     uint64_t *rd = vd;
 589     uint64_t *rn = vn;
 590     uint64_t *rm = vm;
 591     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 592     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 593     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 594     uint32_t t;
 595
 596     t = CR_ST_WORD(d, 0) ^ CR_ST_WORD(n, 0) ^ ror32(CR_ST_WORD(m, 1), 17);
 597     CR_ST_WORD(d, 0) = t ^ ror32(t, 17) ^ ror32(t, 9);
 598
 599     t = CR_ST_WORD(d, 1) ^ CR_ST_WORD(n, 1) ^ ror32(CR_ST_WORD(m, 2), 17);
 600     CR_ST_WORD(d, 1) = t ^ ror32(t, 17) ^ ror32(t, 9);
 601
 602     t = CR_ST_WORD(d, 2) ^ CR_ST_WORD(n, 2) ^ ror32(CR_ST_WORD(m, 3), 17);
 603     CR_ST_WORD(d, 2) = t ^ ror32(t, 17) ^ ror32(t, 9);
 604
 605     t = CR_ST_WORD(d, 3) ^ CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(d, 0), 17);
 606     CR_ST_WORD(d, 3) = t ^ ror32(t, 17) ^ ror32(t, 9);
 607
 608     rd[0] = d.l[0];
 609     rd[1] = d.l[1];
 610
 611     clear_tail_16(vd, desc);
 612 }
 613
 614 void HELPER(crypto_sm3partw2)(void *vd, void *vn, void *vm, uint32_t desc)
 615 {
 616     uint64_t *rd = vd;
 617     uint64_t *rn = vn;
 618     uint64_t *rm = vm;
 619     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 620     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 621     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 622     uint32_t t = CR_ST_WORD(n, 0) ^ ror32(CR_ST_WORD(m, 0), 25);
 623
 624     CR_ST_WORD(d, 0) ^= t;
 625     CR_ST_WORD(d, 1) ^= CR_ST_WORD(n, 1) ^ ror32(CR_ST_WORD(m, 1), 25);
 626     CR_ST_WORD(d, 2) ^= CR_ST_WORD(n, 2) ^ ror32(CR_ST_WORD(m, 2), 25);
 627     CR_ST_WORD(d, 3) ^= CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(m, 3), 25) ^
 628                         ror32(t, 17) ^ ror32(t, 2) ^ ror32(t, 26);
 629
 630     rd[0] = d.l[0];
 631     rd[1] = d.l[1];
 632
 633     clear_tail_16(vd, desc);
 634 }
 635
 636 static inline void QEMU_ALWAYS_INLINE
 637 crypto_sm3tt(uint64_t *rd, uint64_t *rn, uint64_t *rm,
 638              uint32_t desc, uint32_t opcode)
 639 {
 640     union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
 641     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 642     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 643     uint32_t imm2 = simd_data(desc);
 644     uint32_t t;
 645
 646     assert(imm2 < 4);
 647
 648     if (opcode == 0 || opcode == 2) {
 649         /* SM3TT1A, SM3TT2A */
 650         t = par(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1));
 651     } else if (opcode == 1) {
 652         /* SM3TT1B */
 653         t = maj(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1));
 654     } else if (opcode == 3) {
 655         /* SM3TT2B */
 656         t = cho(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1));
 657     } else {
 658         qemu_build_not_reached();
 659     }
 660
 661     t += CR_ST_WORD(d, 0) + CR_ST_WORD(m, imm2);
 662
 663     CR_ST_WORD(d, 0) = CR_ST_WORD(d, 1);
 664
 665     if (opcode < 2) {
 666         /* SM3TT1A, SM3TT1B */
 667         t += CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(d, 3), 20);
 668
 669         CR_ST_WORD(d, 1) = ror32(CR_ST_WORD(d, 2), 23);
 670     } else {
 671         /* SM3TT2A, SM3TT2B */
 672         t += CR_ST_WORD(n, 3);
 673         t ^= rol32(t, 9) ^ rol32(t, 17);
 674
 675         CR_ST_WORD(d, 1) = ror32(CR_ST_WORD(d, 2), 13);
 676     }
 677
 678     CR_ST_WORD(d, 2) = CR_ST_WORD(d, 3);
 679     CR_ST_WORD(d, 3) = t;
 680
 681     rd[0] = d.l[0];
 682     rd[1] = d.l[1];
 683
 684     clear_tail_16(rd, desc);
 685 }
 686
 687 #define DO_SM3TT(NAME, OPCODE) \
 688     void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
 689     { crypto_sm3tt(vd, vn, vm, desc, OPCODE); }
 690
 691 DO_SM3TT(crypto_sm3tt1a, 0)
 692 DO_SM3TT(crypto_sm3tt1b, 1)
 693 DO_SM3TT(crypto_sm3tt2a, 2)
 694 DO_SM3TT(crypto_sm3tt2b, 3)
 695
 696 #undef DO_SM3TT
 697
 698 static void do_crypto_sm4e(uint64_t *rd, uint64_t *rn, uint64_t *rm)
 699 {
 700     union CRYPTO_STATE d = { .l = { rn[0], rn[1] } };
 701     union CRYPTO_STATE n = { .l = { rm[0], rm[1] } };
 702     uint32_t t, i;
 703
 704     for (i = 0; i < 4; i++) {
 705         t = CR_ST_WORD(d, (i + 1) % 4) ^
 706             CR_ST_WORD(d, (i + 2) % 4) ^
 707             CR_ST_WORD(d, (i + 3) % 4) ^
 708             CR_ST_WORD(n, i);
 709
 710         t = sm4_sbox[t & 0xff] |
 711             sm4_sbox[(t >> 8) & 0xff] << 8 |
 712             sm4_sbox[(t >> 16) & 0xff] << 16 |
 713             sm4_sbox[(t >> 24) & 0xff] << 24;
 714
 715         CR_ST_WORD(d, i) ^= t ^ rol32(t, 2) ^ rol32(t, 10) ^ rol32(t, 18) ^
 716                             rol32(t, 24);
 717     }
 718
 719     rd[0] = d.l[0];
 720     rd[1] = d.l[1];
 721 }
 722
 723 void HELPER(crypto_sm4e)(void *vd, void *vn, void *vm, uint32_t desc)
 724 {
 725     intptr_t i, opr_sz = simd_oprsz(desc);
 726
 727     for (i = 0; i < opr_sz; i += 16) {
 728         do_crypto_sm4e(vd + i, vn + i, vm + i);
 729     }
 730     clear_tail(vd, opr_sz, simd_maxsz(desc));
 731 }
 732
 733 static void do_crypto_sm4ekey(uint64_t *rd, uint64_t *rn, uint64_t *rm)
 734 {
 735     union CRYPTO_STATE d;
 736     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
 737     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
 738     uint32_t t, i;
 739
 740     d = n;
 741     for (i = 0; i < 4; i++) {
 742         t = CR_ST_WORD(d, (i + 1) % 4) ^
 743             CR_ST_WORD(d, (i + 2) % 4) ^
 744             CR_ST_WORD(d, (i + 3) % 4) ^
 745             CR_ST_WORD(m, i);
 746
 747         t = sm4_sbox[t & 0xff] |
 748             sm4_sbox[(t >> 8) & 0xff] << 8 |
 749             sm4_sbox[(t >> 16) & 0xff] << 16 |
 750             sm4_sbox[(t >> 24) & 0xff] << 24;
 751
 752         CR_ST_WORD(d, i) ^= t ^ rol32(t, 13) ^ rol32(t, 23);
 753     }
 754
 755     rd[0] = d.l[0];
 756     rd[1] = d.l[1];
 757 }
 758
 759 void HELPER(crypto_sm4ekey)(void *vd, void *vn, void* vm, uint32_t desc)
 760 {
 761     intptr_t i, opr_sz = simd_oprsz(desc);
 762
 763     for (i = 0; i < opr_sz; i += 16) {
 764         do_crypto_sm4ekey(vd + i, vn + i, vm + i);
 765     }
 766     clear_tail(vd, opr_sz, simd_maxsz(desc));
 767 }
 768
 769 void HELPER(crypto_rax1)(void *vd, void *vn, void *vm, uint32_t desc)
 770 {
 771     intptr_t i, opr_sz = simd_oprsz(desc);
 772     uint64_t *d = vd, *n = vn, *m = vm;
 773
 774     for (i = 0; i < opr_sz / 8; ++i) {
 775         d[i] = n[i] ^ rol64(m[i], 1);
 776     }
 777     clear_tail(vd, opr_sz, simd_maxsz(desc));
 778 }