2 * RISC-V Vector Crypto Extension Helpers for QEMU.
4 * Copyright (C) 2023 SiFive, Inc.
5 * Written by Codethink Ltd and SiFive.
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2 or later, as published by the Free Software Foundation.
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * You should have received a copy of the GNU General Public License along with
17 * this program. If not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
21 #include "qemu/host-utils.h"
22 #include "qemu/bitops.h"
23 #include "qemu/bswap.h"
25 #include "crypto/aes.h"
26 #include "crypto/aes-round.h"
27 #include "crypto/sm4.h"
28 #include "exec/memop.h"
29 #include "exec/exec-all.h"
30 #include "exec/helper-proto.h"
31 #include "internals.h"
32 #include "vector_internals.h"
34 static uint64_t clmul64(uint64_t y
, uint64_t x
)
37 for (int j
= 63; j
>= 0; j
--) {
45 static uint64_t clmulh64(uint64_t y
, uint64_t x
)
48 for (int j
= 63; j
>= 1; j
--) {
50 result
^= (x
>> (64 - j
));
56 RVVCALL(OPIVV2
, vclmul_vv
, OP_UUU_D
, H8
, H8
, H8
, clmul64
)
57 GEN_VEXT_VV(vclmul_vv
, 8)
58 RVVCALL(OPIVX2
, vclmul_vx
, OP_UUU_D
, H8
, H8
, clmul64
)
59 GEN_VEXT_VX(vclmul_vx
, 8)
60 RVVCALL(OPIVV2
, vclmulh_vv
, OP_UUU_D
, H8
, H8
, H8
, clmulh64
)
61 GEN_VEXT_VV(vclmulh_vv
, 8)
62 RVVCALL(OPIVX2
, vclmulh_vx
, OP_UUU_D
, H8
, H8
, clmulh64
)
63 GEN_VEXT_VX(vclmulh_vx
, 8)
65 RVVCALL(OPIVV2
, vror_vv_b
, OP_UUU_B
, H1
, H1
, H1
, ror8
)
66 RVVCALL(OPIVV2
, vror_vv_h
, OP_UUU_H
, H2
, H2
, H2
, ror16
)
67 RVVCALL(OPIVV2
, vror_vv_w
, OP_UUU_W
, H4
, H4
, H4
, ror32
)
68 RVVCALL(OPIVV2
, vror_vv_d
, OP_UUU_D
, H8
, H8
, H8
, ror64
)
69 GEN_VEXT_VV(vror_vv_b
, 1)
70 GEN_VEXT_VV(vror_vv_h
, 2)
71 GEN_VEXT_VV(vror_vv_w
, 4)
72 GEN_VEXT_VV(vror_vv_d
, 8)
74 RVVCALL(OPIVX2
, vror_vx_b
, OP_UUU_B
, H1
, H1
, ror8
)
75 RVVCALL(OPIVX2
, vror_vx_h
, OP_UUU_H
, H2
, H2
, ror16
)
76 RVVCALL(OPIVX2
, vror_vx_w
, OP_UUU_W
, H4
, H4
, ror32
)
77 RVVCALL(OPIVX2
, vror_vx_d
, OP_UUU_D
, H8
, H8
, ror64
)
78 GEN_VEXT_VX(vror_vx_b
, 1)
79 GEN_VEXT_VX(vror_vx_h
, 2)
80 GEN_VEXT_VX(vror_vx_w
, 4)
81 GEN_VEXT_VX(vror_vx_d
, 8)
83 RVVCALL(OPIVV2
, vrol_vv_b
, OP_UUU_B
, H1
, H1
, H1
, rol8
)
84 RVVCALL(OPIVV2
, vrol_vv_h
, OP_UUU_H
, H2
, H2
, H2
, rol16
)
85 RVVCALL(OPIVV2
, vrol_vv_w
, OP_UUU_W
, H4
, H4
, H4
, rol32
)
86 RVVCALL(OPIVV2
, vrol_vv_d
, OP_UUU_D
, H8
, H8
, H8
, rol64
)
87 GEN_VEXT_VV(vrol_vv_b
, 1)
88 GEN_VEXT_VV(vrol_vv_h
, 2)
89 GEN_VEXT_VV(vrol_vv_w
, 4)
90 GEN_VEXT_VV(vrol_vv_d
, 8)
92 RVVCALL(OPIVX2
, vrol_vx_b
, OP_UUU_B
, H1
, H1
, rol8
)
93 RVVCALL(OPIVX2
, vrol_vx_h
, OP_UUU_H
, H2
, H2
, rol16
)
94 RVVCALL(OPIVX2
, vrol_vx_w
, OP_UUU_W
, H4
, H4
, rol32
)
95 RVVCALL(OPIVX2
, vrol_vx_d
, OP_UUU_D
, H8
, H8
, rol64
)
96 GEN_VEXT_VX(vrol_vx_b
, 1)
97 GEN_VEXT_VX(vrol_vx_h
, 2)
98 GEN_VEXT_VX(vrol_vx_w
, 4)
99 GEN_VEXT_VX(vrol_vx_d
, 8)
101 static uint64_t brev8(uint64_t val
)
103 val
= ((val
& 0x5555555555555555ull
) << 1) |
104 ((val
& 0xAAAAAAAAAAAAAAAAull
) >> 1);
105 val
= ((val
& 0x3333333333333333ull
) << 2) |
106 ((val
& 0xCCCCCCCCCCCCCCCCull
) >> 2);
107 val
= ((val
& 0x0F0F0F0F0F0F0F0Full
) << 4) |
108 ((val
& 0xF0F0F0F0F0F0F0F0ull
) >> 4);
113 RVVCALL(OPIVV1
, vbrev8_v_b
, OP_UU_B
, H1
, H1
, brev8
)
114 RVVCALL(OPIVV1
, vbrev8_v_h
, OP_UU_H
, H2
, H2
, brev8
)
115 RVVCALL(OPIVV1
, vbrev8_v_w
, OP_UU_W
, H4
, H4
, brev8
)
116 RVVCALL(OPIVV1
, vbrev8_v_d
, OP_UU_D
, H8
, H8
, brev8
)
117 GEN_VEXT_V(vbrev8_v_b
, 1)
118 GEN_VEXT_V(vbrev8_v_h
, 2)
119 GEN_VEXT_V(vbrev8_v_w
, 4)
120 GEN_VEXT_V(vbrev8_v_d
, 8)
122 #define DO_IDENTITY(a) (a)
123 RVVCALL(OPIVV1
, vrev8_v_b
, OP_UU_B
, H1
, H1
, DO_IDENTITY
)
124 RVVCALL(OPIVV1
, vrev8_v_h
, OP_UU_H
, H2
, H2
, bswap16
)
125 RVVCALL(OPIVV1
, vrev8_v_w
, OP_UU_W
, H4
, H4
, bswap32
)
126 RVVCALL(OPIVV1
, vrev8_v_d
, OP_UU_D
, H8
, H8
, bswap64
)
127 GEN_VEXT_V(vrev8_v_b
, 1)
128 GEN_VEXT_V(vrev8_v_h
, 2)
129 GEN_VEXT_V(vrev8_v_w
, 4)
130 GEN_VEXT_V(vrev8_v_d
, 8)
132 #define DO_ANDN(a, b) ((a) & ~(b))
133 RVVCALL(OPIVV2
, vandn_vv_b
, OP_UUU_B
, H1
, H1
, H1
, DO_ANDN
)
134 RVVCALL(OPIVV2
, vandn_vv_h
, OP_UUU_H
, H2
, H2
, H2
, DO_ANDN
)
135 RVVCALL(OPIVV2
, vandn_vv_w
, OP_UUU_W
, H4
, H4
, H4
, DO_ANDN
)
136 RVVCALL(OPIVV2
, vandn_vv_d
, OP_UUU_D
, H8
, H8
, H8
, DO_ANDN
)
137 GEN_VEXT_VV(vandn_vv_b
, 1)
138 GEN_VEXT_VV(vandn_vv_h
, 2)
139 GEN_VEXT_VV(vandn_vv_w
, 4)
140 GEN_VEXT_VV(vandn_vv_d
, 8)
142 RVVCALL(OPIVX2
, vandn_vx_b
, OP_UUU_B
, H1
, H1
, DO_ANDN
)
143 RVVCALL(OPIVX2
, vandn_vx_h
, OP_UUU_H
, H2
, H2
, DO_ANDN
)
144 RVVCALL(OPIVX2
, vandn_vx_w
, OP_UUU_W
, H4
, H4
, DO_ANDN
)
145 RVVCALL(OPIVX2
, vandn_vx_d
, OP_UUU_D
, H8
, H8
, DO_ANDN
)
146 GEN_VEXT_VX(vandn_vx_b
, 1)
147 GEN_VEXT_VX(vandn_vx_h
, 2)
148 GEN_VEXT_VX(vandn_vx_w
, 4)
149 GEN_VEXT_VX(vandn_vx_d
, 8)
151 RVVCALL(OPIVV1
, vbrev_v_b
, OP_UU_B
, H1
, H1
, revbit8
)
152 RVVCALL(OPIVV1
, vbrev_v_h
, OP_UU_H
, H2
, H2
, revbit16
)
153 RVVCALL(OPIVV1
, vbrev_v_w
, OP_UU_W
, H4
, H4
, revbit32
)
154 RVVCALL(OPIVV1
, vbrev_v_d
, OP_UU_D
, H8
, H8
, revbit64
)
155 GEN_VEXT_V(vbrev_v_b
, 1)
156 GEN_VEXT_V(vbrev_v_h
, 2)
157 GEN_VEXT_V(vbrev_v_w
, 4)
158 GEN_VEXT_V(vbrev_v_d
, 8)
160 RVVCALL(OPIVV1
, vclz_v_b
, OP_UU_B
, H1
, H1
, clz8
)
161 RVVCALL(OPIVV1
, vclz_v_h
, OP_UU_H
, H2
, H2
, clz16
)
162 RVVCALL(OPIVV1
, vclz_v_w
, OP_UU_W
, H4
, H4
, clz32
)
163 RVVCALL(OPIVV1
, vclz_v_d
, OP_UU_D
, H8
, H8
, clz64
)
164 GEN_VEXT_V(vclz_v_b
, 1)
165 GEN_VEXT_V(vclz_v_h
, 2)
166 GEN_VEXT_V(vclz_v_w
, 4)
167 GEN_VEXT_V(vclz_v_d
, 8)
169 RVVCALL(OPIVV1
, vctz_v_b
, OP_UU_B
, H1
, H1
, ctz8
)
170 RVVCALL(OPIVV1
, vctz_v_h
, OP_UU_H
, H2
, H2
, ctz16
)
171 RVVCALL(OPIVV1
, vctz_v_w
, OP_UU_W
, H4
, H4
, ctz32
)
172 RVVCALL(OPIVV1
, vctz_v_d
, OP_UU_D
, H8
, H8
, ctz64
)
173 GEN_VEXT_V(vctz_v_b
, 1)
174 GEN_VEXT_V(vctz_v_h
, 2)
175 GEN_VEXT_V(vctz_v_w
, 4)
176 GEN_VEXT_V(vctz_v_d
, 8)
178 RVVCALL(OPIVV1
, vcpop_v_b
, OP_UU_B
, H1
, H1
, ctpop8
)
179 RVVCALL(OPIVV1
, vcpop_v_h
, OP_UU_H
, H2
, H2
, ctpop16
)
180 RVVCALL(OPIVV1
, vcpop_v_w
, OP_UU_W
, H4
, H4
, ctpop32
)
181 RVVCALL(OPIVV1
, vcpop_v_d
, OP_UU_D
, H8
, H8
, ctpop64
)
182 GEN_VEXT_V(vcpop_v_b
, 1)
183 GEN_VEXT_V(vcpop_v_h
, 2)
184 GEN_VEXT_V(vcpop_v_w
, 4)
185 GEN_VEXT_V(vcpop_v_d
, 8)
187 #define DO_SLL(N, M) (N << (M & (sizeof(N) * 8 - 1)))
188 RVVCALL(OPIVV2
, vwsll_vv_b
, WOP_UUU_B
, H2
, H1
, H1
, DO_SLL
)
189 RVVCALL(OPIVV2
, vwsll_vv_h
, WOP_UUU_H
, H4
, H2
, H2
, DO_SLL
)
190 RVVCALL(OPIVV2
, vwsll_vv_w
, WOP_UUU_W
, H8
, H4
, H4
, DO_SLL
)
191 GEN_VEXT_VV(vwsll_vv_b
, 2)
192 GEN_VEXT_VV(vwsll_vv_h
, 4)
193 GEN_VEXT_VV(vwsll_vv_w
, 8)
195 RVVCALL(OPIVX2
, vwsll_vx_b
, WOP_UUU_B
, H2
, H1
, DO_SLL
)
196 RVVCALL(OPIVX2
, vwsll_vx_h
, WOP_UUU_H
, H4
, H2
, DO_SLL
)
197 RVVCALL(OPIVX2
, vwsll_vx_w
, WOP_UUU_W
, H8
, H4
, DO_SLL
)
198 GEN_VEXT_VX(vwsll_vx_b
, 2)
199 GEN_VEXT_VX(vwsll_vx_h
, 4)
200 GEN_VEXT_VX(vwsll_vx_w
, 8)
202 void HELPER(egs_check
)(uint32_t egs
, CPURISCVState
*env
)
204 uint32_t vl
= env
->vl
;
205 uint32_t vstart
= env
->vstart
;
207 if (vl
% egs
!= 0 || vstart
% egs
!= 0) {
208 riscv_raise_exception(env
, RISCV_EXCP_ILLEGAL_INST
, GETPC());
212 static inline void xor_round_key(AESState
*round_state
, AESState
*round_key
)
214 round_state
->v
= round_state
->v
^ round_key
->v
;
217 #define GEN_ZVKNED_HELPER_VV(NAME, ...) \
218 void HELPER(NAME)(void *vd, void *vs2, CPURISCVState *env, \
221 uint32_t vl = env->vl; \
222 uint32_t total_elems = vext_get_total_elems(env, desc, 4); \
223 uint32_t vta = vext_vta(desc); \
225 for (uint32_t i = env->vstart / 4; i < env->vl / 4; i++) { \
226 AESState round_key; \
227 round_key.d[0] = *((uint64_t *)vs2 + H8(i * 2 + 0)); \
228 round_key.d[1] = *((uint64_t *)vs2 + H8(i * 2 + 1)); \
229 AESState round_state; \
230 round_state.d[0] = *((uint64_t *)vd + H8(i * 2 + 0)); \
231 round_state.d[1] = *((uint64_t *)vd + H8(i * 2 + 1)); \
233 *((uint64_t *)vd + H8(i * 2 + 0)) = round_state.d[0]; \
234 *((uint64_t *)vd + H8(i * 2 + 1)) = round_state.d[1]; \
237 /* set tail elements to 1s */ \
238 vext_set_elems_1s(vd, vta, vl * 4, total_elems * 4); \
241 #define GEN_ZVKNED_HELPER_VS(NAME, ...) \
242 void HELPER(NAME)(void *vd, void *vs2, CPURISCVState *env, \
245 uint32_t vl = env->vl; \
246 uint32_t total_elems = vext_get_total_elems(env, desc, 4); \
247 uint32_t vta = vext_vta(desc); \
249 for (uint32_t i = env->vstart / 4; i < env->vl / 4; i++) { \
250 AESState round_key; \
251 round_key.d[0] = *((uint64_t *)vs2 + H8(0)); \
252 round_key.d[1] = *((uint64_t *)vs2 + H8(1)); \
253 AESState round_state; \
254 round_state.d[0] = *((uint64_t *)vd + H8(i * 2 + 0)); \
255 round_state.d[1] = *((uint64_t *)vd + H8(i * 2 + 1)); \
257 *((uint64_t *)vd + H8(i * 2 + 0)) = round_state.d[0]; \
258 *((uint64_t *)vd + H8(i * 2 + 1)) = round_state.d[1]; \
261 /* set tail elements to 1s */ \
262 vext_set_elems_1s(vd, vta, vl * 4, total_elems * 4); \
265 GEN_ZVKNED_HELPER_VV(vaesef_vv
, aesenc_SB_SR_AK(&round_state
,
269 GEN_ZVKNED_HELPER_VS(vaesef_vs
, aesenc_SB_SR_AK(&round_state
,
273 GEN_ZVKNED_HELPER_VV(vaesdf_vv
, aesdec_ISB_ISR_AK(&round_state
,
277 GEN_ZVKNED_HELPER_VS(vaesdf_vs
, aesdec_ISB_ISR_AK(&round_state
,
281 GEN_ZVKNED_HELPER_VV(vaesem_vv
, aesenc_SB_SR_MC_AK(&round_state
,
285 GEN_ZVKNED_HELPER_VS(vaesem_vs
, aesenc_SB_SR_MC_AK(&round_state
,
289 GEN_ZVKNED_HELPER_VV(vaesdm_vv
, aesdec_ISB_ISR_AK_IMC(&round_state
,
293 GEN_ZVKNED_HELPER_VS(vaesdm_vs
, aesdec_ISB_ISR_AK_IMC(&round_state
,
297 GEN_ZVKNED_HELPER_VS(vaesz_vs
, xor_round_key(&round_state
, &round_key
);)
299 void HELPER(vaeskf1_vi
)(void *vd_vptr
, void *vs2_vptr
, uint32_t uimm
,
300 CPURISCVState
*env
, uint32_t desc
)
302 uint32_t *vd
= vd_vptr
;
303 uint32_t *vs2
= vs2_vptr
;
304 uint32_t vl
= env
->vl
;
305 uint32_t total_elems
= vext_get_total_elems(env
, desc
, 4);
306 uint32_t vta
= vext_vta(desc
);
309 if (uimm
> 10 || uimm
== 0) {
313 for (uint32_t i
= env
->vstart
/ 4; i
< env
->vl
/ 4; i
++) {
315 static const uint32_t rcon
[] = {
316 0x00000001, 0x00000002, 0x00000004, 0x00000008, 0x00000010,
317 0x00000020, 0x00000040, 0x00000080, 0x0000001B, 0x00000036,
320 rk
[0] = vs2
[i
* 4 + H4(0)];
321 rk
[1] = vs2
[i
* 4 + H4(1)];
322 rk
[2] = vs2
[i
* 4 + H4(2)];
323 rk
[3] = vs2
[i
* 4 + H4(3)];
324 tmp
= ror32(rk
[3], 8);
326 rk
[4] = rk
[0] ^ (((uint32_t)AES_sbox
[(tmp
>> 24) & 0xff] << 24) |
327 ((uint32_t)AES_sbox
[(tmp
>> 16) & 0xff] << 16) |
328 ((uint32_t)AES_sbox
[(tmp
>> 8) & 0xff] << 8) |
329 ((uint32_t)AES_sbox
[(tmp
>> 0) & 0xff] << 0))
331 rk
[5] = rk
[1] ^ rk
[4];
332 rk
[6] = rk
[2] ^ rk
[5];
333 rk
[7] = rk
[3] ^ rk
[6];
335 vd
[i
* 4 + H4(0)] = rk
[4];
336 vd
[i
* 4 + H4(1)] = rk
[5];
337 vd
[i
* 4 + H4(2)] = rk
[6];
338 vd
[i
* 4 + H4(3)] = rk
[7];
341 /* set tail elements to 1s */
342 vext_set_elems_1s(vd
, vta
, vl
* 4, total_elems
* 4);
345 void HELPER(vaeskf2_vi
)(void *vd_vptr
, void *vs2_vptr
, uint32_t uimm
,
346 CPURISCVState
*env
, uint32_t desc
)
348 uint32_t *vd
= vd_vptr
;
349 uint32_t *vs2
= vs2_vptr
;
350 uint32_t vl
= env
->vl
;
351 uint32_t total_elems
= vext_get_total_elems(env
, desc
, 4);
352 uint32_t vta
= vext_vta(desc
);
355 if (uimm
> 14 || uimm
< 2) {
359 for (uint32_t i
= env
->vstart
/ 4; i
< env
->vl
/ 4; i
++) {
360 uint32_t rk
[12], tmp
;
361 static const uint32_t rcon
[] = {
362 0x00000001, 0x00000002, 0x00000004, 0x00000008, 0x00000010,
363 0x00000020, 0x00000040, 0x00000080, 0x0000001B, 0x00000036,
366 rk
[0] = vd
[i
* 4 + H4(0)];
367 rk
[1] = vd
[i
* 4 + H4(1)];
368 rk
[2] = vd
[i
* 4 + H4(2)];
369 rk
[3] = vd
[i
* 4 + H4(3)];
370 rk
[4] = vs2
[i
* 4 + H4(0)];
371 rk
[5] = vs2
[i
* 4 + H4(1)];
372 rk
[6] = vs2
[i
* 4 + H4(2)];
373 rk
[7] = vs2
[i
* 4 + H4(3)];
376 tmp
= ror32(rk
[7], 8);
377 rk
[8] = rk
[0] ^ (((uint32_t)AES_sbox
[(tmp
>> 24) & 0xff] << 24) |
378 ((uint32_t)AES_sbox
[(tmp
>> 16) & 0xff] << 16) |
379 ((uint32_t)AES_sbox
[(tmp
>> 8) & 0xff] << 8) |
380 ((uint32_t)AES_sbox
[(tmp
>> 0) & 0xff] << 0))
381 ^ rcon
[(uimm
- 1) / 2];
383 rk
[8] = rk
[0] ^ (((uint32_t)AES_sbox
[(rk
[7] >> 24) & 0xff] << 24) |
384 ((uint32_t)AES_sbox
[(rk
[7] >> 16) & 0xff] << 16) |
385 ((uint32_t)AES_sbox
[(rk
[7] >> 8) & 0xff] << 8) |
386 ((uint32_t)AES_sbox
[(rk
[7] >> 0) & 0xff] << 0));
388 rk
[9] = rk
[1] ^ rk
[8];
389 rk
[10] = rk
[2] ^ rk
[9];
390 rk
[11] = rk
[3] ^ rk
[10];
392 vd
[i
* 4 + H4(0)] = rk
[8];
393 vd
[i
* 4 + H4(1)] = rk
[9];
394 vd
[i
* 4 + H4(2)] = rk
[10];
395 vd
[i
* 4 + H4(3)] = rk
[11];
398 /* set tail elements to 1s */
399 vext_set_elems_1s(vd
, vta
, vl
* 4, total_elems
* 4);
402 static inline uint32_t sig0_sha256(uint32_t x
)
404 return ror32(x
, 7) ^ ror32(x
, 18) ^ (x
>> 3);
407 static inline uint32_t sig1_sha256(uint32_t x
)
409 return ror32(x
, 17) ^ ror32(x
, 19) ^ (x
>> 10);
412 static inline uint64_t sig0_sha512(uint64_t x
)
414 return ror64(x
, 1) ^ ror64(x
, 8) ^ (x
>> 7);
417 static inline uint64_t sig1_sha512(uint64_t x
)
419 return ror64(x
, 19) ^ ror64(x
, 61) ^ (x
>> 6);
422 static inline void vsha2ms_e32(uint32_t *vd
, uint32_t *vs1
, uint32_t *vs2
)
425 res
[0] = sig1_sha256(vs1
[H4(2)]) + vs2
[H4(1)] + sig0_sha256(vd
[H4(1)]) +
427 res
[1] = sig1_sha256(vs1
[H4(3)]) + vs2
[H4(2)] + sig0_sha256(vd
[H4(2)]) +
430 sig1_sha256(res
[0]) + vs2
[H4(3)] + sig0_sha256(vd
[H4(3)]) + vd
[H4(2)];
432 sig1_sha256(res
[1]) + vs1
[H4(0)] + sig0_sha256(vs2
[H4(0)]) + vd
[H4(3)];
439 static inline void vsha2ms_e64(uint64_t *vd
, uint64_t *vs1
, uint64_t *vs2
)
442 res
[0] = sig1_sha512(vs1
[2]) + vs2
[1] + sig0_sha512(vd
[1]) + vd
[0];
443 res
[1] = sig1_sha512(vs1
[3]) + vs2
[2] + sig0_sha512(vd
[2]) + vd
[1];
444 res
[2] = sig1_sha512(res
[0]) + vs2
[3] + sig0_sha512(vd
[3]) + vd
[2];
445 res
[3] = sig1_sha512(res
[1]) + vs1
[0] + sig0_sha512(vs2
[0]) + vd
[3];
452 void HELPER(vsha2ms_vv
)(void *vd
, void *vs1
, void *vs2
, CPURISCVState
*env
,
455 uint32_t sew
= FIELD_EX64(env
->vtype
, VTYPE
, VSEW
);
456 uint32_t esz
= sew
== MO_32
? 4 : 8;
457 uint32_t total_elems
;
458 uint32_t vta
= vext_vta(desc
);
460 for (uint32_t i
= env
->vstart
/ 4; i
< env
->vl
/ 4; i
++) {
462 vsha2ms_e32(((uint32_t *)vd
) + i
* 4, ((uint32_t *)vs1
) + i
* 4,
463 ((uint32_t *)vs2
) + i
* 4);
465 /* If not 32 then SEW should be 64 */
466 vsha2ms_e64(((uint64_t *)vd
) + i
* 4, ((uint64_t *)vs1
) + i
* 4,
467 ((uint64_t *)vs2
) + i
* 4);
470 /* set tail elements to 1s */
471 total_elems
= vext_get_total_elems(env
, desc
, esz
);
472 vext_set_elems_1s(vd
, vta
, env
->vl
* esz
, total_elems
* esz
);
476 static inline uint64_t sum0_64(uint64_t x
)
478 return ror64(x
, 28) ^ ror64(x
, 34) ^ ror64(x
, 39);
481 static inline uint32_t sum0_32(uint32_t x
)
483 return ror32(x
, 2) ^ ror32(x
, 13) ^ ror32(x
, 22);
486 static inline uint64_t sum1_64(uint64_t x
)
488 return ror64(x
, 14) ^ ror64(x
, 18) ^ ror64(x
, 41);
491 static inline uint32_t sum1_32(uint32_t x
)
493 return ror32(x
, 6) ^ ror32(x
, 11) ^ ror32(x
, 25);
496 #define ch(x, y, z) ((x & y) ^ ((~x) & z))
498 #define maj(x, y, z) ((x & y) ^ (x & z) ^ (y & z))
500 static void vsha2c_64(uint64_t *vs2
, uint64_t *vd
, uint64_t *vs1
)
502 uint64_t a
= vs2
[3], b
= vs2
[2], e
= vs2
[1], f
= vs2
[0];
503 uint64_t c
= vd
[3], d
= vd
[2], g
= vd
[1], h
= vd
[0];
504 uint64_t W0
= vs1
[0], W1
= vs1
[1];
505 uint64_t T1
= h
+ sum1_64(e
) + ch(e
, f
, g
) + W0
;
506 uint64_t T2
= sum0_64(a
) + maj(a
, b
, c
);
517 T1
= h
+ sum1_64(e
) + ch(e
, f
, g
) + W1
;
518 T2
= sum0_64(a
) + maj(a
, b
, c
);
534 static void vsha2c_32(uint32_t *vs2
, uint32_t *vd
, uint32_t *vs1
)
536 uint32_t a
= vs2
[H4(3)], b
= vs2
[H4(2)], e
= vs2
[H4(1)], f
= vs2
[H4(0)];
537 uint32_t c
= vd
[H4(3)], d
= vd
[H4(2)], g
= vd
[H4(1)], h
= vd
[H4(0)];
538 uint32_t W0
= vs1
[H4(0)], W1
= vs1
[H4(1)];
539 uint32_t T1
= h
+ sum1_32(e
) + ch(e
, f
, g
) + W0
;
540 uint32_t T2
= sum0_32(a
) + maj(a
, b
, c
);
551 T1
= h
+ sum1_32(e
) + ch(e
, f
, g
) + W1
;
552 T2
= sum0_32(a
) + maj(a
, b
, c
);
568 void HELPER(vsha2ch32_vv
)(void *vd
, void *vs1
, void *vs2
, CPURISCVState
*env
,
571 const uint32_t esz
= 4;
572 uint32_t total_elems
;
573 uint32_t vta
= vext_vta(desc
);
575 for (uint32_t i
= env
->vstart
/ 4; i
< env
->vl
/ 4; i
++) {
576 vsha2c_32(((uint32_t *)vs2
) + 4 * i
, ((uint32_t *)vd
) + 4 * i
,
577 ((uint32_t *)vs1
) + 4 * i
+ 2);
580 /* set tail elements to 1s */
581 total_elems
= vext_get_total_elems(env
, desc
, esz
);
582 vext_set_elems_1s(vd
, vta
, env
->vl
* esz
, total_elems
* esz
);
586 void HELPER(vsha2ch64_vv
)(void *vd
, void *vs1
, void *vs2
, CPURISCVState
*env
,
589 const uint32_t esz
= 8;
590 uint32_t total_elems
;
591 uint32_t vta
= vext_vta(desc
);
593 for (uint32_t i
= env
->vstart
/ 4; i
< env
->vl
/ 4; i
++) {
594 vsha2c_64(((uint64_t *)vs2
) + 4 * i
, ((uint64_t *)vd
) + 4 * i
,
595 ((uint64_t *)vs1
) + 4 * i
+ 2);
598 /* set tail elements to 1s */
599 total_elems
= vext_get_total_elems(env
, desc
, esz
);
600 vext_set_elems_1s(vd
, vta
, env
->vl
* esz
, total_elems
* esz
);
604 void HELPER(vsha2cl32_vv
)(void *vd
, void *vs1
, void *vs2
, CPURISCVState
*env
,
607 const uint32_t esz
= 4;
608 uint32_t total_elems
;
609 uint32_t vta
= vext_vta(desc
);
611 for (uint32_t i
= env
->vstart
/ 4; i
< env
->vl
/ 4; i
++) {
612 vsha2c_32(((uint32_t *)vs2
) + 4 * i
, ((uint32_t *)vd
) + 4 * i
,
613 (((uint32_t *)vs1
) + 4 * i
));
616 /* set tail elements to 1s */
617 total_elems
= vext_get_total_elems(env
, desc
, esz
);
618 vext_set_elems_1s(vd
, vta
, env
->vl
* esz
, total_elems
* esz
);
622 void HELPER(vsha2cl64_vv
)(void *vd
, void *vs1
, void *vs2
, CPURISCVState
*env
,
626 uint32_t total_elems
;
627 uint32_t vta
= vext_vta(desc
);
629 for (uint32_t i
= env
->vstart
/ 4; i
< env
->vl
/ 4; i
++) {
630 vsha2c_64(((uint64_t *)vs2
) + 4 * i
, ((uint64_t *)vd
) + 4 * i
,
631 (((uint64_t *)vs1
) + 4 * i
));
634 /* set tail elements to 1s */
635 total_elems
= vext_get_total_elems(env
, desc
, esz
);
636 vext_set_elems_1s(vd
, vta
, env
->vl
* esz
, total_elems
* esz
);
640 static inline uint32_t p1(uint32_t x
)
642 return x
^ rol32(x
, 15) ^ rol32(x
, 23);
645 static inline uint32_t zvksh_w(uint32_t m16
, uint32_t m9
, uint32_t m3
,
646 uint32_t m13
, uint32_t m6
)
648 return p1(m16
^ m9
^ rol32(m3
, 15)) ^ rol32(m13
, 7) ^ m6
;
651 void HELPER(vsm3me_vv
)(void *vd_vptr
, void *vs1_vptr
, void *vs2_vptr
,
652 CPURISCVState
*env
, uint32_t desc
)
654 uint32_t esz
= memop_size(FIELD_EX64(env
->vtype
, VTYPE
, VSEW
));
655 uint32_t total_elems
= vext_get_total_elems(env
, desc
, esz
);
656 uint32_t vta
= vext_vta(desc
);
657 uint32_t *vd
= vd_vptr
;
658 uint32_t *vs1
= vs1_vptr
;
659 uint32_t *vs2
= vs2_vptr
;
661 for (int i
= env
->vstart
/ 8; i
< env
->vl
/ 8; i
++) {
663 for (int j
= 0; j
< 8; j
++) {
664 w
[j
] = bswap32(vs1
[H4((i
* 8) + j
)]);
665 w
[j
+ 8] = bswap32(vs2
[H4((i
* 8) + j
)]);
667 for (int j
= 0; j
< 8; j
++) {
669 zvksh_w(w
[j
], w
[j
+ 7], w
[j
+ 13], w
[j
+ 3], w
[j
+ 10]);
671 for (int j
= 0; j
< 8; j
++) {
672 vd
[(i
* 8) + j
] = bswap32(w
[H4(j
+ 16)]);
675 vext_set_elems_1s(vd_vptr
, vta
, env
->vl
* esz
, total_elems
* esz
);
679 static inline uint32_t ff1(uint32_t x
, uint32_t y
, uint32_t z
)
684 static inline uint32_t ff2(uint32_t x
, uint32_t y
, uint32_t z
)
686 return (x
& y
) | (x
& z
) | (y
& z
);
689 static inline uint32_t ff_j(uint32_t x
, uint32_t y
, uint32_t z
, uint32_t j
)
691 return (j
<= 15) ? ff1(x
, y
, z
) : ff2(x
, y
, z
);
694 static inline uint32_t gg1(uint32_t x
, uint32_t y
, uint32_t z
)
699 static inline uint32_t gg2(uint32_t x
, uint32_t y
, uint32_t z
)
701 return (x
& y
) | (~x
& z
);
704 static inline uint32_t gg_j(uint32_t x
, uint32_t y
, uint32_t z
, uint32_t j
)
706 return (j
<= 15) ? gg1(x
, y
, z
) : gg2(x
, y
, z
);
709 static inline uint32_t t_j(uint32_t j
)
711 return (j
<= 15) ? 0x79cc4519 : 0x7a879d8a;
714 static inline uint32_t p_0(uint32_t x
)
716 return x
^ rol32(x
, 9) ^ rol32(x
, 17);
719 static void sm3c(uint32_t *vd
, uint32_t *vs1
, uint32_t *vs2
, uint32_t uimm
)
723 uint32_t ss1
, ss2
, tt1
, tt2
;
724 x0
= vs2
[0] ^ vs2
[4];
725 x1
= vs2
[1] ^ vs2
[5];
727 ss1
= rol32(rol32(vs1
[0], 12) + vs1
[4] + rol32(t_j(j
), j
% 32), 7);
728 ss2
= ss1
^ rol32(vs1
[0], 12);
729 tt1
= ff_j(vs1
[0], vs1
[1], vs1
[2], j
) + vs1
[3] + ss2
+ x0
;
730 tt2
= gg_j(vs1
[4], vs1
[5], vs1
[6], j
) + vs1
[7] + ss1
+ vs2
[0];
732 vd
[3] = rol32(vs1
[1], 9);
736 vd
[7] = rol32(vs1
[5], 19);
740 ss1
= rol32(rol32(vd
[1], 12) + vd
[5] + rol32(t_j(j
), j
% 32), 7);
741 ss2
= ss1
^ rol32(vd
[1], 12);
742 tt1
= ff_j(vd
[1], vs1
[1], vd
[3], j
) + vs1
[3] + ss2
+ x1
;
743 tt2
= gg_j(vd
[5], vs1
[5], vd
[7], j
) + vs1
[7] + ss1
+ vs2
[1];
744 vd
[2] = rol32(vs1
[1], 9);
746 vd
[6] = rol32(vs1
[5], 19);
750 void HELPER(vsm3c_vi
)(void *vd_vptr
, void *vs2_vptr
, uint32_t uimm
,
751 CPURISCVState
*env
, uint32_t desc
)
753 uint32_t esz
= memop_size(FIELD_EX64(env
->vtype
, VTYPE
, VSEW
));
754 uint32_t total_elems
= vext_get_total_elems(env
, desc
, esz
);
755 uint32_t vta
= vext_vta(desc
);
756 uint32_t *vd
= vd_vptr
;
757 uint32_t *vs2
= vs2_vptr
;
758 uint32_t v1
[8], v2
[8], v3
[8];
760 for (int i
= env
->vstart
/ 8; i
< env
->vl
/ 8; i
++) {
761 for (int k
= 0; k
< 8; k
++) {
762 v2
[k
] = bswap32(vd
[H4(i
* 8 + k
)]);
763 v3
[k
] = bswap32(vs2
[H4(i
* 8 + k
)]);
765 sm3c(v1
, v2
, v3
, uimm
);
766 for (int k
= 0; k
< 8; k
++) {
767 vd
[i
* 8 + k
] = bswap32(v1
[H4(k
)]);
770 vext_set_elems_1s(vd_vptr
, vta
, env
->vl
* esz
, total_elems
* esz
);
774 void HELPER(vghsh_vv
)(void *vd_vptr
, void *vs1_vptr
, void *vs2_vptr
,
775 CPURISCVState
*env
, uint32_t desc
)
777 uint64_t *vd
= vd_vptr
;
778 uint64_t *vs1
= vs1_vptr
;
779 uint64_t *vs2
= vs2_vptr
;
780 uint32_t vta
= vext_vta(desc
);
781 uint32_t total_elems
= vext_get_total_elems(env
, desc
, 4);
783 for (uint32_t i
= env
->vstart
/ 4; i
< env
->vl
/ 4; i
++) {
784 uint64_t Y
[2] = {vd
[i
* 2 + 0], vd
[i
* 2 + 1]};
785 uint64_t H
[2] = {brev8(vs2
[i
* 2 + 0]), brev8(vs2
[i
* 2 + 1])};
786 uint64_t X
[2] = {vs1
[i
* 2 + 0], vs1
[i
* 2 + 1]};
787 uint64_t Z
[2] = {0, 0};
789 uint64_t S
[2] = {brev8(Y
[0] ^ X
[0]), brev8(Y
[1] ^ X
[1])};
791 for (int j
= 0; j
< 128; j
++) {
792 if ((S
[j
/ 64] >> (j
% 64)) & 1) {
796 bool reduce
= ((H
[1] >> 63) & 1);
797 H
[1] = H
[1] << 1 | H
[0] >> 63;
804 vd
[i
* 2 + 0] = brev8(Z
[0]);
805 vd
[i
* 2 + 1] = brev8(Z
[1]);
807 /* set tail elements to 1s */
808 vext_set_elems_1s(vd
, vta
, env
->vl
* 4, total_elems
* 4);
812 void HELPER(vgmul_vv
)(void *vd_vptr
, void *vs2_vptr
, CPURISCVState
*env
,
815 uint64_t *vd
= vd_vptr
;
816 uint64_t *vs2
= vs2_vptr
;
817 uint32_t vta
= vext_vta(desc
);
818 uint32_t total_elems
= vext_get_total_elems(env
, desc
, 4);
820 for (uint32_t i
= env
->vstart
/ 4; i
< env
->vl
/ 4; i
++) {
821 uint64_t Y
[2] = {brev8(vd
[i
* 2 + 0]), brev8(vd
[i
* 2 + 1])};
822 uint64_t H
[2] = {brev8(vs2
[i
* 2 + 0]), brev8(vs2
[i
* 2 + 1])};
823 uint64_t Z
[2] = {0, 0};
825 for (int j
= 0; j
< 128; j
++) {
826 if ((Y
[j
/ 64] >> (j
% 64)) & 1) {
830 bool reduce
= ((H
[1] >> 63) & 1);
831 H
[1] = H
[1] << 1 | H
[0] >> 63;
838 vd
[i
* 2 + 0] = brev8(Z
[0]);
839 vd
[i
* 2 + 1] = brev8(Z
[1]);
841 /* set tail elements to 1s */
842 vext_set_elems_1s(vd
, vta
, env
->vl
* 4, total_elems
* 4);
846 void HELPER(vsm4k_vi
)(void *vd
, void *vs2
, uint32_t uimm5
, CPURISCVState
*env
,
849 const uint32_t egs
= 4;
850 uint32_t rnd
= uimm5
& 0x7;
851 uint32_t group_start
= env
->vstart
/ egs
;
852 uint32_t group_end
= env
->vl
/ egs
;
853 uint32_t esz
= sizeof(uint32_t);
854 uint32_t total_elems
= vext_get_total_elems(env
, desc
, esz
);
856 for (uint32_t i
= group_start
; i
< group_end
; ++i
) {
857 uint32_t vstart
= i
* egs
;
858 uint32_t vend
= (i
+ 1) * egs
;
859 uint32_t rk
[4] = {0};
860 uint32_t tmp
[8] = {0};
862 for (uint32_t j
= vstart
; j
< vend
; ++j
) {
863 rk
[j
- vstart
] = *((uint32_t *)vs2
+ H4(j
));
866 for (uint32_t j
= 0; j
< egs
; ++j
) {
870 for (uint32_t j
= 0; j
< egs
; ++j
) {
872 b
= tmp
[j
+ 1] ^ tmp
[j
+ 2] ^ tmp
[j
+ 3] ^ sm4_ck
[rnd
* 4 + j
];
876 tmp
[j
+ 4] = tmp
[j
] ^ (s
^ rol32(s
, 13) ^ rol32(s
, 23));
879 for (uint32_t j
= vstart
; j
< vend
; ++j
) {
880 *((uint32_t *)vd
+ H4(j
)) = tmp
[egs
+ (j
- vstart
)];
885 /* set tail elements to 1s */
886 vext_set_elems_1s(vd
, vext_vta(desc
), env
->vl
* esz
, total_elems
* esz
);
889 static void do_sm4_round(uint32_t *rk
, uint32_t *buf
)
891 const uint32_t egs
= 4;
894 for (uint32_t j
= egs
; j
< egs
* 2; ++j
) {
895 b
= buf
[j
- 3] ^ buf
[j
- 2] ^ buf
[j
- 1] ^ rk
[j
- 4];
899 buf
[j
] = buf
[j
- 4] ^ (s
^ rol32(s
, 2) ^ rol32(s
, 10) ^ rol32(s
, 18) ^
904 void HELPER(vsm4r_vv
)(void *vd
, void *vs2
, CPURISCVState
*env
, uint32_t desc
)
906 const uint32_t egs
= 4;
907 uint32_t group_start
= env
->vstart
/ egs
;
908 uint32_t group_end
= env
->vl
/ egs
;
909 uint32_t esz
= sizeof(uint32_t);
910 uint32_t total_elems
= vext_get_total_elems(env
, desc
, esz
);
912 for (uint32_t i
= group_start
; i
< group_end
; ++i
) {
913 uint32_t vstart
= i
* egs
;
914 uint32_t vend
= (i
+ 1) * egs
;
915 uint32_t rk
[4] = {0};
916 uint32_t tmp
[8] = {0};
918 for (uint32_t j
= vstart
; j
< vend
; ++j
) {
919 rk
[j
- vstart
] = *((uint32_t *)vs2
+ H4(j
));
922 for (uint32_t j
= vstart
; j
< vend
; ++j
) {
923 tmp
[j
- vstart
] = *((uint32_t *)vd
+ H4(j
));
926 do_sm4_round(rk
, tmp
);
928 for (uint32_t j
= vstart
; j
< vend
; ++j
) {
929 *((uint32_t *)vd
+ H4(j
)) = tmp
[egs
+ (j
- vstart
)];
934 /* set tail elements to 1s */
935 vext_set_elems_1s(vd
, vext_vta(desc
), env
->vl
* esz
, total_elems
* esz
);
938 void HELPER(vsm4r_vs
)(void *vd
, void *vs2
, CPURISCVState
*env
, uint32_t desc
)
940 const uint32_t egs
= 4;
941 uint32_t group_start
= env
->vstart
/ egs
;
942 uint32_t group_end
= env
->vl
/ egs
;
943 uint32_t esz
= sizeof(uint32_t);
944 uint32_t total_elems
= vext_get_total_elems(env
, desc
, esz
);
946 for (uint32_t i
= group_start
; i
< group_end
; ++i
) {
947 uint32_t vstart
= i
* egs
;
948 uint32_t vend
= (i
+ 1) * egs
;
949 uint32_t rk
[4] = {0};
950 uint32_t tmp
[8] = {0};
952 for (uint32_t j
= 0; j
< egs
; ++j
) {
953 rk
[j
] = *((uint32_t *)vs2
+ H4(j
));
956 for (uint32_t j
= vstart
; j
< vend
; ++j
) {
957 tmp
[j
- vstart
] = *((uint32_t *)vd
+ H4(j
));
960 do_sm4_round(rk
, tmp
);
962 for (uint32_t j
= vstart
; j
< vend
; ++j
) {
963 *((uint32_t *)vd
+ H4(j
)) = tmp
[egs
+ (j
- vstart
)];
968 /* set tail elements to 1s */
969 vext_set_elems_1s(vd
, vext_vta(desc
), env
->vl
* esz
, total_elems
* esz
);