2 * ARM v8.3-PAuth Operations
4 * Copyright (c) 2019 Linaro, Ltd.
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "internals.h"
23 #include "exec/exec-all.h"
24 #include "exec/cpu_ldst.h"
25 #include "exec/helper-proto.h"
26 #include "tcg/tcg-gvec-desc.h"
27 #include "qemu/xxhash.h"
30 static uint64_t pac_cell_shuffle(uint64_t i
)
34 o
|= extract64(i
, 52, 4);
35 o
|= extract64(i
, 24, 4) << 4;
36 o
|= extract64(i
, 44, 4) << 8;
37 o
|= extract64(i
, 0, 4) << 12;
39 o
|= extract64(i
, 28, 4) << 16;
40 o
|= extract64(i
, 48, 4) << 20;
41 o
|= extract64(i
, 4, 4) << 24;
42 o
|= extract64(i
, 40, 4) << 28;
44 o
|= extract64(i
, 32, 4) << 32;
45 o
|= extract64(i
, 12, 4) << 36;
46 o
|= extract64(i
, 56, 4) << 40;
47 o
|= extract64(i
, 20, 4) << 44;
49 o
|= extract64(i
, 8, 4) << 48;
50 o
|= extract64(i
, 36, 4) << 52;
51 o
|= extract64(i
, 16, 4) << 56;
52 o
|= extract64(i
, 60, 4) << 60;
57 static uint64_t pac_cell_inv_shuffle(uint64_t i
)
61 o
|= extract64(i
, 12, 4);
62 o
|= extract64(i
, 24, 4) << 4;
63 o
|= extract64(i
, 48, 4) << 8;
64 o
|= extract64(i
, 36, 4) << 12;
66 o
|= extract64(i
, 56, 4) << 16;
67 o
|= extract64(i
, 44, 4) << 20;
68 o
|= extract64(i
, 4, 4) << 24;
69 o
|= extract64(i
, 16, 4) << 28;
71 o
|= i
& MAKE_64BIT_MASK(32, 4);
72 o
|= extract64(i
, 52, 4) << 36;
73 o
|= extract64(i
, 28, 4) << 40;
74 o
|= extract64(i
, 8, 4) << 44;
76 o
|= extract64(i
, 20, 4) << 48;
77 o
|= extract64(i
, 0, 4) << 52;
78 o
|= extract64(i
, 40, 4) << 56;
79 o
|= i
& MAKE_64BIT_MASK(60, 4);
84 static uint64_t pac_sub(uint64_t i
)
86 static const uint8_t sub
[16] = {
87 0xb, 0x6, 0x8, 0xf, 0xc, 0x0, 0x9, 0xe,
88 0x3, 0x7, 0x4, 0x5, 0xd, 0x2, 0x1, 0xa,
93 for (b
= 0; b
< 64; b
+= 4) {
94 o
|= (uint64_t)sub
[(i
>> b
) & 0xf] << b
;
99 static uint64_t pac_inv_sub(uint64_t i
)
101 static const uint8_t inv_sub
[16] = {
102 0x5, 0xe, 0xd, 0x8, 0xa, 0xb, 0x1, 0x9,
103 0x2, 0x6, 0xf, 0x0, 0x4, 0xc, 0x7, 0x3,
108 for (b
= 0; b
< 64; b
+= 4) {
109 o
|= (uint64_t)inv_sub
[(i
>> b
) & 0xf] << b
;
114 static int rot_cell(int cell
, int n
)
116 /* 4-bit rotate left by n. */
118 return extract32(cell
, 4 - n
, 4);
121 static uint64_t pac_mult(uint64_t i
)
126 for (b
= 0; b
< 4 * 4; b
+= 4) {
127 int i0
, i4
, i8
, ic
, t0
, t1
, t2
, t3
;
129 i0
= extract64(i
, b
, 4);
130 i4
= extract64(i
, b
+ 4 * 4, 4);
131 i8
= extract64(i
, b
+ 8 * 4, 4);
132 ic
= extract64(i
, b
+ 12 * 4, 4);
134 t0
= rot_cell(i8
, 1) ^ rot_cell(i4
, 2) ^ rot_cell(i0
, 1);
135 t1
= rot_cell(ic
, 1) ^ rot_cell(i4
, 1) ^ rot_cell(i0
, 2);
136 t2
= rot_cell(ic
, 2) ^ rot_cell(i8
, 1) ^ rot_cell(i0
, 1);
137 t3
= rot_cell(ic
, 1) ^ rot_cell(i8
, 2) ^ rot_cell(i4
, 1);
139 o
|= (uint64_t)t3
<< b
;
140 o
|= (uint64_t)t2
<< (b
+ 4 * 4);
141 o
|= (uint64_t)t1
<< (b
+ 8 * 4);
142 o
|= (uint64_t)t0
<< (b
+ 12 * 4);
147 static uint64_t tweak_cell_rot(uint64_t cell
)
149 return (cell
>> 1) | (((cell
^ (cell
>> 1)) & 1) << 3);
152 static uint64_t tweak_shuffle(uint64_t i
)
156 o
|= extract64(i
, 16, 4) << 0;
157 o
|= extract64(i
, 20, 4) << 4;
158 o
|= tweak_cell_rot(extract64(i
, 24, 4)) << 8;
159 o
|= extract64(i
, 28, 4) << 12;
161 o
|= tweak_cell_rot(extract64(i
, 44, 4)) << 16;
162 o
|= extract64(i
, 8, 4) << 20;
163 o
|= extract64(i
, 12, 4) << 24;
164 o
|= tweak_cell_rot(extract64(i
, 32, 4)) << 28;
166 o
|= extract64(i
, 48, 4) << 32;
167 o
|= extract64(i
, 52, 4) << 36;
168 o
|= extract64(i
, 56, 4) << 40;
169 o
|= tweak_cell_rot(extract64(i
, 60, 4)) << 44;
171 o
|= tweak_cell_rot(extract64(i
, 0, 4)) << 48;
172 o
|= extract64(i
, 4, 4) << 52;
173 o
|= tweak_cell_rot(extract64(i
, 40, 4)) << 56;
174 o
|= tweak_cell_rot(extract64(i
, 36, 4)) << 60;
179 static uint64_t tweak_cell_inv_rot(uint64_t cell
)
181 return ((cell
<< 1) & 0xf) | ((cell
& 1) ^ (cell
>> 3));
184 static uint64_t tweak_inv_shuffle(uint64_t i
)
188 o
|= tweak_cell_inv_rot(extract64(i
, 48, 4));
189 o
|= extract64(i
, 52, 4) << 4;
190 o
|= extract64(i
, 20, 4) << 8;
191 o
|= extract64(i
, 24, 4) << 12;
193 o
|= extract64(i
, 0, 4) << 16;
194 o
|= extract64(i
, 4, 4) << 20;
195 o
|= tweak_cell_inv_rot(extract64(i
, 8, 4)) << 24;
196 o
|= extract64(i
, 12, 4) << 28;
198 o
|= tweak_cell_inv_rot(extract64(i
, 28, 4)) << 32;
199 o
|= tweak_cell_inv_rot(extract64(i
, 60, 4)) << 36;
200 o
|= tweak_cell_inv_rot(extract64(i
, 56, 4)) << 40;
201 o
|= tweak_cell_inv_rot(extract64(i
, 16, 4)) << 44;
203 o
|= extract64(i
, 32, 4) << 48;
204 o
|= extract64(i
, 36, 4) << 52;
205 o
|= extract64(i
, 40, 4) << 56;
206 o
|= tweak_cell_inv_rot(extract64(i
, 44, 4)) << 60;
211 static uint64_t pauth_computepac_architected(uint64_t data
, uint64_t modifier
,
214 static const uint64_t RC
[5] = {
215 0x0000000000000000ull
,
216 0x13198A2E03707344ull
,
217 0xA4093822299F31D0ull
,
218 0x082EFA98EC4E6C89ull
,
219 0x452821E638D01377ull
,
221 const uint64_t alpha
= 0xC0AC29B7C97C50DDull
;
223 * Note that in the ARM pseudocode, key0 contains bits <127:64>
224 * and key1 contains bits <63:0> of the 128-bit key.
226 uint64_t key0
= key
.hi
, key1
= key
.lo
;
227 uint64_t workingval
, runningmod
, roundkey
, modk0
;
230 modk0
= (key0
<< 63) | ((key0
>> 1) ^ (key0
>> 63));
231 runningmod
= modifier
;
232 workingval
= data
^ key0
;
234 for (i
= 0; i
<= 4; ++i
) {
235 roundkey
= key1
^ runningmod
;
236 workingval
^= roundkey
;
239 workingval
= pac_cell_shuffle(workingval
);
240 workingval
= pac_mult(workingval
);
242 workingval
= pac_sub(workingval
);
243 runningmod
= tweak_shuffle(runningmod
);
245 roundkey
= modk0
^ runningmod
;
246 workingval
^= roundkey
;
247 workingval
= pac_cell_shuffle(workingval
);
248 workingval
= pac_mult(workingval
);
249 workingval
= pac_sub(workingval
);
250 workingval
= pac_cell_shuffle(workingval
);
251 workingval
= pac_mult(workingval
);
253 workingval
= pac_cell_inv_shuffle(workingval
);
254 workingval
= pac_inv_sub(workingval
);
255 workingval
= pac_mult(workingval
);
256 workingval
= pac_cell_inv_shuffle(workingval
);
258 workingval
^= runningmod
;
259 for (i
= 0; i
<= 4; ++i
) {
260 workingval
= pac_inv_sub(workingval
);
262 workingval
= pac_mult(workingval
);
263 workingval
= pac_cell_inv_shuffle(workingval
);
265 runningmod
= tweak_inv_shuffle(runningmod
);
266 roundkey
= key1
^ runningmod
;
267 workingval
^= RC
[4 - i
];
268 workingval
^= roundkey
;
276 static uint64_t pauth_computepac_impdef(uint64_t data
, uint64_t modifier
,
279 return qemu_xxhash64_4(data
, modifier
, key
.lo
, key
.hi
);
282 static uint64_t pauth_computepac(CPUARMState
*env
, uint64_t data
,
283 uint64_t modifier
, ARMPACKey key
)
285 if (cpu_isar_feature(aa64_pauth_arch
, env_archcpu(env
))) {
286 return pauth_computepac_architected(data
, modifier
, key
);
288 return pauth_computepac_impdef(data
, modifier
, key
);
292 static uint64_t pauth_addpac(CPUARMState
*env
, uint64_t ptr
, uint64_t modifier
,
293 ARMPACKey
*key
, bool data
)
295 ARMMMUIdx mmu_idx
= arm_stage1_mmu_idx(env
);
296 ARMVAParameters param
= aa64_va_parameters(env
, ptr
, mmu_idx
, data
);
297 uint64_t pac
, ext_ptr
, ext
, test
;
298 int bot_bit
, top_bit
;
300 /* If tagged pointers are in use, use ptr<55>, otherwise ptr<63>. */
302 ext
= sextract64(ptr
, 55, 1);
304 ext
= sextract64(ptr
, 63, 1);
307 /* Build a pointer with known good extension bits. */
308 top_bit
= 64 - 8 * param
.tbi
;
309 bot_bit
= 64 - param
.tsz
;
310 ext_ptr
= deposit64(ptr
, bot_bit
, top_bit
- bot_bit
, ext
);
312 pac
= pauth_computepac(env
, ext_ptr
, modifier
, *key
);
315 * Check if the ptr has good extension bits and corrupt the
316 * pointer authentication code if not.
318 test
= sextract64(ptr
, bot_bit
, top_bit
- bot_bit
);
319 if (test
!= 0 && test
!= -1) {
321 * Note that our top_bit is one greater than the pseudocode's
322 * version, hence "- 2" here.
324 pac
^= MAKE_64BIT_MASK(top_bit
- 2, 1);
328 * Preserve the determination between upper and lower at bit 55,
329 * and insert pointer authentication code.
332 ptr
&= ~MAKE_64BIT_MASK(bot_bit
, 55 - bot_bit
+ 1);
333 pac
&= MAKE_64BIT_MASK(bot_bit
, 54 - bot_bit
+ 1);
335 ptr
&= MAKE_64BIT_MASK(0, bot_bit
);
336 pac
&= ~(MAKE_64BIT_MASK(55, 1) | MAKE_64BIT_MASK(0, bot_bit
));
338 ext
&= MAKE_64BIT_MASK(55, 1);
339 return pac
| ext
| ptr
;
342 static uint64_t pauth_original_ptr(uint64_t ptr
, ARMVAParameters param
)
344 /* Note that bit 55 is used whether or not the regime has 2 ranges. */
345 uint64_t extfield
= sextract64(ptr
, 55, 1);
346 int bot_pac_bit
= 64 - param
.tsz
;
347 int top_pac_bit
= 64 - 8 * param
.tbi
;
349 return deposit64(ptr
, bot_pac_bit
, top_pac_bit
- bot_pac_bit
, extfield
);
352 static uint64_t pauth_auth(CPUARMState
*env
, uint64_t ptr
, uint64_t modifier
,
353 ARMPACKey
*key
, bool data
, int keynumber
)
355 ARMMMUIdx mmu_idx
= arm_stage1_mmu_idx(env
);
356 ARMVAParameters param
= aa64_va_parameters(env
, ptr
, mmu_idx
, data
);
357 int bot_bit
, top_bit
;
358 uint64_t pac
, orig_ptr
, test
;
360 orig_ptr
= pauth_original_ptr(ptr
, param
);
361 pac
= pauth_computepac(env
, orig_ptr
, modifier
, *key
);
362 bot_bit
= 64 - param
.tsz
;
363 top_bit
= 64 - 8 * param
.tbi
;
365 test
= (pac
^ ptr
) & ~MAKE_64BIT_MASK(55, 1);
366 if (unlikely(extract64(test
, bot_bit
, top_bit
- bot_bit
))) {
367 int error_code
= (keynumber
<< 1) | (keynumber
^ 1);
369 return deposit64(orig_ptr
, 53, 2, error_code
);
371 return deposit64(orig_ptr
, 61, 2, error_code
);
377 static uint64_t pauth_strip(CPUARMState
*env
, uint64_t ptr
, bool data
)
379 ARMMMUIdx mmu_idx
= arm_stage1_mmu_idx(env
);
380 ARMVAParameters param
= aa64_va_parameters(env
, ptr
, mmu_idx
, data
);
382 return pauth_original_ptr(ptr
, param
);
385 static void QEMU_NORETURN
pauth_trap(CPUARMState
*env
, int target_el
,
388 raise_exception_ra(env
, EXCP_UDEF
, syn_pactrap(), target_el
, ra
);
391 static void pauth_check_trap(CPUARMState
*env
, int el
, uintptr_t ra
)
393 if (el
< 2 && arm_feature(env
, ARM_FEATURE_EL2
)) {
394 uint64_t hcr
= arm_hcr_el2_eff(env
);
395 bool trap
= !(hcr
& HCR_API
);
397 /* Trap only applies to EL1&0 regime. */
398 trap
&= (hcr
& (HCR_E2H
| HCR_TGE
)) != (HCR_E2H
| HCR_TGE
);
400 /* FIXME: ARMv8.3-NV: HCR_NV trap takes precedence for ERETA[AB]. */
402 pauth_trap(env
, 2, ra
);
405 if (el
< 3 && arm_feature(env
, ARM_FEATURE_EL3
)) {
406 if (!(env
->cp15
.scr_el3
& SCR_API
)) {
407 pauth_trap(env
, 3, ra
);
412 static bool pauth_key_enabled(CPUARMState
*env
, int el
, uint32_t bit
)
414 return (arm_sctlr(env
, el
) & bit
) != 0;
417 uint64_t HELPER(pacia
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
419 int el
= arm_current_el(env
);
420 if (!pauth_key_enabled(env
, el
, SCTLR_EnIA
)) {
423 pauth_check_trap(env
, el
, GETPC());
424 return pauth_addpac(env
, x
, y
, &env
->keys
.apia
, false);
427 uint64_t HELPER(pacib
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
429 int el
= arm_current_el(env
);
430 if (!pauth_key_enabled(env
, el
, SCTLR_EnIB
)) {
433 pauth_check_trap(env
, el
, GETPC());
434 return pauth_addpac(env
, x
, y
, &env
->keys
.apib
, false);
437 uint64_t HELPER(pacda
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
439 int el
= arm_current_el(env
);
440 if (!pauth_key_enabled(env
, el
, SCTLR_EnDA
)) {
443 pauth_check_trap(env
, el
, GETPC());
444 return pauth_addpac(env
, x
, y
, &env
->keys
.apda
, true);
447 uint64_t HELPER(pacdb
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
449 int el
= arm_current_el(env
);
450 if (!pauth_key_enabled(env
, el
, SCTLR_EnDB
)) {
453 pauth_check_trap(env
, el
, GETPC());
454 return pauth_addpac(env
, x
, y
, &env
->keys
.apdb
, true);
457 uint64_t HELPER(pacga
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
461 pauth_check_trap(env
, arm_current_el(env
), GETPC());
462 pac
= pauth_computepac(env
, x
, y
, env
->keys
.apga
);
464 return pac
& 0xffffffff00000000ull
;
467 uint64_t HELPER(autia
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
469 int el
= arm_current_el(env
);
470 if (!pauth_key_enabled(env
, el
, SCTLR_EnIA
)) {
473 pauth_check_trap(env
, el
, GETPC());
474 return pauth_auth(env
, x
, y
, &env
->keys
.apia
, false, 0);
477 uint64_t HELPER(autib
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
479 int el
= arm_current_el(env
);
480 if (!pauth_key_enabled(env
, el
, SCTLR_EnIB
)) {
483 pauth_check_trap(env
, el
, GETPC());
484 return pauth_auth(env
, x
, y
, &env
->keys
.apib
, false, 1);
487 uint64_t HELPER(autda
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
489 int el
= arm_current_el(env
);
490 if (!pauth_key_enabled(env
, el
, SCTLR_EnDA
)) {
493 pauth_check_trap(env
, el
, GETPC());
494 return pauth_auth(env
, x
, y
, &env
->keys
.apda
, true, 0);
497 uint64_t HELPER(autdb
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
499 int el
= arm_current_el(env
);
500 if (!pauth_key_enabled(env
, el
, SCTLR_EnDB
)) {
503 pauth_check_trap(env
, el
, GETPC());
504 return pauth_auth(env
, x
, y
, &env
->keys
.apdb
, true, 1);
507 uint64_t HELPER(xpaci
)(CPUARMState
*env
, uint64_t a
)
509 return pauth_strip(env
, a
, false);
512 uint64_t HELPER(xpacd
)(CPUARMState
*env
, uint64_t a
)
514 return pauth_strip(env
, a
, true);