?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge remote-tracking branch 'remotes/alex.williamson/tags/vfio-update-20211101.0...
[qemu.git] / target / arm / pauth_helper.c
blobcd6df18150bfdcd9f80b7fccc6566c23135f1b9c
1 /*
2 * ARM v8.3-PAuth Operations
3 *
4 * Copyright (c) 2019 Linaro, Ltd.
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "cpu.h"
22 #include "internals.h"
23 #include "exec/exec-all.h"
24 #include "exec/cpu_ldst.h"
25 #include "exec/helper-proto.h"
26 #include "tcg/tcg-gvec-desc.h"
27 #include "qemu/xxhash.h"
28
29
30 static uint64_t pac_cell_shuffle(uint64_t i)
31 {
32 uint64_t o = 0;
33
34 o |= extract64(i, 52, 4);
35 o |= extract64(i, 24, 4) << 4;
36 o |= extract64(i, 44, 4) << 8;
37 o |= extract64(i, 0, 4) << 12;
38
39 o |= extract64(i, 28, 4) << 16;
40 o |= extract64(i, 48, 4) << 20;
41 o |= extract64(i, 4, 4) << 24;
42 o |= extract64(i, 40, 4) << 28;
43
44 o |= extract64(i, 32, 4) << 32;
45 o |= extract64(i, 12, 4) << 36;
46 o |= extract64(i, 56, 4) << 40;
47 o |= extract64(i, 20, 4) << 44;
48
49 o |= extract64(i, 8, 4) << 48;
50 o |= extract64(i, 36, 4) << 52;
51 o |= extract64(i, 16, 4) << 56;
52 o |= extract64(i, 60, 4) << 60;
53
54 return o;
55 }
56
57 static uint64_t pac_cell_inv_shuffle(uint64_t i)
58 {
59 uint64_t o = 0;
60
61 o |= extract64(i, 12, 4);
62 o |= extract64(i, 24, 4) << 4;
63 o |= extract64(i, 48, 4) << 8;
64 o |= extract64(i, 36, 4) << 12;
65
66 o |= extract64(i, 56, 4) << 16;
67 o |= extract64(i, 44, 4) << 20;
68 o |= extract64(i, 4, 4) << 24;
69 o |= extract64(i, 16, 4) << 28;
70
71 o |= i & MAKE_64BIT_MASK(32, 4);
72 o |= extract64(i, 52, 4) << 36;
73 o |= extract64(i, 28, 4) << 40;
74 o |= extract64(i, 8, 4) << 44;
75
76 o |= extract64(i, 20, 4) << 48;
77 o |= extract64(i, 0, 4) << 52;
78 o |= extract64(i, 40, 4) << 56;
79 o |= i & MAKE_64BIT_MASK(60, 4);
80
81 return o;
82 }
83
84 static uint64_t pac_sub(uint64_t i)
85 {
86 static const uint8_t sub[16] = {
87 0xb, 0x6, 0x8, 0xf, 0xc, 0x0, 0x9, 0xe,
88 0x3, 0x7, 0x4, 0x5, 0xd, 0x2, 0x1, 0xa,
89 };
90 uint64_t o = 0;
91 int b;
92
93 for (b = 0; b < 64; b += 4) {
94 o |= (uint64_t)sub[(i >> b) & 0xf] << b;
95 }
96 return o;
97 }
98
99 static uint64_t pac_inv_sub(uint64_t i)
100 {
101 static const uint8_t inv_sub[16] = {
102 0x5, 0xe, 0xd, 0x8, 0xa, 0xb, 0x1, 0x9,
103 0x2, 0x6, 0xf, 0x0, 0x4, 0xc, 0x7, 0x3,
104 };
105 uint64_t o = 0;
106 int b;
107
108 for (b = 0; b < 64; b += 4) {
109 o |= (uint64_t)inv_sub[(i >> b) & 0xf] << b;
110 }
111 return o;
112 }
113
114 static int rot_cell(int cell, int n)
115 {
116 /* 4-bit rotate left by n. */
117 cell |= cell << 4;
118 return extract32(cell, 4 - n, 4);
119 }
120
121 static uint64_t pac_mult(uint64_t i)
122 {
123 uint64_t o = 0;
124 int b;
125
126 for (b = 0; b < 4 * 4; b += 4) {
127 int i0, i4, i8, ic, t0, t1, t2, t3;
128
129 i0 = extract64(i, b, 4);
130 i4 = extract64(i, b + 4 * 4, 4);
131 i8 = extract64(i, b + 8 * 4, 4);
132 ic = extract64(i, b + 12 * 4, 4);
133
134 t0 = rot_cell(i8, 1) ^ rot_cell(i4, 2) ^ rot_cell(i0, 1);
135 t1 = rot_cell(ic, 1) ^ rot_cell(i4, 1) ^ rot_cell(i0, 2);
136 t2 = rot_cell(ic, 2) ^ rot_cell(i8, 1) ^ rot_cell(i0, 1);
137 t3 = rot_cell(ic, 1) ^ rot_cell(i8, 2) ^ rot_cell(i4, 1);
138
139 o |= (uint64_t)t3 << b;
140 o |= (uint64_t)t2 << (b + 4 * 4);
141 o |= (uint64_t)t1 << (b + 8 * 4);
142 o |= (uint64_t)t0 << (b + 12 * 4);
143 }
144 return o;
145 }
146
147 static uint64_t tweak_cell_rot(uint64_t cell)
148 {
149 return (cell >> 1) | (((cell ^ (cell >> 1)) & 1) << 3);
150 }
151
152 static uint64_t tweak_shuffle(uint64_t i)
153 {
154 uint64_t o = 0;
155
156 o |= extract64(i, 16, 4) << 0;
157 o |= extract64(i, 20, 4) << 4;
158 o |= tweak_cell_rot(extract64(i, 24, 4)) << 8;
159 o |= extract64(i, 28, 4) << 12;
160
161 o |= tweak_cell_rot(extract64(i, 44, 4)) << 16;
162 o |= extract64(i, 8, 4) << 20;
163 o |= extract64(i, 12, 4) << 24;
164 o |= tweak_cell_rot(extract64(i, 32, 4)) << 28;
165
166 o |= extract64(i, 48, 4) << 32;
167 o |= extract64(i, 52, 4) << 36;
168 o |= extract64(i, 56, 4) << 40;
169 o |= tweak_cell_rot(extract64(i, 60, 4)) << 44;
170
171 o |= tweak_cell_rot(extract64(i, 0, 4)) << 48;
172 o |= extract64(i, 4, 4) << 52;
173 o |= tweak_cell_rot(extract64(i, 40, 4)) << 56;
174 o |= tweak_cell_rot(extract64(i, 36, 4)) << 60;
175
176 return o;
177 }
178
179 static uint64_t tweak_cell_inv_rot(uint64_t cell)
180 {
181 return ((cell << 1) & 0xf) | ((cell & 1) ^ (cell >> 3));
182 }
183
184 static uint64_t tweak_inv_shuffle(uint64_t i)
185 {
186 uint64_t o = 0;
187
188 o |= tweak_cell_inv_rot(extract64(i, 48, 4));
189 o |= extract64(i, 52, 4) << 4;
190 o |= extract64(i, 20, 4) << 8;
191 o |= extract64(i, 24, 4) << 12;
192
193 o |= extract64(i, 0, 4) << 16;
194 o |= extract64(i, 4, 4) << 20;
195 o |= tweak_cell_inv_rot(extract64(i, 8, 4)) << 24;
196 o |= extract64(i, 12, 4) << 28;
197
198 o |= tweak_cell_inv_rot(extract64(i, 28, 4)) << 32;
199 o |= tweak_cell_inv_rot(extract64(i, 60, 4)) << 36;
200 o |= tweak_cell_inv_rot(extract64(i, 56, 4)) << 40;
201 o |= tweak_cell_inv_rot(extract64(i, 16, 4)) << 44;
202
203 o |= extract64(i, 32, 4) << 48;
204 o |= extract64(i, 36, 4) << 52;
205 o |= extract64(i, 40, 4) << 56;
206 o |= tweak_cell_inv_rot(extract64(i, 44, 4)) << 60;
207
208 return o;
209 }
210
211 static uint64_t pauth_computepac_architected(uint64_t data, uint64_t modifier,
212 ARMPACKey key)
213 {
214 static const uint64_t RC[5] = {
215 0x0000000000000000ull,
216 0x13198A2E03707344ull,
217 0xA4093822299F31D0ull,
218 0x082EFA98EC4E6C89ull,
219 0x452821E638D01377ull,
220 };
221 const uint64_t alpha = 0xC0AC29B7C97C50DDull;
222 /*
223 * Note that in the ARM pseudocode, key0 contains bits <127:64>
224 * and key1 contains bits <63:0> of the 128-bit key.
225 */
226 uint64_t key0 = key.hi, key1 = key.lo;
227 uint64_t workingval, runningmod, roundkey, modk0;
228 int i;
229
230 modk0 = (key0 << 63) | ((key0 >> 1) ^ (key0 >> 63));
231 runningmod = modifier;
232 workingval = data ^ key0;
233
234 for (i = 0; i <= 4; ++i) {
235 roundkey = key1 ^ runningmod;
236 workingval ^= roundkey;
237 workingval ^= RC[i];
238 if (i > 0) {
239 workingval = pac_cell_shuffle(workingval);
240 workingval = pac_mult(workingval);
241 }
242 workingval = pac_sub(workingval);
243 runningmod = tweak_shuffle(runningmod);
244 }
245 roundkey = modk0 ^ runningmod;
246 workingval ^= roundkey;
247 workingval = pac_cell_shuffle(workingval);
248 workingval = pac_mult(workingval);
249 workingval = pac_sub(workingval);
250 workingval = pac_cell_shuffle(workingval);
251 workingval = pac_mult(workingval);
252 workingval ^= key1;
253 workingval = pac_cell_inv_shuffle(workingval);
254 workingval = pac_inv_sub(workingval);
255 workingval = pac_mult(workingval);
256 workingval = pac_cell_inv_shuffle(workingval);
257 workingval ^= key0;
258 workingval ^= runningmod;
259 for (i = 0; i <= 4; ++i) {
260 workingval = pac_inv_sub(workingval);
261 if (i < 4) {
262 workingval = pac_mult(workingval);
263 workingval = pac_cell_inv_shuffle(workingval);
264 }
265 runningmod = tweak_inv_shuffle(runningmod);
266 roundkey = key1 ^ runningmod;
267 workingval ^= RC[4 - i];
268 workingval ^= roundkey;
269 workingval ^= alpha;
270 }
271 workingval ^= modk0;
272
273 return workingval;
274 }
275
276 static uint64_t pauth_computepac_impdef(uint64_t data, uint64_t modifier,
277 ARMPACKey key)
278 {
279 return qemu_xxhash64_4(data, modifier, key.lo, key.hi);
280 }
281
282 static uint64_t pauth_computepac(CPUARMState *env, uint64_t data,
283 uint64_t modifier, ARMPACKey key)
284 {
285 if (cpu_isar_feature(aa64_pauth_arch, env_archcpu(env))) {
286 return pauth_computepac_architected(data, modifier, key);
287 } else {
288 return pauth_computepac_impdef(data, modifier, key);
289 }
290 }
291
292 static uint64_t pauth_addpac(CPUARMState *env, uint64_t ptr, uint64_t modifier,
293 ARMPACKey *key, bool data)
294 {
295 ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env);
296 ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data);
297 uint64_t pac, ext_ptr, ext, test;
298 int bot_bit, top_bit;
299
300 /* If tagged pointers are in use, use ptr<55>, otherwise ptr<63>. */
301 if (param.tbi) {
302 ext = sextract64(ptr, 55, 1);
303 } else {
304 ext = sextract64(ptr, 63, 1);
305 }
306
307 /* Build a pointer with known good extension bits. */
308 top_bit = 64 - 8 * param.tbi;
309 bot_bit = 64 - param.tsz;
310 ext_ptr = deposit64(ptr, bot_bit, top_bit - bot_bit, ext);
311
312 pac = pauth_computepac(env, ext_ptr, modifier, *key);
313
314 /*
315 * Check if the ptr has good extension bits and corrupt the
316 * pointer authentication code if not.
317 */
318 test = sextract64(ptr, bot_bit, top_bit - bot_bit);
319 if (test != 0 && test != -1) {
320 /*
321 * Note that our top_bit is one greater than the pseudocode's
322 * version, hence "- 2" here.
323 */
324 pac ^= MAKE_64BIT_MASK(top_bit - 2, 1);
325 }
326
327 /*
328 * Preserve the determination between upper and lower at bit 55,
329 * and insert pointer authentication code.
330 */
331 if (param.tbi) {
332 ptr &= ~MAKE_64BIT_MASK(bot_bit, 55 - bot_bit + 1);
333 pac &= MAKE_64BIT_MASK(bot_bit, 54 - bot_bit + 1);
334 } else {
335 ptr &= MAKE_64BIT_MASK(0, bot_bit);
336 pac &= ~(MAKE_64BIT_MASK(55, 1) | MAKE_64BIT_MASK(0, bot_bit));
337 }
338 ext &= MAKE_64BIT_MASK(55, 1);
339 return pac | ext | ptr;
340 }
341
342 static uint64_t pauth_original_ptr(uint64_t ptr, ARMVAParameters param)
343 {
344 /* Note that bit 55 is used whether or not the regime has 2 ranges. */
345 uint64_t extfield = sextract64(ptr, 55, 1);
346 int bot_pac_bit = 64 - param.tsz;
347 int top_pac_bit = 64 - 8 * param.tbi;
348
349 return deposit64(ptr, bot_pac_bit, top_pac_bit - bot_pac_bit, extfield);
350 }
351
352 static uint64_t pauth_auth(CPUARMState *env, uint64_t ptr, uint64_t modifier,
353 ARMPACKey *key, bool data, int keynumber)
354 {
355 ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env);
356 ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data);
357 int bot_bit, top_bit;
358 uint64_t pac, orig_ptr, test;
359
360 orig_ptr = pauth_original_ptr(ptr, param);
361 pac = pauth_computepac(env, orig_ptr, modifier, *key);
362 bot_bit = 64 - param.tsz;
363 top_bit = 64 - 8 * param.tbi;
364
365 test = (pac ^ ptr) & ~MAKE_64BIT_MASK(55, 1);
366 if (unlikely(extract64(test, bot_bit, top_bit - bot_bit))) {
367 int error_code = (keynumber << 1) | (keynumber ^ 1);
368 if (param.tbi) {
369 return deposit64(orig_ptr, 53, 2, error_code);
370 } else {
371 return deposit64(orig_ptr, 61, 2, error_code);
372 }
373 }
374 return orig_ptr;
375 }
376
377 static uint64_t pauth_strip(CPUARMState *env, uint64_t ptr, bool data)
378 {
379 ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env);
380 ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data);
381
382 return pauth_original_ptr(ptr, param);
383 }
384
385 static void QEMU_NORETURN pauth_trap(CPUARMState *env, int target_el,
386 uintptr_t ra)
387 {
388 raise_exception_ra(env, EXCP_UDEF, syn_pactrap(), target_el, ra);
389 }
390
391 static void pauth_check_trap(CPUARMState *env, int el, uintptr_t ra)
392 {
393 if (el < 2 && arm_feature(env, ARM_FEATURE_EL2)) {
394 uint64_t hcr = arm_hcr_el2_eff(env);
395 bool trap = !(hcr & HCR_API);
396 if (el == 0) {
397 /* Trap only applies to EL1&0 regime. */
398 trap &= (hcr & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE);
399 }
400 /* FIXME: ARMv8.3-NV: HCR_NV trap takes precedence for ERETA[AB]. */
401 if (trap) {
402 pauth_trap(env, 2, ra);
403 }
404 }
405 if (el < 3 && arm_feature(env, ARM_FEATURE_EL3)) {
406 if (!(env->cp15.scr_el3 & SCR_API)) {
407 pauth_trap(env, 3, ra);
408 }
409 }
410 }
411
412 static bool pauth_key_enabled(CPUARMState *env, int el, uint32_t bit)
413 {
414 return (arm_sctlr(env, el) & bit) != 0;
415 }
416
417 uint64_t HELPER(pacia)(CPUARMState *env, uint64_t x, uint64_t y)
418 {
419 int el = arm_current_el(env);
420 if (!pauth_key_enabled(env, el, SCTLR_EnIA)) {
421 return x;
422 }
423 pauth_check_trap(env, el, GETPC());
424 return pauth_addpac(env, x, y, &env->keys.apia, false);
425 }
426
427 uint64_t HELPER(pacib)(CPUARMState *env, uint64_t x, uint64_t y)
428 {
429 int el = arm_current_el(env);
430 if (!pauth_key_enabled(env, el, SCTLR_EnIB)) {
431 return x;
432 }
433 pauth_check_trap(env, el, GETPC());
434 return pauth_addpac(env, x, y, &env->keys.apib, false);
435 }
436
437 uint64_t HELPER(pacda)(CPUARMState *env, uint64_t x, uint64_t y)
438 {
439 int el = arm_current_el(env);
440 if (!pauth_key_enabled(env, el, SCTLR_EnDA)) {
441 return x;
442 }
443 pauth_check_trap(env, el, GETPC());
444 return pauth_addpac(env, x, y, &env->keys.apda, true);
445 }
446
447 uint64_t HELPER(pacdb)(CPUARMState *env, uint64_t x, uint64_t y)
448 {
449 int el = arm_current_el(env);
450 if (!pauth_key_enabled(env, el, SCTLR_EnDB)) {
451 return x;
452 }
453 pauth_check_trap(env, el, GETPC());
454 return pauth_addpac(env, x, y, &env->keys.apdb, true);
455 }
456
457 uint64_t HELPER(pacga)(CPUARMState *env, uint64_t x, uint64_t y)
458 {
459 uint64_t pac;
460
461 pauth_check_trap(env, arm_current_el(env), GETPC());
462 pac = pauth_computepac(env, x, y, env->keys.apga);
463
464 return pac & 0xffffffff00000000ull;
465 }
466
467 uint64_t HELPER(autia)(CPUARMState *env, uint64_t x, uint64_t y)
468 {
469 int el = arm_current_el(env);
470 if (!pauth_key_enabled(env, el, SCTLR_EnIA)) {
471 return x;
472 }
473 pauth_check_trap(env, el, GETPC());
474 return pauth_auth(env, x, y, &env->keys.apia, false, 0);
475 }
476
477 uint64_t HELPER(autib)(CPUARMState *env, uint64_t x, uint64_t y)
478 {
479 int el = arm_current_el(env);
480 if (!pauth_key_enabled(env, el, SCTLR_EnIB)) {
481 return x;
482 }
483 pauth_check_trap(env, el, GETPC());
484 return pauth_auth(env, x, y, &env->keys.apib, false, 1);
485 }
486
487 uint64_t HELPER(autda)(CPUARMState *env, uint64_t x, uint64_t y)
488 {
489 int el = arm_current_el(env);
490 if (!pauth_key_enabled(env, el, SCTLR_EnDA)) {
491 return x;
492 }
493 pauth_check_trap(env, el, GETPC());
494 return pauth_auth(env, x, y, &env->keys.apda, true, 0);
495 }
496
497 uint64_t HELPER(autdb)(CPUARMState *env, uint64_t x, uint64_t y)
498 {
499 int el = arm_current_el(env);
500 if (!pauth_key_enabled(env, el, SCTLR_EnDB)) {
501 return x;
502 }
503 pauth_check_trap(env, el, GETPC());
504 return pauth_auth(env, x, y, &env->keys.apdb, true, 1);
505 }
506
507 uint64_t HELPER(xpaci)(CPUARMState *env, uint64_t a)
508 {
509 return pauth_strip(env, a, false);
510 }
511
512 uint64_t HELPER(xpacd)(CPUARMState *env, uint64_t a)
513 {
514 return pauth_strip(env, a, true);
515 }
QEmu