?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
hw/arm/smmuv3: Support and advertise nesting
[qemu/ar7.git] / hw / arm / smmuv3.c
blob3db6c7c135780849a50b288eef6867eda5052244
1 /*
2 * Copyright (C) 2014-2016 Broadcom Corporation
3 * Copyright (c) 2017 Red Hat, Inc.
4 * Written by Prem Mallappa, Eric Auger
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, see <http://www.gnu.org/licenses/>.
17 */
18
19 #include "qemu/osdep.h"
20 #include "qemu/bitops.h"
21 #include "hw/irq.h"
22 #include "hw/sysbus.h"
23 #include "migration/vmstate.h"
24 #include "hw/qdev-properties.h"
25 #include "hw/qdev-core.h"
26 #include "hw/pci/pci.h"
27 #include "cpu.h"
28 #include "trace.h"
29 #include "qemu/log.h"
30 #include "qemu/error-report.h"
31 #include "qapi/error.h"
32
33 #include "hw/arm/smmuv3.h"
34 #include "smmuv3-internal.h"
35 #include "smmu-internal.h"
36
37 #define PTW_RECORD_FAULT(ptw_info, cfg) (((ptw_info).stage == SMMU_STAGE_1 && \
38 (cfg)->record_faults) || \
39 ((ptw_info).stage == SMMU_STAGE_2 && \
40 (cfg)->s2cfg.record_faults))
41
42 /**
43 * smmuv3_trigger_irq - pulse @irq if enabled and update
44 * GERROR register in case of GERROR interrupt
45 *
46 * @irq: irq type
47 * @gerror_mask: mask of gerrors to toggle (relevant if @irq is GERROR)
48 */
49 static void smmuv3_trigger_irq(SMMUv3State *s, SMMUIrq irq,
50 uint32_t gerror_mask)
51 {
52
53 bool pulse = false;
54
55 switch (irq) {
56 case SMMU_IRQ_EVTQ:
57 pulse = smmuv3_eventq_irq_enabled(s);
58 break;
59 case SMMU_IRQ_PRIQ:
60 qemu_log_mask(LOG_UNIMP, "PRI not yet supported\n");
61 break;
62 case SMMU_IRQ_CMD_SYNC:
63 pulse = true;
64 break;
65 case SMMU_IRQ_GERROR:
66 {
67 uint32_t pending = s->gerror ^ s->gerrorn;
68 uint32_t new_gerrors = ~pending & gerror_mask;
69
70 if (!new_gerrors) {
71 /* only toggle non pending errors */
72 return;
73 }
74 s->gerror ^= new_gerrors;
75 trace_smmuv3_write_gerror(new_gerrors, s->gerror);
76
77 pulse = smmuv3_gerror_irq_enabled(s);
78 break;
79 }
80 }
81 if (pulse) {
82 trace_smmuv3_trigger_irq(irq);
83 qemu_irq_pulse(s->irq[irq]);
84 }
85 }
86
87 static void smmuv3_write_gerrorn(SMMUv3State *s, uint32_t new_gerrorn)
88 {
89 uint32_t pending = s->gerror ^ s->gerrorn;
90 uint32_t toggled = s->gerrorn ^ new_gerrorn;
91
92 if (toggled & ~pending) {
93 qemu_log_mask(LOG_GUEST_ERROR,
94 "guest toggles non pending errors = 0x%x\n",
95 toggled & ~pending);
96 }
97
98 /*
99 * We do not raise any error in case guest toggles bits corresponding
100 * to not active IRQs (CONSTRAINED UNPREDICTABLE)
101 */
102 s->gerrorn = new_gerrorn;
103
104 trace_smmuv3_write_gerrorn(toggled & pending, s->gerrorn);
105 }
106
107 static inline MemTxResult queue_read(SMMUQueue *q, Cmd *cmd)
108 {
109 dma_addr_t addr = Q_CONS_ENTRY(q);
110 MemTxResult ret;
111 int i;
112
113 ret = dma_memory_read(&address_space_memory, addr, cmd, sizeof(Cmd),
114 MEMTXATTRS_UNSPECIFIED);
115 if (ret != MEMTX_OK) {
116 return ret;
117 }
118 for (i = 0; i < ARRAY_SIZE(cmd->word); i++) {
119 le32_to_cpus(&cmd->word[i]);
120 }
121 return ret;
122 }
123
124 static MemTxResult queue_write(SMMUQueue *q, Evt *evt_in)
125 {
126 dma_addr_t addr = Q_PROD_ENTRY(q);
127 MemTxResult ret;
128 Evt evt = *evt_in;
129 int i;
130
131 for (i = 0; i < ARRAY_SIZE(evt.word); i++) {
132 cpu_to_le32s(&evt.word[i]);
133 }
134 ret = dma_memory_write(&address_space_memory, addr, &evt, sizeof(Evt),
135 MEMTXATTRS_UNSPECIFIED);
136 if (ret != MEMTX_OK) {
137 return ret;
138 }
139
140 queue_prod_incr(q);
141 return MEMTX_OK;
142 }
143
144 static MemTxResult smmuv3_write_eventq(SMMUv3State *s, Evt *evt)
145 {
146 SMMUQueue *q = &s->eventq;
147 MemTxResult r;
148
149 if (!smmuv3_eventq_enabled(s)) {
150 return MEMTX_ERROR;
151 }
152
153 if (smmuv3_q_full(q)) {
154 return MEMTX_ERROR;
155 }
156
157 r = queue_write(q, evt);
158 if (r != MEMTX_OK) {
159 return r;
160 }
161
162 if (!smmuv3_q_empty(q)) {
163 smmuv3_trigger_irq(s, SMMU_IRQ_EVTQ, 0);
164 }
165 return MEMTX_OK;
166 }
167
168 void smmuv3_record_event(SMMUv3State *s, SMMUEventInfo *info)
169 {
170 Evt evt = {};
171 MemTxResult r;
172
173 if (!smmuv3_eventq_enabled(s)) {
174 return;
175 }
176
177 EVT_SET_TYPE(&evt, info->type);
178 EVT_SET_SID(&evt, info->sid);
179
180 switch (info->type) {
181 case SMMU_EVT_NONE:
182 return;
183 case SMMU_EVT_F_UUT:
184 EVT_SET_SSID(&evt, info->u.f_uut.ssid);
185 EVT_SET_SSV(&evt, info->u.f_uut.ssv);
186 EVT_SET_ADDR(&evt, info->u.f_uut.addr);
187 EVT_SET_RNW(&evt, info->u.f_uut.rnw);
188 EVT_SET_PNU(&evt, info->u.f_uut.pnu);
189 EVT_SET_IND(&evt, info->u.f_uut.ind);
190 break;
191 case SMMU_EVT_C_BAD_STREAMID:
192 EVT_SET_SSID(&evt, info->u.c_bad_streamid.ssid);
193 EVT_SET_SSV(&evt, info->u.c_bad_streamid.ssv);
194 break;
195 case SMMU_EVT_F_STE_FETCH:
196 EVT_SET_SSID(&evt, info->u.f_ste_fetch.ssid);
197 EVT_SET_SSV(&evt, info->u.f_ste_fetch.ssv);
198 EVT_SET_ADDR2(&evt, info->u.f_ste_fetch.addr);
199 break;
200 case SMMU_EVT_C_BAD_STE:
201 EVT_SET_SSID(&evt, info->u.c_bad_ste.ssid);
202 EVT_SET_SSV(&evt, info->u.c_bad_ste.ssv);
203 break;
204 case SMMU_EVT_F_STREAM_DISABLED:
205 break;
206 case SMMU_EVT_F_TRANS_FORBIDDEN:
207 EVT_SET_ADDR(&evt, info->u.f_transl_forbidden.addr);
208 EVT_SET_RNW(&evt, info->u.f_transl_forbidden.rnw);
209 break;
210 case SMMU_EVT_C_BAD_SUBSTREAMID:
211 EVT_SET_SSID(&evt, info->u.c_bad_substream.ssid);
212 break;
213 case SMMU_EVT_F_CD_FETCH:
214 EVT_SET_SSID(&evt, info->u.f_cd_fetch.ssid);
215 EVT_SET_SSV(&evt, info->u.f_cd_fetch.ssv);
216 EVT_SET_ADDR(&evt, info->u.f_cd_fetch.addr);
217 break;
218 case SMMU_EVT_C_BAD_CD:
219 EVT_SET_SSID(&evt, info->u.c_bad_cd.ssid);
220 EVT_SET_SSV(&evt, info->u.c_bad_cd.ssv);
221 break;
222 case SMMU_EVT_F_WALK_EABT:
223 case SMMU_EVT_F_TRANSLATION:
224 case SMMU_EVT_F_ADDR_SIZE:
225 case SMMU_EVT_F_ACCESS:
226 case SMMU_EVT_F_PERMISSION:
227 EVT_SET_STALL(&evt, info->u.f_walk_eabt.stall);
228 EVT_SET_STAG(&evt, info->u.f_walk_eabt.stag);
229 EVT_SET_SSID(&evt, info->u.f_walk_eabt.ssid);
230 EVT_SET_SSV(&evt, info->u.f_walk_eabt.ssv);
231 EVT_SET_S2(&evt, info->u.f_walk_eabt.s2);
232 EVT_SET_ADDR(&evt, info->u.f_walk_eabt.addr);
233 EVT_SET_RNW(&evt, info->u.f_walk_eabt.rnw);
234 EVT_SET_PNU(&evt, info->u.f_walk_eabt.pnu);
235 EVT_SET_IND(&evt, info->u.f_walk_eabt.ind);
236 EVT_SET_CLASS(&evt, info->u.f_walk_eabt.class);
237 EVT_SET_ADDR2(&evt, info->u.f_walk_eabt.addr2);
238 break;
239 case SMMU_EVT_F_CFG_CONFLICT:
240 EVT_SET_SSID(&evt, info->u.f_cfg_conflict.ssid);
241 EVT_SET_SSV(&evt, info->u.f_cfg_conflict.ssv);
242 break;
243 /* rest is not implemented */
244 case SMMU_EVT_F_BAD_ATS_TREQ:
245 case SMMU_EVT_F_TLB_CONFLICT:
246 case SMMU_EVT_E_PAGE_REQ:
247 default:
248 g_assert_not_reached();
249 }
250
251 trace_smmuv3_record_event(smmu_event_string(info->type), info->sid);
252 r = smmuv3_write_eventq(s, &evt);
253 if (r != MEMTX_OK) {
254 smmuv3_trigger_irq(s, SMMU_IRQ_GERROR, R_GERROR_EVENTQ_ABT_ERR_MASK);
255 }
256 info->recorded = true;
257 }
258
259 static void smmuv3_init_regs(SMMUv3State *s)
260 {
261 /* Based on sys property, the stages supported in smmu will be advertised.*/
262 if (s->stage && !strcmp("2", s->stage)) {
263 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S2P, 1);
264 } else if (s->stage && !strcmp("nested", s->stage)) {
265 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S1P, 1);
266 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S2P, 1);
267 } else {
268 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S1P, 1);
269 }
270
271 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TTF, 2); /* AArch64 PTW only */
272 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, COHACC, 1); /* IO coherent */
273 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, ASID16, 1); /* 16-bit ASID */
274 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, VMID16, 1); /* 16-bit VMID */
275 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TTENDIAN, 2); /* little endian */
276 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, STALL_MODEL, 1); /* No stall */
277 /* terminated transaction will always be aborted/error returned */
278 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TERM_MODEL, 1);
279 /* 2-level stream table supported */
280 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, STLEVEL, 1);
281
282 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, SIDSIZE, SMMU_IDR1_SIDSIZE);
283 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, EVENTQS, SMMU_EVENTQS);
284 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, CMDQS, SMMU_CMDQS);
285
286 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, HAD, 1);
287 if (FIELD_EX32(s->idr[0], IDR0, S2P)) {
288 /* XNX is a stage-2-specific feature */
289 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, XNX, 1);
290 }
291 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, RIL, 1);
292 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, BBML, 2);
293
294 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, OAS, SMMU_IDR5_OAS); /* 44 bits */
295 /* 4K, 16K and 64K granule support */
296 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN4K, 1);
297 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN16K, 1);
298 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN64K, 1);
299
300 s->cmdq.base = deposit64(s->cmdq.base, 0, 5, SMMU_CMDQS);
301 s->cmdq.prod = 0;
302 s->cmdq.cons = 0;
303 s->cmdq.entry_size = sizeof(struct Cmd);
304 s->eventq.base = deposit64(s->eventq.base, 0, 5, SMMU_EVENTQS);
305 s->eventq.prod = 0;
306 s->eventq.cons = 0;
307 s->eventq.entry_size = sizeof(struct Evt);
308
309 s->features = 0;
310 s->sid_split = 0;
311 s->aidr = 0x1;
312 s->cr[0] = 0;
313 s->cr0ack = 0;
314 s->irq_ctrl = 0;
315 s->gerror = 0;
316 s->gerrorn = 0;
317 s->statusr = 0;
318 s->gbpa = SMMU_GBPA_RESET_VAL;
319 }
320
321 static int smmu_get_ste(SMMUv3State *s, dma_addr_t addr, STE *buf,
322 SMMUEventInfo *event)
323 {
324 int ret, i;
325
326 trace_smmuv3_get_ste(addr);
327 /* TODO: guarantee 64-bit single-copy atomicity */
328 ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf),
329 MEMTXATTRS_UNSPECIFIED);
330 if (ret != MEMTX_OK) {
331 qemu_log_mask(LOG_GUEST_ERROR,
332 "Cannot fetch pte at address=0x%"PRIx64"\n", addr);
333 event->type = SMMU_EVT_F_STE_FETCH;
334 event->u.f_ste_fetch.addr = addr;
335 return -EINVAL;
336 }
337 for (i = 0; i < ARRAY_SIZE(buf->word); i++) {
338 le32_to_cpus(&buf->word[i]);
339 }
340 return 0;
341
342 }
343
344 static SMMUTranslationStatus smmuv3_do_translate(SMMUv3State *s, hwaddr addr,
345 SMMUTransCfg *cfg,
346 SMMUEventInfo *event,
347 IOMMUAccessFlags flag,
348 SMMUTLBEntry **out_entry,
349 SMMUTranslationClass class);
350 /* @ssid > 0 not supported yet */
351 static int smmu_get_cd(SMMUv3State *s, STE *ste, SMMUTransCfg *cfg,
352 uint32_t ssid, CD *buf, SMMUEventInfo *event)
353 {
354 dma_addr_t addr = STE_CTXPTR(ste);
355 int ret, i;
356 SMMUTranslationStatus status;
357 SMMUTLBEntry *entry;
358
359 trace_smmuv3_get_cd(addr);
360
361 if (cfg->stage == SMMU_NESTED) {
362 status = smmuv3_do_translate(s, addr, cfg, event,
363 IOMMU_RO, &entry, SMMU_CLASS_CD);
364
365 /* Same PTW faults are reported but with CLASS = CD. */
366 if (status != SMMU_TRANS_SUCCESS) {
367 return -EINVAL;
368 }
369
370 addr = CACHED_ENTRY_TO_ADDR(entry, addr);
371 }
372
373 /* TODO: guarantee 64-bit single-copy atomicity */
374 ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf),
375 MEMTXATTRS_UNSPECIFIED);
376 if (ret != MEMTX_OK) {
377 qemu_log_mask(LOG_GUEST_ERROR,
378 "Cannot fetch pte at address=0x%"PRIx64"\n", addr);
379 event->type = SMMU_EVT_F_CD_FETCH;
380 event->u.f_ste_fetch.addr = addr;
381 return -EINVAL;
382 }
383 for (i = 0; i < ARRAY_SIZE(buf->word); i++) {
384 le32_to_cpus(&buf->word[i]);
385 }
386 return 0;
387 }
388
389 /*
390 * Max valid value is 39 when SMMU_IDR3.STT == 0.
391 * In architectures after SMMUv3.0:
392 * - If STE.S2TG selects a 4KB or 16KB granule, the minimum valid value for this
393 * field is MAX(16, 64-IAS)
394 * - If STE.S2TG selects a 64KB granule, the minimum valid value for this field
395 * is (64-IAS).
396 * As we only support AA64, IAS = OAS.
397 */
398 static bool s2t0sz_valid(SMMUTransCfg *cfg)
399 {
400 if (cfg->s2cfg.tsz > 39) {
401 return false;
402 }
403
404 if (cfg->s2cfg.granule_sz == 16) {
405 return (cfg->s2cfg.tsz >= 64 - oas2bits(SMMU_IDR5_OAS));
406 }
407
408 return (cfg->s2cfg.tsz >= MAX(64 - oas2bits(SMMU_IDR5_OAS), 16));
409 }
410
411 /*
412 * Return true if s2 page table config is valid.
413 * This checks with the configured start level, ias_bits and granularity we can
414 * have a valid page table as described in ARM ARM D8.2 Translation process.
415 * The idea here is to see for the highest possible number of IPA bits, how
416 * many concatenated tables we would need, if it is more than 16, then this is
417 * not possible.
418 */
419 static bool s2_pgtable_config_valid(uint8_t sl0, uint8_t t0sz, uint8_t gran)
420 {
421 int level = get_start_level(sl0, gran);
422 uint64_t ipa_bits = 64 - t0sz;
423 uint64_t max_ipa = (1ULL << ipa_bits) - 1;
424 int nr_concat = pgd_concat_idx(level, gran, max_ipa) + 1;
425
426 return nr_concat <= VMSA_MAX_S2_CONCAT;
427 }
428
429 static int decode_ste_s2_cfg(SMMUTransCfg *cfg, STE *ste)
430 {
431 if (STE_S2AA64(ste) == 0x0) {
432 qemu_log_mask(LOG_UNIMP,
433 "SMMUv3 AArch32 tables not supported\n");
434 g_assert_not_reached();
435 }
436
437 switch (STE_S2TG(ste)) {
438 case 0x0: /* 4KB */
439 cfg->s2cfg.granule_sz = 12;
440 break;
441 case 0x1: /* 64KB */
442 cfg->s2cfg.granule_sz = 16;
443 break;
444 case 0x2: /* 16KB */
445 cfg->s2cfg.granule_sz = 14;
446 break;
447 default:
448 qemu_log_mask(LOG_GUEST_ERROR,
449 "SMMUv3 bad STE S2TG: %x\n", STE_S2TG(ste));
450 goto bad_ste;
451 }
452
453 cfg->s2cfg.vttb = STE_S2TTB(ste);
454
455 cfg->s2cfg.sl0 = STE_S2SL0(ste);
456 /* FEAT_TTST not supported. */
457 if (cfg->s2cfg.sl0 == 0x3) {
458 qemu_log_mask(LOG_UNIMP, "SMMUv3 S2SL0 = 0x3 has no meaning!\n");
459 goto bad_ste;
460 }
461
462 /* For AA64, The effective S2PS size is capped to the OAS. */
463 cfg->s2cfg.eff_ps = oas2bits(MIN(STE_S2PS(ste), SMMU_IDR5_OAS));
464 /*
465 * It is ILLEGAL for the address in S2TTB to be outside the range
466 * described by the effective S2PS value.
467 */
468 if (cfg->s2cfg.vttb & ~(MAKE_64BIT_MASK(0, cfg->s2cfg.eff_ps))) {
469 qemu_log_mask(LOG_GUEST_ERROR,
470 "SMMUv3 S2TTB too large 0x%" PRIx64
471 ", effective PS %d bits\n",
472 cfg->s2cfg.vttb, cfg->s2cfg.eff_ps);
473 goto bad_ste;
474 }
475
476 cfg->s2cfg.tsz = STE_S2T0SZ(ste);
477
478 if (!s2t0sz_valid(cfg)) {
479 qemu_log_mask(LOG_GUEST_ERROR, "SMMUv3 bad STE S2T0SZ = %d\n",
480 cfg->s2cfg.tsz);
481 goto bad_ste;
482 }
483
484 if (!s2_pgtable_config_valid(cfg->s2cfg.sl0, cfg->s2cfg.tsz,
485 cfg->s2cfg.granule_sz)) {
486 qemu_log_mask(LOG_GUEST_ERROR,
487 "SMMUv3 STE stage 2 config not valid!\n");
488 goto bad_ste;
489 }
490
491 /* Only LE supported(IDR0.TTENDIAN). */
492 if (STE_S2ENDI(ste)) {
493 qemu_log_mask(LOG_GUEST_ERROR,
494 "SMMUv3 STE_S2ENDI only supports LE!\n");
495 goto bad_ste;
496 }
497
498 cfg->s2cfg.affd = STE_S2AFFD(ste);
499
500 cfg->s2cfg.record_faults = STE_S2R(ste);
501 /* As stall is not supported. */
502 if (STE_S2S(ste)) {
503 qemu_log_mask(LOG_UNIMP, "SMMUv3 Stall not implemented!\n");
504 goto bad_ste;
505 }
506
507 return 0;
508
509 bad_ste:
510 return -EINVAL;
511 }
512
513 static void decode_ste_config(SMMUTransCfg *cfg, uint32_t config)
514 {
515
516 if (STE_CFG_ABORT(config)) {
517 cfg->aborted = true;
518 return;
519 }
520 if (STE_CFG_BYPASS(config)) {
521 cfg->bypassed = true;
522 return;
523 }
524
525 if (STE_CFG_S1_ENABLED(config)) {
526 cfg->stage = SMMU_STAGE_1;
527 }
528
529 if (STE_CFG_S2_ENABLED(config)) {
530 cfg->stage |= SMMU_STAGE_2;
531 }
532 }
533
534 /* Returns < 0 in case of invalid STE, 0 otherwise */
535 static int decode_ste(SMMUv3State *s, SMMUTransCfg *cfg,
536 STE *ste, SMMUEventInfo *event)
537 {
538 uint32_t config;
539 int ret;
540
541 if (!STE_VALID(ste)) {
542 if (!event->inval_ste_allowed) {
543 qemu_log_mask(LOG_GUEST_ERROR, "invalid STE\n");
544 }
545 goto bad_ste;
546 }
547
548 config = STE_CONFIG(ste);
549
550 decode_ste_config(cfg, config);
551
552 if (cfg->aborted || cfg->bypassed) {
553 return 0;
554 }
555
556 /*
557 * If a stage is enabled in SW while not advertised, throw bad ste
558 * according to user manual(IHI0070E) "5.2 Stream Table Entry".
559 */
560 if (!STAGE1_SUPPORTED(s) && STE_CFG_S1_ENABLED(config)) {
561 qemu_log_mask(LOG_GUEST_ERROR, "SMMUv3 S1 used but not supported.\n");
562 goto bad_ste;
563 }
564 if (!STAGE2_SUPPORTED(s) && STE_CFG_S2_ENABLED(config)) {
565 qemu_log_mask(LOG_GUEST_ERROR, "SMMUv3 S2 used but not supported.\n");
566 goto bad_ste;
567 }
568
569 if (STAGE2_SUPPORTED(s)) {
570 /* VMID is considered even if s2 is disabled. */
571 cfg->s2cfg.vmid = STE_S2VMID(ste);
572 } else {
573 /* Default to -1 */
574 cfg->s2cfg.vmid = -1;
575 }
576
577 if (STE_CFG_S2_ENABLED(config)) {
578 /*
579 * Stage-1 OAS defaults to OAS even if not enabled as it would be used
580 * in input address check for stage-2.
581 */
582 cfg->oas = oas2bits(SMMU_IDR5_OAS);
583 ret = decode_ste_s2_cfg(cfg, ste);
584 if (ret) {
585 goto bad_ste;
586 }
587 }
588
589 if (STE_S1CDMAX(ste) != 0) {
590 qemu_log_mask(LOG_UNIMP,
591 "SMMUv3 does not support multiple context descriptors yet\n");
592 goto bad_ste;
593 }
594
595 if (STE_S1STALLD(ste)) {
596 qemu_log_mask(LOG_UNIMP,
597 "SMMUv3 S1 stalling fault model not allowed yet\n");
598 goto bad_ste;
599 }
600 return 0;
601
602 bad_ste:
603 event->type = SMMU_EVT_C_BAD_STE;
604 return -EINVAL;
605 }
606
607 /**
608 * smmu_find_ste - Return the stream table entry associated
609 * to the sid
610 *
611 * @s: smmuv3 handle
612 * @sid: stream ID
613 * @ste: returned stream table entry
614 * @event: handle to an event info
615 *
616 * Supports linear and 2-level stream table
617 * Return 0 on success, -EINVAL otherwise
618 */
619 static int smmu_find_ste(SMMUv3State *s, uint32_t sid, STE *ste,
620 SMMUEventInfo *event)
621 {
622 dma_addr_t addr, strtab_base;
623 uint32_t log2size;
624 int strtab_size_shift;
625 int ret;
626
627 trace_smmuv3_find_ste(sid, s->features, s->sid_split);
628 log2size = FIELD_EX32(s->strtab_base_cfg, STRTAB_BASE_CFG, LOG2SIZE);
629 /*
630 * Check SID range against both guest-configured and implementation limits
631 */
632 if (sid >= (1 << MIN(log2size, SMMU_IDR1_SIDSIZE))) {
633 event->type = SMMU_EVT_C_BAD_STREAMID;
634 return -EINVAL;
635 }
636 if (s->features & SMMU_FEATURE_2LVL_STE) {
637 int l1_ste_offset, l2_ste_offset, max_l2_ste, span, i;
638 dma_addr_t l1ptr, l2ptr;
639 STEDesc l1std;
640
641 /*
642 * Align strtab base address to table size. For this purpose, assume it
643 * is not bounded by SMMU_IDR1_SIDSIZE.
644 */
645 strtab_size_shift = MAX(5, (int)log2size - s->sid_split - 1 + 3);
646 strtab_base = s->strtab_base & SMMU_BASE_ADDR_MASK &
647 ~MAKE_64BIT_MASK(0, strtab_size_shift);
648 l1_ste_offset = sid >> s->sid_split;
649 l2_ste_offset = sid & ((1 << s->sid_split) - 1);
650 l1ptr = (dma_addr_t)(strtab_base + l1_ste_offset * sizeof(l1std));
651 /* TODO: guarantee 64-bit single-copy atomicity */
652 ret = dma_memory_read(&address_space_memory, l1ptr, &l1std,
653 sizeof(l1std), MEMTXATTRS_UNSPECIFIED);
654 if (ret != MEMTX_OK) {
655 qemu_log_mask(LOG_GUEST_ERROR,
656 "Could not read L1PTR at 0X%"PRIx64"\n", l1ptr);
657 event->type = SMMU_EVT_F_STE_FETCH;
658 event->u.f_ste_fetch.addr = l1ptr;
659 return -EINVAL;
660 }
661 for (i = 0; i < ARRAY_SIZE(l1std.word); i++) {
662 le32_to_cpus(&l1std.word[i]);
663 }
664
665 span = L1STD_SPAN(&l1std);
666
667 if (!span) {
668 /* l2ptr is not valid */
669 if (!event->inval_ste_allowed) {
670 qemu_log_mask(LOG_GUEST_ERROR,
671 "invalid sid=%d (L1STD span=0)\n", sid);
672 }
673 event->type = SMMU_EVT_C_BAD_STREAMID;
674 return -EINVAL;
675 }
676 max_l2_ste = (1 << span) - 1;
677 l2ptr = l1std_l2ptr(&l1std);
678 trace_smmuv3_find_ste_2lvl(s->strtab_base, l1ptr, l1_ste_offset,
679 l2ptr, l2_ste_offset, max_l2_ste);
680 if (l2_ste_offset > max_l2_ste) {
681 qemu_log_mask(LOG_GUEST_ERROR,
682 "l2_ste_offset=%d > max_l2_ste=%d\n",
683 l2_ste_offset, max_l2_ste);
684 event->type = SMMU_EVT_C_BAD_STE;
685 return -EINVAL;
686 }
687 addr = l2ptr + l2_ste_offset * sizeof(*ste);
688 } else {
689 strtab_size_shift = log2size + 5;
690 strtab_base = s->strtab_base & SMMU_BASE_ADDR_MASK &
691 ~MAKE_64BIT_MASK(0, strtab_size_shift);
692 addr = strtab_base + sid * sizeof(*ste);
693 }
694
695 if (smmu_get_ste(s, addr, ste, event)) {
696 return -EINVAL;
697 }
698
699 return 0;
700 }
701
702 static int decode_cd(SMMUv3State *s, SMMUTransCfg *cfg,
703 CD *cd, SMMUEventInfo *event)
704 {
705 int ret = -EINVAL;
706 int i;
707 SMMUTranslationStatus status;
708 SMMUTLBEntry *entry;
709
710 if (!CD_VALID(cd) || !CD_AARCH64(cd)) {
711 goto bad_cd;
712 }
713 if (!CD_A(cd)) {
714 goto bad_cd; /* SMMU_IDR0.TERM_MODEL == 1 */
715 }
716 if (CD_S(cd)) {
717 goto bad_cd; /* !STE_SECURE && SMMU_IDR0.STALL_MODEL == 1 */
718 }
719 if (CD_HA(cd) || CD_HD(cd)) {
720 goto bad_cd; /* HTTU = 0 */
721 }
722
723 /* we support only those at the moment */
724 cfg->aa64 = true;
725
726 cfg->oas = oas2bits(CD_IPS(cd));
727 cfg->oas = MIN(oas2bits(SMMU_IDR5_OAS), cfg->oas);
728 cfg->tbi = CD_TBI(cd);
729 cfg->asid = CD_ASID(cd);
730 cfg->affd = CD_AFFD(cd);
731
732 trace_smmuv3_decode_cd(cfg->oas);
733
734 /* decode data dependent on TT */
735 for (i = 0; i <= 1; i++) {
736 int tg, tsz;
737 SMMUTransTableInfo *tt = &cfg->tt[i];
738
739 cfg->tt[i].disabled = CD_EPD(cd, i);
740 if (cfg->tt[i].disabled) {
741 continue;
742 }
743
744 tsz = CD_TSZ(cd, i);
745 if (tsz < 16 || tsz > 39) {
746 goto bad_cd;
747 }
748
749 tg = CD_TG(cd, i);
750 tt->granule_sz = tg2granule(tg, i);
751 if ((tt->granule_sz != 12 && tt->granule_sz != 14 &&
752 tt->granule_sz != 16) || CD_ENDI(cd)) {
753 goto bad_cd;
754 }
755
756 tt->tsz = tsz;
757 tt->ttb = CD_TTB(cd, i);
758
759 if (tt->ttb & ~(MAKE_64BIT_MASK(0, cfg->oas))) {
760 goto bad_cd;
761 }
762
763 /* Translate the TTBx, from IPA to PA if nesting is enabled. */
764 if (cfg->stage == SMMU_NESTED) {
765 status = smmuv3_do_translate(s, tt->ttb, cfg, event, IOMMU_RO,
766 &entry, SMMU_CLASS_TT);
767 /*
768 * Same PTW faults are reported but with CLASS = TT.
769 * If TTBx is larger than the effective stage 1 output addres
770 * size, it reports C_BAD_CD, which is handled by the above case.
771 */
772 if (status != SMMU_TRANS_SUCCESS) {
773 return -EINVAL;
774 }
775 tt->ttb = CACHED_ENTRY_TO_ADDR(entry, tt->ttb);
776 }
777
778 tt->had = CD_HAD(cd, i);
779 trace_smmuv3_decode_cd_tt(i, tt->tsz, tt->ttb, tt->granule_sz, tt->had);
780 }
781
782 cfg->record_faults = CD_R(cd);
783
784 return 0;
785
786 bad_cd:
787 event->type = SMMU_EVT_C_BAD_CD;
788 return ret;
789 }
790
791 /**
792 * smmuv3_decode_config - Prepare the translation configuration
793 * for the @mr iommu region
794 * @mr: iommu memory region the translation config must be prepared for
795 * @cfg: output translation configuration which is populated through
796 * the different configuration decoding steps
797 * @event: must be zero'ed by the caller
798 *
799 * return < 0 in case of config decoding error (@event is filled
800 * accordingly). Return 0 otherwise.
801 */
802 static int smmuv3_decode_config(IOMMUMemoryRegion *mr, SMMUTransCfg *cfg,
803 SMMUEventInfo *event)
804 {
805 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
806 uint32_t sid = smmu_get_sid(sdev);
807 SMMUv3State *s = sdev->smmu;
808 int ret;
809 STE ste;
810 CD cd;
811
812 /* ASID defaults to -1 (if s1 is not supported). */
813 cfg->asid = -1;
814
815 ret = smmu_find_ste(s, sid, &ste, event);
816 if (ret) {
817 return ret;
818 }
819
820 ret = decode_ste(s, cfg, &ste, event);
821 if (ret) {
822 return ret;
823 }
824
825 if (cfg->aborted || cfg->bypassed || (cfg->stage == SMMU_STAGE_2)) {
826 return 0;
827 }
828
829 ret = smmu_get_cd(s, &ste, cfg, 0 /* ssid */, &cd, event);
830 if (ret) {
831 return ret;
832 }
833
834 return decode_cd(s, cfg, &cd, event);
835 }
836
837 /**
838 * smmuv3_get_config - Look up for a cached copy of configuration data for
839 * @sdev and on cache miss performs a configuration structure decoding from
840 * guest RAM.
841 *
842 * @sdev: SMMUDevice handle
843 * @event: output event info
844 *
845 * The configuration cache contains data resulting from both STE and CD
846 * decoding under the form of an SMMUTransCfg struct. The hash table is indexed
847 * by the SMMUDevice handle.
848 */
849 static SMMUTransCfg *smmuv3_get_config(SMMUDevice *sdev, SMMUEventInfo *event)
850 {
851 SMMUv3State *s = sdev->smmu;
852 SMMUState *bc = &s->smmu_state;
853 SMMUTransCfg *cfg;
854
855 cfg = g_hash_table_lookup(bc->configs, sdev);
856 if (cfg) {
857 sdev->cfg_cache_hits++;
858 trace_smmuv3_config_cache_hit(smmu_get_sid(sdev),
859 sdev->cfg_cache_hits, sdev->cfg_cache_misses,
860 100 * sdev->cfg_cache_hits /
861 (sdev->cfg_cache_hits + sdev->cfg_cache_misses));
862 } else {
863 sdev->cfg_cache_misses++;
864 trace_smmuv3_config_cache_miss(smmu_get_sid(sdev),
865 sdev->cfg_cache_hits, sdev->cfg_cache_misses,
866 100 * sdev->cfg_cache_hits /
867 (sdev->cfg_cache_hits + sdev->cfg_cache_misses));
868 cfg = g_new0(SMMUTransCfg, 1);
869
870 if (!smmuv3_decode_config(&sdev->iommu, cfg, event)) {
871 g_hash_table_insert(bc->configs, sdev, cfg);
872 } else {
873 g_free(cfg);
874 cfg = NULL;
875 }
876 }
877 return cfg;
878 }
879
880 static void smmuv3_flush_config(SMMUDevice *sdev)
881 {
882 SMMUv3State *s = sdev->smmu;
883 SMMUState *bc = &s->smmu_state;
884
885 trace_smmuv3_config_cache_inv(smmu_get_sid(sdev));
886 g_hash_table_remove(bc->configs, sdev);
887 }
888
889 /* Do translation with TLB lookup. */
890 static SMMUTranslationStatus smmuv3_do_translate(SMMUv3State *s, hwaddr addr,
891 SMMUTransCfg *cfg,
892 SMMUEventInfo *event,
893 IOMMUAccessFlags flag,
894 SMMUTLBEntry **out_entry,
895 SMMUTranslationClass class)
896 {
897 SMMUPTWEventInfo ptw_info = {};
898 SMMUState *bs = ARM_SMMU(s);
899 SMMUTLBEntry *cached_entry = NULL;
900 int asid, stage;
901 bool desc_s2_translation = class != SMMU_CLASS_IN;
902
903 /*
904 * The function uses the argument class to identify which stage is used:
905 * - CLASS = IN: Means an input translation, determine the stage from STE.
906 * - CLASS = CD: Means the addr is an IPA of the CD, and it would be
907 * translated using the stage-2.
908 * - CLASS = TT: Means the addr is an IPA of the stage-1 translation table
909 * and it would be translated using the stage-2.
910 * For the last 2 cases instead of having intrusive changes in the common
911 * logic, we modify the cfg to be a stage-2 translation only in case of
912 * nested, and then restore it after.
913 */
914 if (desc_s2_translation) {
915 asid = cfg->asid;
916 stage = cfg->stage;
917 cfg->asid = -1;
918 cfg->stage = SMMU_STAGE_2;
919 }
920
921 cached_entry = smmu_translate(bs, cfg, addr, flag, &ptw_info);
922
923 if (desc_s2_translation) {
924 cfg->asid = asid;
925 cfg->stage = stage;
926 }
927
928 if (!cached_entry) {
929 /* All faults from PTW has S2 field. */
930 event->u.f_walk_eabt.s2 = (ptw_info.stage == SMMU_STAGE_2);
931 /*
932 * Fault class is set as follows based on "class" input to
933 * the function and to "ptw_info" from "smmu_translate()"
934 * For stage-1:
935 * - EABT => CLASS_TT (hardcoded)
936 * - other events => CLASS_IN (input to function)
937 * For stage-2 => CLASS_IN (input to function)
938 * For nested, for all events:
939 * - CD fetch => CLASS_CD (input to function)
940 * - walking stage 1 translation table => CLASS_TT (from
941 * is_ipa_descriptor or input in case of TTBx)
942 * - s2 translation => CLASS_IN (input to function)
943 */
944 class = ptw_info.is_ipa_descriptor ? SMMU_CLASS_TT : class;
945 switch (ptw_info.type) {
946 case SMMU_PTW_ERR_WALK_EABT:
947 event->type = SMMU_EVT_F_WALK_EABT;
948 event->u.f_walk_eabt.rnw = flag & 0x1;
949 event->u.f_walk_eabt.class = (ptw_info.stage == SMMU_STAGE_2) ?
950 class : SMMU_CLASS_TT;
951 event->u.f_walk_eabt.addr2 = ptw_info.addr;
952 break;
953 case SMMU_PTW_ERR_TRANSLATION:
954 if (PTW_RECORD_FAULT(ptw_info, cfg)) {
955 event->type = SMMU_EVT_F_TRANSLATION;
956 event->u.f_translation.addr2 = ptw_info.addr;
957 event->u.f_translation.class = class;
958 event->u.f_translation.rnw = flag & 0x1;
959 }
960 break;
961 case SMMU_PTW_ERR_ADDR_SIZE:
962 if (PTW_RECORD_FAULT(ptw_info, cfg)) {
963 event->type = SMMU_EVT_F_ADDR_SIZE;
964 event->u.f_addr_size.addr2 = ptw_info.addr;
965 event->u.f_addr_size.class = class;
966 event->u.f_addr_size.rnw = flag & 0x1;
967 }
968 break;
969 case SMMU_PTW_ERR_ACCESS:
970 if (PTW_RECORD_FAULT(ptw_info, cfg)) {
971 event->type = SMMU_EVT_F_ACCESS;
972 event->u.f_access.addr2 = ptw_info.addr;
973 event->u.f_access.class = class;
974 event->u.f_access.rnw = flag & 0x1;
975 }
976 break;
977 case SMMU_PTW_ERR_PERMISSION:
978 if (PTW_RECORD_FAULT(ptw_info, cfg)) {
979 event->type = SMMU_EVT_F_PERMISSION;
980 event->u.f_permission.addr2 = ptw_info.addr;
981 event->u.f_permission.class = class;
982 event->u.f_permission.rnw = flag & 0x1;
983 }
984 break;
985 default:
986 g_assert_not_reached();
987 }
988 return SMMU_TRANS_ERROR;
989 }
990 *out_entry = cached_entry;
991 return SMMU_TRANS_SUCCESS;
992 }
993
994 /*
995 * Sets the InputAddr for an SMMU_TRANS_ERROR, as it can't be
996 * set from all contexts, as smmuv3_get_config() can return
997 * translation faults in case of nested translation (for CD
998 * and TTBx). But in that case the iova is not known.
999 */
1000 static void smmuv3_fixup_event(SMMUEventInfo *event, hwaddr iova)
1001 {
1002 switch (event->type) {
1003 case SMMU_EVT_F_WALK_EABT:
1004 case SMMU_EVT_F_TRANSLATION:
1005 case SMMU_EVT_F_ADDR_SIZE:
1006 case SMMU_EVT_F_ACCESS:
1007 case SMMU_EVT_F_PERMISSION:
1008 event->u.f_walk_eabt.addr = iova;
1009 break;
1010 default:
1011 break;
1012 }
1013 }
1014
1015 /* Entry point to SMMU, does everything. */
1016 static IOMMUTLBEntry smmuv3_translate(IOMMUMemoryRegion *mr, hwaddr addr,
1017 IOMMUAccessFlags flag, int iommu_idx)
1018 {
1019 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
1020 SMMUv3State *s = sdev->smmu;
1021 uint32_t sid = smmu_get_sid(sdev);
1022 SMMUEventInfo event = {.type = SMMU_EVT_NONE,
1023 .sid = sid,
1024 .inval_ste_allowed = false};
1025 SMMUTranslationStatus status;
1026 SMMUTransCfg *cfg = NULL;
1027 IOMMUTLBEntry entry = {
1028 .target_as = &address_space_memory,
1029 .iova = addr,
1030 .translated_addr = addr,
1031 .addr_mask = ~(hwaddr)0,
1032 .perm = IOMMU_NONE,
1033 };
1034 SMMUTLBEntry *cached_entry = NULL;
1035
1036 qemu_mutex_lock(&s->mutex);
1037
1038 if (!smmu_enabled(s)) {
1039 if (FIELD_EX32(s->gbpa, GBPA, ABORT)) {
1040 status = SMMU_TRANS_ABORT;
1041 } else {
1042 status = SMMU_TRANS_DISABLE;
1043 }
1044 goto epilogue;
1045 }
1046
1047 cfg = smmuv3_get_config(sdev, &event);
1048 if (!cfg) {
1049 status = SMMU_TRANS_ERROR;
1050 goto epilogue;
1051 }
1052
1053 if (cfg->aborted) {
1054 status = SMMU_TRANS_ABORT;
1055 goto epilogue;
1056 }
1057
1058 if (cfg->bypassed) {
1059 status = SMMU_TRANS_BYPASS;
1060 goto epilogue;
1061 }
1062
1063 status = smmuv3_do_translate(s, addr, cfg, &event, flag,
1064 &cached_entry, SMMU_CLASS_IN);
1065
1066 epilogue:
1067 qemu_mutex_unlock(&s->mutex);
1068 switch (status) {
1069 case SMMU_TRANS_SUCCESS:
1070 entry.perm = cached_entry->entry.perm;
1071 entry.translated_addr = CACHED_ENTRY_TO_ADDR(cached_entry, addr);
1072 entry.addr_mask = cached_entry->entry.addr_mask;
1073 trace_smmuv3_translate_success(mr->parent_obj.name, sid, addr,
1074 entry.translated_addr, entry.perm,
1075 cfg->stage);
1076 break;
1077 case SMMU_TRANS_DISABLE:
1078 entry.perm = flag;
1079 entry.addr_mask = ~TARGET_PAGE_MASK;
1080 trace_smmuv3_translate_disable(mr->parent_obj.name, sid, addr,
1081 entry.perm);
1082 break;
1083 case SMMU_TRANS_BYPASS:
1084 entry.perm = flag;
1085 entry.addr_mask = ~TARGET_PAGE_MASK;
1086 trace_smmuv3_translate_bypass(mr->parent_obj.name, sid, addr,
1087 entry.perm);
1088 break;
1089 case SMMU_TRANS_ABORT:
1090 /* no event is recorded on abort */
1091 trace_smmuv3_translate_abort(mr->parent_obj.name, sid, addr,
1092 entry.perm);
1093 break;
1094 case SMMU_TRANS_ERROR:
1095 smmuv3_fixup_event(&event, addr);
1096 qemu_log_mask(LOG_GUEST_ERROR,
1097 "%s translation failed for iova=0x%"PRIx64" (%s)\n",
1098 mr->parent_obj.name, addr, smmu_event_string(event.type));
1099 smmuv3_record_event(s, &event);
1100 break;
1101 }
1102
1103 return entry;
1104 }
1105
1106 /**
1107 * smmuv3_notify_iova - call the notifier @n for a given
1108 * @asid and @iova tuple.
1109 *
1110 * @mr: IOMMU mr region handle
1111 * @n: notifier to be called
1112 * @asid: address space ID or negative value if we don't care
1113 * @vmid: virtual machine ID or negative value if we don't care
1114 * @iova: iova
1115 * @tg: translation granule (if communicated through range invalidation)
1116 * @num_pages: number of @granule sized pages (if tg != 0), otherwise 1
1117 * @stage: Which stage(1 or 2) is used
1118 */
1119 static void smmuv3_notify_iova(IOMMUMemoryRegion *mr,
1120 IOMMUNotifier *n,
1121 int asid, int vmid,
1122 dma_addr_t iova, uint8_t tg,
1123 uint64_t num_pages, int stage)
1124 {
1125 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
1126 SMMUEventInfo eventinfo = {.inval_ste_allowed = true};
1127 SMMUTransCfg *cfg = smmuv3_get_config(sdev, &eventinfo);
1128 IOMMUTLBEvent event;
1129 uint8_t granule;
1130
1131 if (!cfg) {
1132 return;
1133 }
1134
1135 /*
1136 * stage is passed from TLB invalidation commands which can be either
1137 * stage-1 or stage-2.
1138 * However, IOMMUTLBEvent only understands IOVA, for stage-1 or stage-2
1139 * SMMU instances we consider the input address as the IOVA, but when
1140 * nesting is used, we can't mix stage-1 and stage-2 addresses, so for
1141 * nesting only stage-1 is considered the IOVA and would be notified.
1142 */
1143 if ((stage == SMMU_STAGE_2) && (cfg->stage == SMMU_NESTED))
1144 return;
1145
1146 if (!tg) {
1147 SMMUTransTableInfo *tt;
1148
1149 if (asid >= 0 && cfg->asid != asid) {
1150 return;
1151 }
1152
1153 if (vmid >= 0 && cfg->s2cfg.vmid != vmid) {
1154 return;
1155 }
1156
1157 if (stage == SMMU_STAGE_1) {
1158 tt = select_tt(cfg, iova);
1159 if (!tt) {
1160 return;
1161 }
1162 granule = tt->granule_sz;
1163 } else {
1164 granule = cfg->s2cfg.granule_sz;
1165 }
1166
1167 } else {
1168 granule = tg * 2 + 10;
1169 }
1170
1171 event.type = IOMMU_NOTIFIER_UNMAP;
1172 event.entry.target_as = &address_space_memory;
1173 event.entry.iova = iova;
1174 event.entry.addr_mask = num_pages * (1 << granule) - 1;
1175 event.entry.perm = IOMMU_NONE;
1176
1177 memory_region_notify_iommu_one(n, &event);
1178 }
1179
1180 /* invalidate an asid/vmid/iova range tuple in all mr's */
1181 static void smmuv3_inv_notifiers_iova(SMMUState *s, int asid, int vmid,
1182 dma_addr_t iova, uint8_t tg,
1183 uint64_t num_pages, int stage)
1184 {
1185 SMMUDevice *sdev;
1186
1187 QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) {
1188 IOMMUMemoryRegion *mr = &sdev->iommu;
1189 IOMMUNotifier *n;
1190
1191 trace_smmuv3_inv_notifiers_iova(mr->parent_obj.name, asid, vmid,
1192 iova, tg, num_pages, stage);
1193
1194 IOMMU_NOTIFIER_FOREACH(n, mr) {
1195 smmuv3_notify_iova(mr, n, asid, vmid, iova, tg, num_pages, stage);
1196 }
1197 }
1198 }
1199
1200 static void smmuv3_range_inval(SMMUState *s, Cmd *cmd, SMMUStage stage)
1201 {
1202 dma_addr_t end, addr = CMD_ADDR(cmd);
1203 uint8_t type = CMD_TYPE(cmd);
1204 int vmid = -1;
1205 uint8_t scale = CMD_SCALE(cmd);
1206 uint8_t num = CMD_NUM(cmd);
1207 uint8_t ttl = CMD_TTL(cmd);
1208 bool leaf = CMD_LEAF(cmd);
1209 uint8_t tg = CMD_TG(cmd);
1210 uint64_t num_pages;
1211 uint8_t granule;
1212 int asid = -1;
1213 SMMUv3State *smmuv3 = ARM_SMMUV3(s);
1214
1215 /* Only consider VMID if stage-2 is supported. */
1216 if (STAGE2_SUPPORTED(smmuv3)) {
1217 vmid = CMD_VMID(cmd);
1218 }
1219
1220 if (type == SMMU_CMD_TLBI_NH_VA) {
1221 asid = CMD_ASID(cmd);
1222 }
1223
1224 if (!tg) {
1225 trace_smmuv3_range_inval(vmid, asid, addr, tg, 1, ttl, leaf, stage);
1226 smmuv3_inv_notifiers_iova(s, asid, vmid, addr, tg, 1, stage);
1227 if (stage == SMMU_STAGE_1) {
1228 smmu_iotlb_inv_iova(s, asid, vmid, addr, tg, 1, ttl);
1229 } else {
1230 smmu_iotlb_inv_ipa(s, vmid, addr, tg, 1, ttl);
1231 }
1232 return;
1233 }
1234
1235 /* RIL in use */
1236
1237 num_pages = (num + 1) * BIT_ULL(scale);
1238 granule = tg * 2 + 10;
1239
1240 /* Split invalidations into ^2 range invalidations */
1241 end = addr + (num_pages << granule) - 1;
1242
1243 while (addr != end + 1) {
1244 uint64_t mask = dma_aligned_pow2_mask(addr, end, 64);
1245
1246 num_pages = (mask + 1) >> granule;
1247 trace_smmuv3_range_inval(vmid, asid, addr, tg, num_pages,
1248 ttl, leaf, stage);
1249 smmuv3_inv_notifiers_iova(s, asid, vmid, addr, tg, num_pages, stage);
1250 if (stage == SMMU_STAGE_1) {
1251 smmu_iotlb_inv_iova(s, asid, vmid, addr, tg, num_pages, ttl);
1252 } else {
1253 smmu_iotlb_inv_ipa(s, vmid, addr, tg, num_pages, ttl);
1254 }
1255 addr += mask + 1;
1256 }
1257 }
1258
1259 static gboolean
1260 smmuv3_invalidate_ste(gpointer key, gpointer value, gpointer user_data)
1261 {
1262 SMMUDevice *sdev = (SMMUDevice *)key;
1263 uint32_t sid = smmu_get_sid(sdev);
1264 SMMUSIDRange *sid_range = (SMMUSIDRange *)user_data;
1265
1266 if (sid < sid_range->start || sid > sid_range->end) {
1267 return false;
1268 }
1269 trace_smmuv3_config_cache_inv(sid);
1270 return true;
1271 }
1272
1273 static int smmuv3_cmdq_consume(SMMUv3State *s)
1274 {
1275 SMMUState *bs = ARM_SMMU(s);
1276 SMMUCmdError cmd_error = SMMU_CERROR_NONE;
1277 SMMUQueue *q = &s->cmdq;
1278 SMMUCommandType type = 0;
1279
1280 if (!smmuv3_cmdq_enabled(s)) {
1281 return 0;
1282 }
1283 /*
1284 * some commands depend on register values, typically CR0. In case those
1285 * register values change while handling the command, spec says it
1286 * is UNPREDICTABLE whether the command is interpreted under the new
1287 * or old value.
1288 */
1289
1290 while (!smmuv3_q_empty(q)) {
1291 uint32_t pending = s->gerror ^ s->gerrorn;
1292 Cmd cmd;
1293
1294 trace_smmuv3_cmdq_consume(Q_PROD(q), Q_CONS(q),
1295 Q_PROD_WRAP(q), Q_CONS_WRAP(q));
1296
1297 if (FIELD_EX32(pending, GERROR, CMDQ_ERR)) {
1298 break;
1299 }
1300
1301 if (queue_read(q, &cmd) != MEMTX_OK) {
1302 cmd_error = SMMU_CERROR_ABT;
1303 break;
1304 }
1305
1306 type = CMD_TYPE(&cmd);
1307
1308 trace_smmuv3_cmdq_opcode(smmu_cmd_string(type));
1309
1310 qemu_mutex_lock(&s->mutex);
1311 switch (type) {
1312 case SMMU_CMD_SYNC:
1313 if (CMD_SYNC_CS(&cmd) & CMD_SYNC_SIG_IRQ) {
1314 smmuv3_trigger_irq(s, SMMU_IRQ_CMD_SYNC, 0);
1315 }
1316 break;
1317 case SMMU_CMD_PREFETCH_CONFIG:
1318 case SMMU_CMD_PREFETCH_ADDR:
1319 break;
1320 case SMMU_CMD_CFGI_STE:
1321 {
1322 uint32_t sid = CMD_SID(&cmd);
1323 SMMUDevice *sdev = smmu_find_sdev(bs, sid);
1324
1325 if (CMD_SSEC(&cmd)) {
1326 cmd_error = SMMU_CERROR_ILL;
1327 break;
1328 }
1329
1330 if (!sdev) {
1331 break;
1332 }
1333
1334 trace_smmuv3_cmdq_cfgi_ste(sid);
1335 smmuv3_flush_config(sdev);
1336
1337 break;
1338 }
1339 case SMMU_CMD_CFGI_STE_RANGE: /* same as SMMU_CMD_CFGI_ALL */
1340 {
1341 uint32_t sid = CMD_SID(&cmd), mask;
1342 uint8_t range = CMD_STE_RANGE(&cmd);
1343 SMMUSIDRange sid_range;
1344
1345 if (CMD_SSEC(&cmd)) {
1346 cmd_error = SMMU_CERROR_ILL;
1347 break;
1348 }
1349
1350 mask = (1ULL << (range + 1)) - 1;
1351 sid_range.start = sid & ~mask;
1352 sid_range.end = sid_range.start + mask;
1353
1354 trace_smmuv3_cmdq_cfgi_ste_range(sid_range.start, sid_range.end);
1355 g_hash_table_foreach_remove(bs->configs, smmuv3_invalidate_ste,
1356 &sid_range);
1357 break;
1358 }
1359 case SMMU_CMD_CFGI_CD:
1360 case SMMU_CMD_CFGI_CD_ALL:
1361 {
1362 uint32_t sid = CMD_SID(&cmd);
1363 SMMUDevice *sdev = smmu_find_sdev(bs, sid);
1364
1365 if (CMD_SSEC(&cmd)) {
1366 cmd_error = SMMU_CERROR_ILL;
1367 break;
1368 }
1369
1370 if (!sdev) {
1371 break;
1372 }
1373
1374 trace_smmuv3_cmdq_cfgi_cd(sid);
1375 smmuv3_flush_config(sdev);
1376 break;
1377 }
1378 case SMMU_CMD_TLBI_NH_ASID:
1379 {
1380 int asid = CMD_ASID(&cmd);
1381 int vmid = -1;
1382
1383 if (!STAGE1_SUPPORTED(s)) {
1384 cmd_error = SMMU_CERROR_ILL;
1385 break;
1386 }
1387
1388 /*
1389 * VMID is only matched when stage 2 is supported, otherwise set it
1390 * to -1 as the value used for stage-1 only VMIDs.
1391 */
1392 if (STAGE2_SUPPORTED(s)) {
1393 vmid = CMD_VMID(&cmd);
1394 }
1395
1396 trace_smmuv3_cmdq_tlbi_nh_asid(asid);
1397 smmu_inv_notifiers_all(&s->smmu_state);
1398 smmu_iotlb_inv_asid_vmid(bs, asid, vmid);
1399 break;
1400 }
1401 case SMMU_CMD_TLBI_NH_ALL:
1402 {
1403 int vmid = -1;
1404
1405 if (!STAGE1_SUPPORTED(s)) {
1406 cmd_error = SMMU_CERROR_ILL;
1407 break;
1408 }
1409
1410 /*
1411 * If stage-2 is supported, invalidate for this VMID only, otherwise
1412 * invalidate the whole thing.
1413 */
1414 if (STAGE2_SUPPORTED(s)) {
1415 vmid = CMD_VMID(&cmd);
1416 trace_smmuv3_cmdq_tlbi_nh(vmid);
1417 smmu_iotlb_inv_vmid_s1(bs, vmid);
1418 break;
1419 }
1420 QEMU_FALLTHROUGH;
1421 }
1422 case SMMU_CMD_TLBI_NSNH_ALL:
1423 trace_smmuv3_cmdq_tlbi_nsnh();
1424 smmu_inv_notifiers_all(&s->smmu_state);
1425 smmu_iotlb_inv_all(bs);
1426 break;
1427 case SMMU_CMD_TLBI_NH_VAA:
1428 case SMMU_CMD_TLBI_NH_VA:
1429 if (!STAGE1_SUPPORTED(s)) {
1430 cmd_error = SMMU_CERROR_ILL;
1431 break;
1432 }
1433 smmuv3_range_inval(bs, &cmd, SMMU_STAGE_1);
1434 break;
1435 case SMMU_CMD_TLBI_S12_VMALL:
1436 {
1437 int vmid = CMD_VMID(&cmd);
1438
1439 if (!STAGE2_SUPPORTED(s)) {
1440 cmd_error = SMMU_CERROR_ILL;
1441 break;
1442 }
1443
1444 trace_smmuv3_cmdq_tlbi_s12_vmid(vmid);
1445 smmu_inv_notifiers_all(&s->smmu_state);
1446 smmu_iotlb_inv_vmid(bs, vmid);
1447 break;
1448 }
1449 case SMMU_CMD_TLBI_S2_IPA:
1450 if (!STAGE2_SUPPORTED(s)) {
1451 cmd_error = SMMU_CERROR_ILL;
1452 break;
1453 }
1454 /*
1455 * As currently only either s1 or s2 are supported
1456 * we can reuse same function for s2.
1457 */
1458 smmuv3_range_inval(bs, &cmd, SMMU_STAGE_2);
1459 break;
1460 case SMMU_CMD_TLBI_EL3_ALL:
1461 case SMMU_CMD_TLBI_EL3_VA:
1462 case SMMU_CMD_TLBI_EL2_ALL:
1463 case SMMU_CMD_TLBI_EL2_ASID:
1464 case SMMU_CMD_TLBI_EL2_VA:
1465 case SMMU_CMD_TLBI_EL2_VAA:
1466 case SMMU_CMD_ATC_INV:
1467 case SMMU_CMD_PRI_RESP:
1468 case SMMU_CMD_RESUME:
1469 case SMMU_CMD_STALL_TERM:
1470 trace_smmuv3_unhandled_cmd(type);
1471 break;
1472 default:
1473 cmd_error = SMMU_CERROR_ILL;
1474 break;
1475 }
1476 qemu_mutex_unlock(&s->mutex);
1477 if (cmd_error) {
1478 if (cmd_error == SMMU_CERROR_ILL) {
1479 qemu_log_mask(LOG_GUEST_ERROR,
1480 "Illegal command type: %d\n", CMD_TYPE(&cmd));
1481 }
1482 break;
1483 }
1484 /*
1485 * We only increment the cons index after the completion of
1486 * the command. We do that because the SYNC returns immediately
1487 * and does not check the completion of previous commands
1488 */
1489 queue_cons_incr(q);
1490 }
1491
1492 if (cmd_error) {
1493 trace_smmuv3_cmdq_consume_error(smmu_cmd_string(type), cmd_error);
1494 smmu_write_cmdq_err(s, cmd_error);
1495 smmuv3_trigger_irq(s, SMMU_IRQ_GERROR, R_GERROR_CMDQ_ERR_MASK);
1496 }
1497
1498 trace_smmuv3_cmdq_consume_out(Q_PROD(q), Q_CONS(q),
1499 Q_PROD_WRAP(q), Q_CONS_WRAP(q));
1500
1501 return 0;
1502 }
1503
1504 static MemTxResult smmu_writell(SMMUv3State *s, hwaddr offset,
1505 uint64_t data, MemTxAttrs attrs)
1506 {
1507 switch (offset) {
1508 case A_GERROR_IRQ_CFG0:
1509 s->gerror_irq_cfg0 = data;
1510 return MEMTX_OK;
1511 case A_STRTAB_BASE:
1512 s->strtab_base = data;
1513 return MEMTX_OK;
1514 case A_CMDQ_BASE:
1515 s->cmdq.base = data;
1516 s->cmdq.log2size = extract64(s->cmdq.base, 0, 5);
1517 if (s->cmdq.log2size > SMMU_CMDQS) {
1518 s->cmdq.log2size = SMMU_CMDQS;
1519 }
1520 return MEMTX_OK;
1521 case A_EVENTQ_BASE:
1522 s->eventq.base = data;
1523 s->eventq.log2size = extract64(s->eventq.base, 0, 5);
1524 if (s->eventq.log2size > SMMU_EVENTQS) {
1525 s->eventq.log2size = SMMU_EVENTQS;
1526 }
1527 return MEMTX_OK;
1528 case A_EVENTQ_IRQ_CFG0:
1529 s->eventq_irq_cfg0 = data;
1530 return MEMTX_OK;
1531 default:
1532 qemu_log_mask(LOG_UNIMP,
1533 "%s Unexpected 64-bit access to 0x%"PRIx64" (WI)\n",
1534 __func__, offset);
1535 return MEMTX_OK;
1536 }
1537 }
1538
1539 static MemTxResult smmu_writel(SMMUv3State *s, hwaddr offset,
1540 uint64_t data, MemTxAttrs attrs)
1541 {
1542 switch (offset) {
1543 case A_CR0:
1544 s->cr[0] = data;
1545 s->cr0ack = data & ~SMMU_CR0_RESERVED;
1546 /* in case the command queue has been enabled */
1547 smmuv3_cmdq_consume(s);
1548 return MEMTX_OK;
1549 case A_CR1:
1550 s->cr[1] = data;
1551 return MEMTX_OK;
1552 case A_CR2:
1553 s->cr[2] = data;
1554 return MEMTX_OK;
1555 case A_IRQ_CTRL:
1556 s->irq_ctrl = data;
1557 return MEMTX_OK;
1558 case A_GERRORN:
1559 smmuv3_write_gerrorn(s, data);
1560 /*
1561 * By acknowledging the CMDQ_ERR, SW may notify cmds can
1562 * be processed again
1563 */
1564 smmuv3_cmdq_consume(s);
1565 return MEMTX_OK;
1566 case A_GERROR_IRQ_CFG0: /* 64b */
1567 s->gerror_irq_cfg0 = deposit64(s->gerror_irq_cfg0, 0, 32, data);
1568 return MEMTX_OK;
1569 case A_GERROR_IRQ_CFG0 + 4:
1570 s->gerror_irq_cfg0 = deposit64(s->gerror_irq_cfg0, 32, 32, data);
1571 return MEMTX_OK;
1572 case A_GERROR_IRQ_CFG1:
1573 s->gerror_irq_cfg1 = data;
1574 return MEMTX_OK;
1575 case A_GERROR_IRQ_CFG2:
1576 s->gerror_irq_cfg2 = data;
1577 return MEMTX_OK;
1578 case A_GBPA:
1579 /*
1580 * If UPDATE is not set, the write is ignored. This is the only
1581 * permitted behavior in SMMUv3.2 and later.
1582 */
1583 if (data & R_GBPA_UPDATE_MASK) {
1584 /* Ignore update bit as write is synchronous. */
1585 s->gbpa = data & ~R_GBPA_UPDATE_MASK;
1586 }
1587 return MEMTX_OK;
1588 case A_STRTAB_BASE: /* 64b */
1589 s->strtab_base = deposit64(s->strtab_base, 0, 32, data);
1590 return MEMTX_OK;
1591 case A_STRTAB_BASE + 4:
1592 s->strtab_base = deposit64(s->strtab_base, 32, 32, data);
1593 return MEMTX_OK;
1594 case A_STRTAB_BASE_CFG:
1595 s->strtab_base_cfg = data;
1596 if (FIELD_EX32(data, STRTAB_BASE_CFG, FMT) == 1) {
1597 s->sid_split = FIELD_EX32(data, STRTAB_BASE_CFG, SPLIT);
1598 s->features |= SMMU_FEATURE_2LVL_STE;
1599 }
1600 return MEMTX_OK;
1601 case A_CMDQ_BASE: /* 64b */
1602 s->cmdq.base = deposit64(s->cmdq.base, 0, 32, data);
1603 s->cmdq.log2size = extract64(s->cmdq.base, 0, 5);
1604 if (s->cmdq.log2size > SMMU_CMDQS) {
1605 s->cmdq.log2size = SMMU_CMDQS;
1606 }
1607 return MEMTX_OK;
1608 case A_CMDQ_BASE + 4: /* 64b */
1609 s->cmdq.base = deposit64(s->cmdq.base, 32, 32, data);
1610 return MEMTX_OK;
1611 case A_CMDQ_PROD:
1612 s->cmdq.prod = data;
1613 smmuv3_cmdq_consume(s);
1614 return MEMTX_OK;
1615 case A_CMDQ_CONS:
1616 s->cmdq.cons = data;
1617 return MEMTX_OK;
1618 case A_EVENTQ_BASE: /* 64b */
1619 s->eventq.base = deposit64(s->eventq.base, 0, 32, data);
1620 s->eventq.log2size = extract64(s->eventq.base, 0, 5);
1621 if (s->eventq.log2size > SMMU_EVENTQS) {
1622 s->eventq.log2size = SMMU_EVENTQS;
1623 }
1624 return MEMTX_OK;
1625 case A_EVENTQ_BASE + 4:
1626 s->eventq.base = deposit64(s->eventq.base, 32, 32, data);
1627 return MEMTX_OK;
1628 case A_EVENTQ_PROD:
1629 s->eventq.prod = data;
1630 return MEMTX_OK;
1631 case A_EVENTQ_CONS:
1632 s->eventq.cons = data;
1633 return MEMTX_OK;
1634 case A_EVENTQ_IRQ_CFG0: /* 64b */
1635 s->eventq_irq_cfg0 = deposit64(s->eventq_irq_cfg0, 0, 32, data);
1636 return MEMTX_OK;
1637 case A_EVENTQ_IRQ_CFG0 + 4:
1638 s->eventq_irq_cfg0 = deposit64(s->eventq_irq_cfg0, 32, 32, data);
1639 return MEMTX_OK;
1640 case A_EVENTQ_IRQ_CFG1:
1641 s->eventq_irq_cfg1 = data;
1642 return MEMTX_OK;
1643 case A_EVENTQ_IRQ_CFG2:
1644 s->eventq_irq_cfg2 = data;
1645 return MEMTX_OK;
1646 default:
1647 qemu_log_mask(LOG_UNIMP,
1648 "%s Unexpected 32-bit access to 0x%"PRIx64" (WI)\n",
1649 __func__, offset);
1650 return MEMTX_OK;
1651 }
1652 }
1653
1654 static MemTxResult smmu_write_mmio(void *opaque, hwaddr offset, uint64_t data,
1655 unsigned size, MemTxAttrs attrs)
1656 {
1657 SMMUState *sys = opaque;
1658 SMMUv3State *s = ARM_SMMUV3(sys);
1659 MemTxResult r;
1660
1661 /* CONSTRAINED UNPREDICTABLE choice to have page0/1 be exact aliases */
1662 offset &= ~0x10000;
1663
1664 switch (size) {
1665 case 8:
1666 r = smmu_writell(s, offset, data, attrs);
1667 break;
1668 case 4:
1669 r = smmu_writel(s, offset, data, attrs);
1670 break;
1671 default:
1672 r = MEMTX_ERROR;
1673 break;
1674 }
1675
1676 trace_smmuv3_write_mmio(offset, data, size, r);
1677 return r;
1678 }
1679
1680 static MemTxResult smmu_readll(SMMUv3State *s, hwaddr offset,
1681 uint64_t *data, MemTxAttrs attrs)
1682 {
1683 switch (offset) {
1684 case A_GERROR_IRQ_CFG0:
1685 *data = s->gerror_irq_cfg0;
1686 return MEMTX_OK;
1687 case A_STRTAB_BASE:
1688 *data = s->strtab_base;
1689 return MEMTX_OK;
1690 case A_CMDQ_BASE:
1691 *data = s->cmdq.base;
1692 return MEMTX_OK;
1693 case A_EVENTQ_BASE:
1694 *data = s->eventq.base;
1695 return MEMTX_OK;
1696 default:
1697 *data = 0;
1698 qemu_log_mask(LOG_UNIMP,
1699 "%s Unexpected 64-bit access to 0x%"PRIx64" (RAZ)\n",
1700 __func__, offset);
1701 return MEMTX_OK;
1702 }
1703 }
1704
1705 static MemTxResult smmu_readl(SMMUv3State *s, hwaddr offset,
1706 uint64_t *data, MemTxAttrs attrs)
1707 {
1708 switch (offset) {
1709 case A_IDREGS ... A_IDREGS + 0x2f:
1710 *data = smmuv3_idreg(offset - A_IDREGS);
1711 return MEMTX_OK;
1712 case A_IDR0 ... A_IDR5:
1713 *data = s->idr[(offset - A_IDR0) / 4];
1714 return MEMTX_OK;
1715 case A_IIDR:
1716 *data = s->iidr;
1717 return MEMTX_OK;
1718 case A_AIDR:
1719 *data = s->aidr;
1720 return MEMTX_OK;
1721 case A_CR0:
1722 *data = s->cr[0];
1723 return MEMTX_OK;
1724 case A_CR0ACK:
1725 *data = s->cr0ack;
1726 return MEMTX_OK;
1727 case A_CR1:
1728 *data = s->cr[1];
1729 return MEMTX_OK;
1730 case A_CR2:
1731 *data = s->cr[2];
1732 return MEMTX_OK;
1733 case A_STATUSR:
1734 *data = s->statusr;
1735 return MEMTX_OK;
1736 case A_GBPA:
1737 *data = s->gbpa;
1738 return MEMTX_OK;
1739 case A_IRQ_CTRL:
1740 case A_IRQ_CTRL_ACK:
1741 *data = s->irq_ctrl;
1742 return MEMTX_OK;
1743 case A_GERROR:
1744 *data = s->gerror;
1745 return MEMTX_OK;
1746 case A_GERRORN:
1747 *data = s->gerrorn;
1748 return MEMTX_OK;
1749 case A_GERROR_IRQ_CFG0: /* 64b */
1750 *data = extract64(s->gerror_irq_cfg0, 0, 32);
1751 return MEMTX_OK;
1752 case A_GERROR_IRQ_CFG0 + 4:
1753 *data = extract64(s->gerror_irq_cfg0, 32, 32);
1754 return MEMTX_OK;
1755 case A_GERROR_IRQ_CFG1:
1756 *data = s->gerror_irq_cfg1;
1757 return MEMTX_OK;
1758 case A_GERROR_IRQ_CFG2:
1759 *data = s->gerror_irq_cfg2;
1760 return MEMTX_OK;
1761 case A_STRTAB_BASE: /* 64b */
1762 *data = extract64(s->strtab_base, 0, 32);
1763 return MEMTX_OK;
1764 case A_STRTAB_BASE + 4: /* 64b */
1765 *data = extract64(s->strtab_base, 32, 32);
1766 return MEMTX_OK;
1767 case A_STRTAB_BASE_CFG:
1768 *data = s->strtab_base_cfg;
1769 return MEMTX_OK;
1770 case A_CMDQ_BASE: /* 64b */
1771 *data = extract64(s->cmdq.base, 0, 32);
1772 return MEMTX_OK;
1773 case A_CMDQ_BASE + 4:
1774 *data = extract64(s->cmdq.base, 32, 32);
1775 return MEMTX_OK;
1776 case A_CMDQ_PROD:
1777 *data = s->cmdq.prod;
1778 return MEMTX_OK;
1779 case A_CMDQ_CONS:
1780 *data = s->cmdq.cons;
1781 return MEMTX_OK;
1782 case A_EVENTQ_BASE: /* 64b */
1783 *data = extract64(s->eventq.base, 0, 32);
1784 return MEMTX_OK;
1785 case A_EVENTQ_BASE + 4: /* 64b */
1786 *data = extract64(s->eventq.base, 32, 32);
1787 return MEMTX_OK;
1788 case A_EVENTQ_PROD:
1789 *data = s->eventq.prod;
1790 return MEMTX_OK;
1791 case A_EVENTQ_CONS:
1792 *data = s->eventq.cons;
1793 return MEMTX_OK;
1794 default:
1795 *data = 0;
1796 qemu_log_mask(LOG_UNIMP,
1797 "%s unhandled 32-bit access at 0x%"PRIx64" (RAZ)\n",
1798 __func__, offset);
1799 return MEMTX_OK;
1800 }
1801 }
1802
1803 static MemTxResult smmu_read_mmio(void *opaque, hwaddr offset, uint64_t *data,
1804 unsigned size, MemTxAttrs attrs)
1805 {
1806 SMMUState *sys = opaque;
1807 SMMUv3State *s = ARM_SMMUV3(sys);
1808 MemTxResult r;
1809
1810 /* CONSTRAINED UNPREDICTABLE choice to have page0/1 be exact aliases */
1811 offset &= ~0x10000;
1812
1813 switch (size) {
1814 case 8:
1815 r = smmu_readll(s, offset, data, attrs);
1816 break;
1817 case 4:
1818 r = smmu_readl(s, offset, data, attrs);
1819 break;
1820 default:
1821 r = MEMTX_ERROR;
1822 break;
1823 }
1824
1825 trace_smmuv3_read_mmio(offset, *data, size, r);
1826 return r;
1827 }
1828
1829 static const MemoryRegionOps smmu_mem_ops = {
1830 .read_with_attrs = smmu_read_mmio,
1831 .write_with_attrs = smmu_write_mmio,
1832 .endianness = DEVICE_LITTLE_ENDIAN,
1833 .valid = {
1834 .min_access_size = 4,
1835 .max_access_size = 8,
1836 },
1837 .impl = {
1838 .min_access_size = 4,
1839 .max_access_size = 8,
1840 },
1841 };
1842
1843 static void smmu_init_irq(SMMUv3State *s, SysBusDevice *dev)
1844 {
1845 int i;
1846
1847 for (i = 0; i < ARRAY_SIZE(s->irq); i++) {
1848 sysbus_init_irq(dev, &s->irq[i]);
1849 }
1850 }
1851
1852 static void smmu_reset_hold(Object *obj, ResetType type)
1853 {
1854 SMMUv3State *s = ARM_SMMUV3(obj);
1855 SMMUv3Class *c = ARM_SMMUV3_GET_CLASS(s);
1856
1857 if (c->parent_phases.hold) {
1858 c->parent_phases.hold(obj, type);
1859 }
1860
1861 smmuv3_init_regs(s);
1862 }
1863
1864 static void smmu_realize(DeviceState *d, Error **errp)
1865 {
1866 SMMUState *sys = ARM_SMMU(d);
1867 SMMUv3State *s = ARM_SMMUV3(sys);
1868 SMMUv3Class *c = ARM_SMMUV3_GET_CLASS(s);
1869 SysBusDevice *dev = SYS_BUS_DEVICE(d);
1870 Error *local_err = NULL;
1871
1872 c->parent_realize(d, &local_err);
1873 if (local_err) {
1874 error_propagate(errp, local_err);
1875 return;
1876 }
1877
1878 qemu_mutex_init(&s->mutex);
1879
1880 memory_region_init_io(&sys->iomem, OBJECT(s),
1881 &smmu_mem_ops, sys, TYPE_ARM_SMMUV3, 0x20000);
1882
1883 sys->mrtypename = TYPE_SMMUV3_IOMMU_MEMORY_REGION;
1884
1885 sysbus_init_mmio(dev, &sys->iomem);
1886
1887 smmu_init_irq(s, dev);
1888 }
1889
1890 static const VMStateDescription vmstate_smmuv3_queue = {
1891 .name = "smmuv3_queue",
1892 .version_id = 1,
1893 .minimum_version_id = 1,
1894 .fields = (const VMStateField[]) {
1895 VMSTATE_UINT64(base, SMMUQueue),
1896 VMSTATE_UINT32(prod, SMMUQueue),
1897 VMSTATE_UINT32(cons, SMMUQueue),
1898 VMSTATE_UINT8(log2size, SMMUQueue),
1899 VMSTATE_END_OF_LIST(),
1900 },
1901 };
1902
1903 static bool smmuv3_gbpa_needed(void *opaque)
1904 {
1905 SMMUv3State *s = opaque;
1906
1907 /* Only migrate GBPA if it has different reset value. */
1908 return s->gbpa != SMMU_GBPA_RESET_VAL;
1909 }
1910
1911 static const VMStateDescription vmstate_gbpa = {
1912 .name = "smmuv3/gbpa",
1913 .version_id = 1,
1914 .minimum_version_id = 1,
1915 .needed = smmuv3_gbpa_needed,
1916 .fields = (const VMStateField[]) {
1917 VMSTATE_UINT32(gbpa, SMMUv3State),
1918 VMSTATE_END_OF_LIST()
1919 }
1920 };
1921
1922 static const VMStateDescription vmstate_smmuv3 = {
1923 .name = "smmuv3",
1924 .version_id = 1,
1925 .minimum_version_id = 1,
1926 .priority = MIG_PRI_IOMMU,
1927 .fields = (const VMStateField[]) {
1928 VMSTATE_UINT32(features, SMMUv3State),
1929 VMSTATE_UINT8(sid_size, SMMUv3State),
1930 VMSTATE_UINT8(sid_split, SMMUv3State),
1931
1932 VMSTATE_UINT32_ARRAY(cr, SMMUv3State, 3),
1933 VMSTATE_UINT32(cr0ack, SMMUv3State),
1934 VMSTATE_UINT32(statusr, SMMUv3State),
1935 VMSTATE_UINT32(irq_ctrl, SMMUv3State),
1936 VMSTATE_UINT32(gerror, SMMUv3State),
1937 VMSTATE_UINT32(gerrorn, SMMUv3State),
1938 VMSTATE_UINT64(gerror_irq_cfg0, SMMUv3State),
1939 VMSTATE_UINT32(gerror_irq_cfg1, SMMUv3State),
1940 VMSTATE_UINT32(gerror_irq_cfg2, SMMUv3State),
1941 VMSTATE_UINT64(strtab_base, SMMUv3State),
1942 VMSTATE_UINT32(strtab_base_cfg, SMMUv3State),
1943 VMSTATE_UINT64(eventq_irq_cfg0, SMMUv3State),
1944 VMSTATE_UINT32(eventq_irq_cfg1, SMMUv3State),
1945 VMSTATE_UINT32(eventq_irq_cfg2, SMMUv3State),
1946
1947 VMSTATE_STRUCT(cmdq, SMMUv3State, 0, vmstate_smmuv3_queue, SMMUQueue),
1948 VMSTATE_STRUCT(eventq, SMMUv3State, 0, vmstate_smmuv3_queue, SMMUQueue),
1949
1950 VMSTATE_END_OF_LIST(),
1951 },
1952 .subsections = (const VMStateDescription * const []) {
1953 &vmstate_gbpa,
1954 NULL
1955 }
1956 };
1957
1958 static Property smmuv3_properties[] = {
1959 /*
1960 * Stages of translation advertised.
1961 * "1": Stage 1
1962 * "2": Stage 2
1963 * Defaults to stage 1
1964 */
1965 DEFINE_PROP_STRING("stage", SMMUv3State, stage),
1966 DEFINE_PROP_END_OF_LIST()
1967 };
1968
1969 static void smmuv3_instance_init(Object *obj)
1970 {
1971 /* Nothing much to do here as of now */
1972 }
1973
1974 static void smmuv3_class_init(ObjectClass *klass, void *data)
1975 {
1976 DeviceClass *dc = DEVICE_CLASS(klass);
1977 ResettableClass *rc = RESETTABLE_CLASS(klass);
1978 SMMUv3Class *c = ARM_SMMUV3_CLASS(klass);
1979
1980 dc->vmsd = &vmstate_smmuv3;
1981 resettable_class_set_parent_phases(rc, NULL, smmu_reset_hold, NULL,
1982 &c->parent_phases);
1983 device_class_set_parent_realize(dc, smmu_realize,
1984 &c->parent_realize);
1985 device_class_set_props(dc, smmuv3_properties);
1986 }
1987
1988 static int smmuv3_notify_flag_changed(IOMMUMemoryRegion *iommu,
1989 IOMMUNotifierFlag old,
1990 IOMMUNotifierFlag new,
1991 Error **errp)
1992 {
1993 SMMUDevice *sdev = container_of(iommu, SMMUDevice, iommu);
1994 SMMUv3State *s3 = sdev->smmu;
1995 SMMUState *s = &(s3->smmu_state);
1996
1997 if (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) {
1998 error_setg(errp, "SMMUv3 does not support dev-iotlb yet");
1999 return -EINVAL;
2000 }
2001
2002 if (new & IOMMU_NOTIFIER_MAP) {
2003 error_setg(errp,
2004 "device %02x.%02x.%x requires iommu MAP notifier which is "
2005 "not currently supported", pci_bus_num(sdev->bus),
2006 PCI_SLOT(sdev->devfn), PCI_FUNC(sdev->devfn));
2007 return -EINVAL;
2008 }
2009
2010 if (old == IOMMU_NOTIFIER_NONE) {
2011 trace_smmuv3_notify_flag_add(iommu->parent_obj.name);
2012 QLIST_INSERT_HEAD(&s->devices_with_notifiers, sdev, next);
2013 } else if (new == IOMMU_NOTIFIER_NONE) {
2014 trace_smmuv3_notify_flag_del(iommu->parent_obj.name);
2015 QLIST_REMOVE(sdev, next);
2016 }
2017 return 0;
2018 }
2019
2020 static void smmuv3_iommu_memory_region_class_init(ObjectClass *klass,
2021 void *data)
2022 {
2023 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
2024
2025 imrc->translate = smmuv3_translate;
2026 imrc->notify_flag_changed = smmuv3_notify_flag_changed;
2027 }
2028
2029 static const TypeInfo smmuv3_type_info = {
2030 .name = TYPE_ARM_SMMUV3,
2031 .parent = TYPE_ARM_SMMU,
2032 .instance_size = sizeof(SMMUv3State),
2033 .instance_init = smmuv3_instance_init,
2034 .class_size = sizeof(SMMUv3Class),
2035 .class_init = smmuv3_class_init,
2036 };
2037
2038 static const TypeInfo smmuv3_iommu_memory_region_info = {
2039 .parent = TYPE_IOMMU_MEMORY_REGION,
2040 .name = TYPE_SMMUV3_IOMMU_MEMORY_REGION,
2041 .class_init = smmuv3_iommu_memory_region_class_init,
2042 };
2043
2044 static void smmuv3_register_types(void)
2045 {
2046 type_register(&smmuv3_type_info);
2047 type_register(&smmuv3_iommu_memory_region_info);
2048 }
2049
2050 type_init(smmuv3_register_types)
2051
AR7 emulation for QEMU