?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
qcow2: manually add more coroutine_fn annotations
[qemu.git] / hw / i386 / amd_iommu.c
blob725f69095b9e4c31e9fb3f6483dccc19a9dc9c81
1 /*
2 * QEMU emulation of AMD IOMMU (AMD-Vi)
3 *
4 * Copyright (C) 2011 Eduard - Gabriel Munteanu
5 * Copyright (C) 2015, 2016 David Kiarie Kahurani
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16
17 * You should have received a copy of the GNU General Public License along
18 * with this program; if not, see <http://www.gnu.org/licenses/>.
19 *
20 * Cache implementation inspired by hw/i386/intel_iommu.c
21 */
22
23 #include "qemu/osdep.h"
24 #include "hw/i386/pc.h"
25 #include "hw/pci/msi.h"
26 #include "hw/pci/pci_bus.h"
27 #include "migration/vmstate.h"
28 #include "amd_iommu.h"
29 #include "qapi/error.h"
30 #include "qemu/error-report.h"
31 #include "hw/i386/apic_internal.h"
32 #include "trace.h"
33 #include "hw/i386/apic-msidef.h"
34
35 /* used AMD-Vi MMIO registers */
36 const char *amdvi_mmio_low[] = {
37 "AMDVI_MMIO_DEVTAB_BASE",
38 "AMDVI_MMIO_CMDBUF_BASE",
39 "AMDVI_MMIO_EVTLOG_BASE",
40 "AMDVI_MMIO_CONTROL",
41 "AMDVI_MMIO_EXCL_BASE",
42 "AMDVI_MMIO_EXCL_LIMIT",
43 "AMDVI_MMIO_EXT_FEATURES",
44 "AMDVI_MMIO_PPR_BASE",
45 "UNHANDLED"
46 };
47 const char *amdvi_mmio_high[] = {
48 "AMDVI_MMIO_COMMAND_HEAD",
49 "AMDVI_MMIO_COMMAND_TAIL",
50 "AMDVI_MMIO_EVTLOG_HEAD",
51 "AMDVI_MMIO_EVTLOG_TAIL",
52 "AMDVI_MMIO_STATUS",
53 "AMDVI_MMIO_PPR_HEAD",
54 "AMDVI_MMIO_PPR_TAIL",
55 "UNHANDLED"
56 };
57
58 struct AMDVIAddressSpace {
59 uint8_t bus_num; /* bus number */
60 uint8_t devfn; /* device function */
61 AMDVIState *iommu_state; /* AMDVI - one per machine */
62 MemoryRegion root; /* AMDVI Root memory map region */
63 IOMMUMemoryRegion iommu; /* Device's address translation region */
64 MemoryRegion iommu_ir; /* Device's interrupt remapping region */
65 AddressSpace as; /* device's corresponding address space */
66 };
67
68 /* AMDVI cache entry */
69 typedef struct AMDVIIOTLBEntry {
70 uint16_t domid; /* assigned domain id */
71 uint16_t devid; /* device owning entry */
72 uint64_t perms; /* access permissions */
73 uint64_t translated_addr; /* translated address */
74 uint64_t page_mask; /* physical page size */
75 } AMDVIIOTLBEntry;
76
77 /* configure MMIO registers at startup/reset */
78 static void amdvi_set_quad(AMDVIState *s, hwaddr addr, uint64_t val,
79 uint64_t romask, uint64_t w1cmask)
80 {
81 stq_le_p(&s->mmior[addr], val);
82 stq_le_p(&s->romask[addr], romask);
83 stq_le_p(&s->w1cmask[addr], w1cmask);
84 }
85
86 static uint16_t amdvi_readw(AMDVIState *s, hwaddr addr)
87 {
88 return lduw_le_p(&s->mmior[addr]);
89 }
90
91 static uint32_t amdvi_readl(AMDVIState *s, hwaddr addr)
92 {
93 return ldl_le_p(&s->mmior[addr]);
94 }
95
96 static uint64_t amdvi_readq(AMDVIState *s, hwaddr addr)
97 {
98 return ldq_le_p(&s->mmior[addr]);
99 }
100
101 /* internal write */
102 static void amdvi_writeq_raw(AMDVIState *s, hwaddr addr, uint64_t val)
103 {
104 stq_le_p(&s->mmior[addr], val);
105 }
106
107 /* external write */
108 static void amdvi_writew(AMDVIState *s, hwaddr addr, uint16_t val)
109 {
110 uint16_t romask = lduw_le_p(&s->romask[addr]);
111 uint16_t w1cmask = lduw_le_p(&s->w1cmask[addr]);
112 uint16_t oldval = lduw_le_p(&s->mmior[addr]);
113 stw_le_p(&s->mmior[addr],
114 ((oldval & romask) | (val & ~romask)) & ~(val & w1cmask));
115 }
116
117 static void amdvi_writel(AMDVIState *s, hwaddr addr, uint32_t val)
118 {
119 uint32_t romask = ldl_le_p(&s->romask[addr]);
120 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]);
121 uint32_t oldval = ldl_le_p(&s->mmior[addr]);
122 stl_le_p(&s->mmior[addr],
123 ((oldval & romask) | (val & ~romask)) & ~(val & w1cmask));
124 }
125
126 static void amdvi_writeq(AMDVIState *s, hwaddr addr, uint64_t val)
127 {
128 uint64_t romask = ldq_le_p(&s->romask[addr]);
129 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]);
130 uint32_t oldval = ldq_le_p(&s->mmior[addr]);
131 stq_le_p(&s->mmior[addr],
132 ((oldval & romask) | (val & ~romask)) & ~(val & w1cmask));
133 }
134
135 /* OR a 64-bit register with a 64-bit value */
136 static bool amdvi_test_mask(AMDVIState *s, hwaddr addr, uint64_t val)
137 {
138 return amdvi_readq(s, addr) | val;
139 }
140
141 /* OR a 64-bit register with a 64-bit value storing result in the register */
142 static void amdvi_assign_orq(AMDVIState *s, hwaddr addr, uint64_t val)
143 {
144 amdvi_writeq_raw(s, addr, amdvi_readq(s, addr) | val);
145 }
146
147 /* AND a 64-bit register with a 64-bit value storing result in the register */
148 static void amdvi_assign_andq(AMDVIState *s, hwaddr addr, uint64_t val)
149 {
150 amdvi_writeq_raw(s, addr, amdvi_readq(s, addr) & val);
151 }
152
153 static void amdvi_generate_msi_interrupt(AMDVIState *s)
154 {
155 MSIMessage msg = {};
156 MemTxAttrs attrs = {
157 .requester_id = pci_requester_id(&s->pci.dev)
158 };
159
160 if (msi_enabled(&s->pci.dev)) {
161 msg = msi_get_message(&s->pci.dev, 0);
162 address_space_stl_le(&address_space_memory, msg.address, msg.data,
163 attrs, NULL);
164 }
165 }
166
167 static void amdvi_log_event(AMDVIState *s, uint64_t *evt)
168 {
169 /* event logging not enabled */
170 if (!s->evtlog_enabled || amdvi_test_mask(s, AMDVI_MMIO_STATUS,
171 AMDVI_MMIO_STATUS_EVT_OVF)) {
172 return;
173 }
174
175 /* event log buffer full */
176 if (s->evtlog_tail >= s->evtlog_len) {
177 amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_EVT_OVF);
178 /* generate interrupt */
179 amdvi_generate_msi_interrupt(s);
180 return;
181 }
182
183 if (dma_memory_write(&address_space_memory, s->evtlog + s->evtlog_tail,
184 evt, AMDVI_EVENT_LEN, MEMTXATTRS_UNSPECIFIED)) {
185 trace_amdvi_evntlog_fail(s->evtlog, s->evtlog_tail);
186 }
187
188 s->evtlog_tail += AMDVI_EVENT_LEN;
189 amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_COMP_INT);
190 amdvi_generate_msi_interrupt(s);
191 }
192
193 static void amdvi_setevent_bits(uint64_t *buffer, uint64_t value, int start,
194 int length)
195 {
196 int index = start / 64, bitpos = start % 64;
197 uint64_t mask = MAKE_64BIT_MASK(start, length);
198 buffer[index] &= ~mask;
199 buffer[index] |= (value << bitpos) & mask;
200 }
201 /*
202 * AMDVi event structure
203 * 0:15 -> DeviceID
204 * 48:63 -> event type + miscellaneous info
205 * 64:127 -> related address
206 */
207 static void amdvi_encode_event(uint64_t *evt, uint16_t devid, uint64_t addr,
208 uint16_t info)
209 {
210 evt[0] = 0;
211 evt[1] = 0;
212
213 amdvi_setevent_bits(evt, devid, 0, 16);
214 amdvi_setevent_bits(evt, info, 48, 16);
215 amdvi_setevent_bits(evt, addr, 64, 64);
216 }
217 /* log an error encountered during a page walk
218 *
219 * @addr: virtual address in translation request
220 */
221 static void amdvi_page_fault(AMDVIState *s, uint16_t devid,
222 hwaddr addr, uint16_t info)
223 {
224 uint64_t evt[2];
225
226 info |= AMDVI_EVENT_IOPF_I | AMDVI_EVENT_IOPF;
227 amdvi_encode_event(evt, devid, addr, info);
228 amdvi_log_event(s, evt);
229 pci_word_test_and_set_mask(s->pci.dev.config + PCI_STATUS,
230 PCI_STATUS_SIG_TARGET_ABORT);
231 }
232 /*
233 * log a master abort accessing device table
234 * @devtab : address of device table entry
235 * @info : error flags
236 */
237 static void amdvi_log_devtab_error(AMDVIState *s, uint16_t devid,
238 hwaddr devtab, uint16_t info)
239 {
240 uint64_t evt[2];
241
242 info |= AMDVI_EVENT_DEV_TAB_HW_ERROR;
243
244 amdvi_encode_event(evt, devid, devtab, info);
245 amdvi_log_event(s, evt);
246 pci_word_test_and_set_mask(s->pci.dev.config + PCI_STATUS,
247 PCI_STATUS_SIG_TARGET_ABORT);
248 }
249 /* log an event trying to access command buffer
250 * @addr : address that couldn't be accessed
251 */
252 static void amdvi_log_command_error(AMDVIState *s, hwaddr addr)
253 {
254 uint64_t evt[2];
255 uint16_t info = AMDVI_EVENT_COMMAND_HW_ERROR;
256
257 amdvi_encode_event(evt, 0, addr, info);
258 amdvi_log_event(s, evt);
259 pci_word_test_and_set_mask(s->pci.dev.config + PCI_STATUS,
260 PCI_STATUS_SIG_TARGET_ABORT);
261 }
262 /* log an illegal comand event
263 * @addr : address of illegal command
264 */
265 static void amdvi_log_illegalcom_error(AMDVIState *s, uint16_t info,
266 hwaddr addr)
267 {
268 uint64_t evt[2];
269
270 info |= AMDVI_EVENT_ILLEGAL_COMMAND_ERROR;
271 amdvi_encode_event(evt, 0, addr, info);
272 amdvi_log_event(s, evt);
273 }
274 /* log an error accessing device table
275 *
276 * @devid : device owning the table entry
277 * @devtab : address of device table entry
278 * @info : error flags
279 */
280 static void amdvi_log_illegaldevtab_error(AMDVIState *s, uint16_t devid,
281 hwaddr addr, uint16_t info)
282 {
283 uint64_t evt[2];
284
285 info |= AMDVI_EVENT_ILLEGAL_DEVTAB_ENTRY;
286 amdvi_encode_event(evt, devid, addr, info);
287 amdvi_log_event(s, evt);
288 }
289 /* log an error accessing a PTE entry
290 * @addr : address that couldn't be accessed
291 */
292 static void amdvi_log_pagetab_error(AMDVIState *s, uint16_t devid,
293 hwaddr addr, uint16_t info)
294 {
295 uint64_t evt[2];
296
297 info |= AMDVI_EVENT_PAGE_TAB_HW_ERROR;
298 amdvi_encode_event(evt, devid, addr, info);
299 amdvi_log_event(s, evt);
300 pci_word_test_and_set_mask(s->pci.dev.config + PCI_STATUS,
301 PCI_STATUS_SIG_TARGET_ABORT);
302 }
303
304 static gboolean amdvi_uint64_equal(gconstpointer v1, gconstpointer v2)
305 {
306 return *((const uint64_t *)v1) == *((const uint64_t *)v2);
307 }
308
309 static guint amdvi_uint64_hash(gconstpointer v)
310 {
311 return (guint)*(const uint64_t *)v;
312 }
313
314 static AMDVIIOTLBEntry *amdvi_iotlb_lookup(AMDVIState *s, hwaddr addr,
315 uint64_t devid)
316 {
317 uint64_t key = (addr >> AMDVI_PAGE_SHIFT_4K) |
318 ((uint64_t)(devid) << AMDVI_DEVID_SHIFT);
319 return g_hash_table_lookup(s->iotlb, &key);
320 }
321
322 static void amdvi_iotlb_reset(AMDVIState *s)
323 {
324 assert(s->iotlb);
325 trace_amdvi_iotlb_reset();
326 g_hash_table_remove_all(s->iotlb);
327 }
328
329 static gboolean amdvi_iotlb_remove_by_devid(gpointer key, gpointer value,
330 gpointer user_data)
331 {
332 AMDVIIOTLBEntry *entry = (AMDVIIOTLBEntry *)value;
333 uint16_t devid = *(uint16_t *)user_data;
334 return entry->devid == devid;
335 }
336
337 static void amdvi_iotlb_remove_page(AMDVIState *s, hwaddr addr,
338 uint64_t devid)
339 {
340 uint64_t key = (addr >> AMDVI_PAGE_SHIFT_4K) |
341 ((uint64_t)(devid) << AMDVI_DEVID_SHIFT);
342 g_hash_table_remove(s->iotlb, &key);
343 }
344
345 static void amdvi_update_iotlb(AMDVIState *s, uint16_t devid,
346 uint64_t gpa, IOMMUTLBEntry to_cache,
347 uint16_t domid)
348 {
349 AMDVIIOTLBEntry *entry = g_new(AMDVIIOTLBEntry, 1);
350 uint64_t *key = g_new(uint64_t, 1);
351 uint64_t gfn = gpa >> AMDVI_PAGE_SHIFT_4K;
352
353 /* don't cache erroneous translations */
354 if (to_cache.perm != IOMMU_NONE) {
355 trace_amdvi_cache_update(domid, PCI_BUS_NUM(devid), PCI_SLOT(devid),
356 PCI_FUNC(devid), gpa, to_cache.translated_addr);
357
358 if (g_hash_table_size(s->iotlb) >= AMDVI_IOTLB_MAX_SIZE) {
359 amdvi_iotlb_reset(s);
360 }
361
362 entry->domid = domid;
363 entry->perms = to_cache.perm;
364 entry->translated_addr = to_cache.translated_addr;
365 entry->page_mask = to_cache.addr_mask;
366 *key = gfn | ((uint64_t)(devid) << AMDVI_DEVID_SHIFT);
367 g_hash_table_replace(s->iotlb, key, entry);
368 }
369 }
370
371 static void amdvi_completion_wait(AMDVIState *s, uint64_t *cmd)
372 {
373 /* pad the last 3 bits */
374 hwaddr addr = cpu_to_le64(extract64(cmd[0], 3, 49)) << 3;
375 uint64_t data = cpu_to_le64(cmd[1]);
376
377 if (extract64(cmd[0], 52, 8)) {
378 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
379 s->cmdbuf + s->cmdbuf_head);
380 }
381 if (extract64(cmd[0], 0, 1)) {
382 if (dma_memory_write(&address_space_memory, addr, &data,
383 AMDVI_COMPLETION_DATA_SIZE,
384 MEMTXATTRS_UNSPECIFIED)) {
385 trace_amdvi_completion_wait_fail(addr);
386 }
387 }
388 /* set completion interrupt */
389 if (extract64(cmd[0], 1, 1)) {
390 amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_COMP_INT);
391 /* generate interrupt */
392 amdvi_generate_msi_interrupt(s);
393 }
394 trace_amdvi_completion_wait(addr, data);
395 }
396
397 /* log error without aborting since linux seems to be using reserved bits */
398 static void amdvi_inval_devtab_entry(AMDVIState *s, uint64_t *cmd)
399 {
400 uint16_t devid = cpu_to_le16((uint16_t)extract64(cmd[0], 0, 16));
401
402 /* This command should invalidate internal caches of which there isn't */
403 if (extract64(cmd[0], 16, 44) || cmd[1]) {
404 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
405 s->cmdbuf + s->cmdbuf_head);
406 }
407 trace_amdvi_devtab_inval(PCI_BUS_NUM(devid), PCI_SLOT(devid),
408 PCI_FUNC(devid));
409 }
410
411 static void amdvi_complete_ppr(AMDVIState *s, uint64_t *cmd)
412 {
413 if (extract64(cmd[0], 16, 16) || extract64(cmd[0], 52, 8) ||
414 extract64(cmd[1], 0, 2) || extract64(cmd[1], 3, 29)
415 || extract64(cmd[1], 48, 16)) {
416 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
417 s->cmdbuf + s->cmdbuf_head);
418 }
419 trace_amdvi_ppr_exec();
420 }
421
422 static void amdvi_inval_all(AMDVIState *s, uint64_t *cmd)
423 {
424 if (extract64(cmd[0], 0, 60) || cmd[1]) {
425 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
426 s->cmdbuf + s->cmdbuf_head);
427 }
428
429 amdvi_iotlb_reset(s);
430 trace_amdvi_all_inval();
431 }
432
433 static gboolean amdvi_iotlb_remove_by_domid(gpointer key, gpointer value,
434 gpointer user_data)
435 {
436 AMDVIIOTLBEntry *entry = (AMDVIIOTLBEntry *)value;
437 uint16_t domid = *(uint16_t *)user_data;
438 return entry->domid == domid;
439 }
440
441 /* we don't have devid - we can't remove pages by address */
442 static void amdvi_inval_pages(AMDVIState *s, uint64_t *cmd)
443 {
444 uint16_t domid = cpu_to_le16((uint16_t)extract64(cmd[0], 32, 16));
445
446 if (extract64(cmd[0], 20, 12) || extract64(cmd[0], 48, 12) ||
447 extract64(cmd[1], 3, 9)) {
448 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
449 s->cmdbuf + s->cmdbuf_head);
450 }
451
452 g_hash_table_foreach_remove(s->iotlb, amdvi_iotlb_remove_by_domid,
453 &domid);
454 trace_amdvi_pages_inval(domid);
455 }
456
457 static void amdvi_prefetch_pages(AMDVIState *s, uint64_t *cmd)
458 {
459 if (extract64(cmd[0], 16, 8) || extract64(cmd[0], 52, 8) ||
460 extract64(cmd[1], 1, 1) || extract64(cmd[1], 3, 1) ||
461 extract64(cmd[1], 5, 7)) {
462 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
463 s->cmdbuf + s->cmdbuf_head);
464 }
465
466 trace_amdvi_prefetch_pages();
467 }
468
469 static void amdvi_inval_inttable(AMDVIState *s, uint64_t *cmd)
470 {
471 if (extract64(cmd[0], 16, 44) || cmd[1]) {
472 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
473 s->cmdbuf + s->cmdbuf_head);
474 return;
475 }
476
477 trace_amdvi_intr_inval();
478 }
479
480 /* FIXME: Try to work with the specified size instead of all the pages
481 * when the S bit is on
482 */
483 static void iommu_inval_iotlb(AMDVIState *s, uint64_t *cmd)
484 {
485
486 uint16_t devid = extract64(cmd[0], 0, 16);
487 if (extract64(cmd[1], 1, 1) || extract64(cmd[1], 3, 1) ||
488 extract64(cmd[1], 6, 6)) {
489 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
490 s->cmdbuf + s->cmdbuf_head);
491 return;
492 }
493
494 if (extract64(cmd[1], 0, 1)) {
495 g_hash_table_foreach_remove(s->iotlb, amdvi_iotlb_remove_by_devid,
496 &devid);
497 } else {
498 amdvi_iotlb_remove_page(s, cpu_to_le64(extract64(cmd[1], 12, 52)) << 12,
499 cpu_to_le16(extract64(cmd[1], 0, 16)));
500 }
501 trace_amdvi_iotlb_inval();
502 }
503
504 /* not honouring reserved bits is regarded as an illegal command */
505 static void amdvi_cmdbuf_exec(AMDVIState *s)
506 {
507 uint64_t cmd[2];
508
509 if (dma_memory_read(&address_space_memory, s->cmdbuf + s->cmdbuf_head,
510 cmd, AMDVI_COMMAND_SIZE, MEMTXATTRS_UNSPECIFIED)) {
511 trace_amdvi_command_read_fail(s->cmdbuf, s->cmdbuf_head);
512 amdvi_log_command_error(s, s->cmdbuf + s->cmdbuf_head);
513 return;
514 }
515
516 switch (extract64(cmd[0], 60, 4)) {
517 case AMDVI_CMD_COMPLETION_WAIT:
518 amdvi_completion_wait(s, cmd);
519 break;
520 case AMDVI_CMD_INVAL_DEVTAB_ENTRY:
521 amdvi_inval_devtab_entry(s, cmd);
522 break;
523 case AMDVI_CMD_INVAL_AMDVI_PAGES:
524 amdvi_inval_pages(s, cmd);
525 break;
526 case AMDVI_CMD_INVAL_IOTLB_PAGES:
527 iommu_inval_iotlb(s, cmd);
528 break;
529 case AMDVI_CMD_INVAL_INTR_TABLE:
530 amdvi_inval_inttable(s, cmd);
531 break;
532 case AMDVI_CMD_PREFETCH_AMDVI_PAGES:
533 amdvi_prefetch_pages(s, cmd);
534 break;
535 case AMDVI_CMD_COMPLETE_PPR_REQUEST:
536 amdvi_complete_ppr(s, cmd);
537 break;
538 case AMDVI_CMD_INVAL_AMDVI_ALL:
539 amdvi_inval_all(s, cmd);
540 break;
541 default:
542 trace_amdvi_unhandled_command(extract64(cmd[1], 60, 4));
543 /* log illegal command */
544 amdvi_log_illegalcom_error(s, extract64(cmd[1], 60, 4),
545 s->cmdbuf + s->cmdbuf_head);
546 }
547 }
548
549 static void amdvi_cmdbuf_run(AMDVIState *s)
550 {
551 if (!s->cmdbuf_enabled) {
552 trace_amdvi_command_error(amdvi_readq(s, AMDVI_MMIO_CONTROL));
553 return;
554 }
555
556 /* check if there is work to do. */
557 while (s->cmdbuf_head != s->cmdbuf_tail) {
558 trace_amdvi_command_exec(s->cmdbuf_head, s->cmdbuf_tail, s->cmdbuf);
559 amdvi_cmdbuf_exec(s);
560 s->cmdbuf_head += AMDVI_COMMAND_SIZE;
561 amdvi_writeq_raw(s, AMDVI_MMIO_COMMAND_HEAD, s->cmdbuf_head);
562
563 /* wrap head pointer */
564 if (s->cmdbuf_head >= s->cmdbuf_len * AMDVI_COMMAND_SIZE) {
565 s->cmdbuf_head = 0;
566 }
567 }
568 }
569
570 static void amdvi_mmio_trace(hwaddr addr, unsigned size)
571 {
572 uint8_t index = (addr & ~0x2000) / 8;
573
574 if ((addr & 0x2000)) {
575 /* high table */
576 index = index >= AMDVI_MMIO_REGS_HIGH ? AMDVI_MMIO_REGS_HIGH : index;
577 trace_amdvi_mmio_read(amdvi_mmio_high[index], addr, size, addr & ~0x07);
578 } else {
579 index = index >= AMDVI_MMIO_REGS_LOW ? AMDVI_MMIO_REGS_LOW : index;
580 trace_amdvi_mmio_read(amdvi_mmio_low[index], addr, size, addr & ~0x07);
581 }
582 }
583
584 static uint64_t amdvi_mmio_read(void *opaque, hwaddr addr, unsigned size)
585 {
586 AMDVIState *s = opaque;
587
588 uint64_t val = -1;
589 if (addr + size > AMDVI_MMIO_SIZE) {
590 trace_amdvi_mmio_read_invalid(AMDVI_MMIO_SIZE, addr, size);
591 return (uint64_t)-1;
592 }
593
594 if (size == 2) {
595 val = amdvi_readw(s, addr);
596 } else if (size == 4) {
597 val = amdvi_readl(s, addr);
598 } else if (size == 8) {
599 val = amdvi_readq(s, addr);
600 }
601 amdvi_mmio_trace(addr, size);
602
603 return val;
604 }
605
606 static void amdvi_handle_control_write(AMDVIState *s)
607 {
608 unsigned long control = amdvi_readq(s, AMDVI_MMIO_CONTROL);
609 s->enabled = !!(control & AMDVI_MMIO_CONTROL_AMDVIEN);
610
611 s->ats_enabled = !!(control & AMDVI_MMIO_CONTROL_HTTUNEN);
612 s->evtlog_enabled = s->enabled && !!(control &
613 AMDVI_MMIO_CONTROL_EVENTLOGEN);
614
615 s->evtlog_intr = !!(control & AMDVI_MMIO_CONTROL_EVENTINTEN);
616 s->completion_wait_intr = !!(control & AMDVI_MMIO_CONTROL_COMWAITINTEN);
617 s->cmdbuf_enabled = s->enabled && !!(control &
618 AMDVI_MMIO_CONTROL_CMDBUFLEN);
619 s->ga_enabled = !!(control & AMDVI_MMIO_CONTROL_GAEN);
620
621 /* update the flags depending on the control register */
622 if (s->cmdbuf_enabled) {
623 amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_CMDBUF_RUN);
624 } else {
625 amdvi_assign_andq(s, AMDVI_MMIO_STATUS, ~AMDVI_MMIO_STATUS_CMDBUF_RUN);
626 }
627 if (s->evtlog_enabled) {
628 amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_EVT_RUN);
629 } else {
630 amdvi_assign_andq(s, AMDVI_MMIO_STATUS, ~AMDVI_MMIO_STATUS_EVT_RUN);
631 }
632
633 trace_amdvi_control_status(control);
634 amdvi_cmdbuf_run(s);
635 }
636
637 static inline void amdvi_handle_devtab_write(AMDVIState *s)
638
639 {
640 uint64_t val = amdvi_readq(s, AMDVI_MMIO_DEVICE_TABLE);
641 s->devtab = (val & AMDVI_MMIO_DEVTAB_BASE_MASK);
642
643 /* set device table length */
644 s->devtab_len = ((val & AMDVI_MMIO_DEVTAB_SIZE_MASK) + 1 *
645 (AMDVI_MMIO_DEVTAB_SIZE_UNIT /
646 AMDVI_MMIO_DEVTAB_ENTRY_SIZE));
647 }
648
649 static inline void amdvi_handle_cmdhead_write(AMDVIState *s)
650 {
651 s->cmdbuf_head = amdvi_readq(s, AMDVI_MMIO_COMMAND_HEAD)
652 & AMDVI_MMIO_CMDBUF_HEAD_MASK;
653 amdvi_cmdbuf_run(s);
654 }
655
656 static inline void amdvi_handle_cmdbase_write(AMDVIState *s)
657 {
658 s->cmdbuf = amdvi_readq(s, AMDVI_MMIO_COMMAND_BASE)
659 & AMDVI_MMIO_CMDBUF_BASE_MASK;
660 s->cmdbuf_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_CMDBUF_SIZE_BYTE)
661 & AMDVI_MMIO_CMDBUF_SIZE_MASK);
662 s->cmdbuf_head = s->cmdbuf_tail = 0;
663 }
664
665 static inline void amdvi_handle_cmdtail_write(AMDVIState *s)
666 {
667 s->cmdbuf_tail = amdvi_readq(s, AMDVI_MMIO_COMMAND_TAIL)
668 & AMDVI_MMIO_CMDBUF_TAIL_MASK;
669 amdvi_cmdbuf_run(s);
670 }
671
672 static inline void amdvi_handle_excllim_write(AMDVIState *s)
673 {
674 uint64_t val = amdvi_readq(s, AMDVI_MMIO_EXCL_LIMIT);
675 s->excl_limit = (val & AMDVI_MMIO_EXCL_LIMIT_MASK) |
676 AMDVI_MMIO_EXCL_LIMIT_LOW;
677 }
678
679 static inline void amdvi_handle_evtbase_write(AMDVIState *s)
680 {
681 uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_BASE);
682 s->evtlog = val & AMDVI_MMIO_EVTLOG_BASE_MASK;
683 s->evtlog_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_EVTLOG_SIZE_BYTE)
684 & AMDVI_MMIO_EVTLOG_SIZE_MASK);
685 }
686
687 static inline void amdvi_handle_evttail_write(AMDVIState *s)
688 {
689 uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_TAIL);
690 s->evtlog_tail = val & AMDVI_MMIO_EVTLOG_TAIL_MASK;
691 }
692
693 static inline void amdvi_handle_evthead_write(AMDVIState *s)
694 {
695 uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_HEAD);
696 s->evtlog_head = val & AMDVI_MMIO_EVTLOG_HEAD_MASK;
697 }
698
699 static inline void amdvi_handle_pprbase_write(AMDVIState *s)
700 {
701 uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_BASE);
702 s->ppr_log = val & AMDVI_MMIO_PPRLOG_BASE_MASK;
703 s->pprlog_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_PPRLOG_SIZE_BYTE)
704 & AMDVI_MMIO_PPRLOG_SIZE_MASK);
705 }
706
707 static inline void amdvi_handle_pprhead_write(AMDVIState *s)
708 {
709 uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_HEAD);
710 s->pprlog_head = val & AMDVI_MMIO_PPRLOG_HEAD_MASK;
711 }
712
713 static inline void amdvi_handle_pprtail_write(AMDVIState *s)
714 {
715 uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_TAIL);
716 s->pprlog_tail = val & AMDVI_MMIO_PPRLOG_TAIL_MASK;
717 }
718
719 /* FIXME: something might go wrong if System Software writes in chunks
720 * of one byte but linux writes in chunks of 4 bytes so currently it
721 * works correctly with linux but will definitely be busted if software
722 * reads/writes 8 bytes
723 */
724 static void amdvi_mmio_reg_write(AMDVIState *s, unsigned size, uint64_t val,
725 hwaddr addr)
726 {
727 if (size == 2) {
728 amdvi_writew(s, addr, val);
729 } else if (size == 4) {
730 amdvi_writel(s, addr, val);
731 } else if (size == 8) {
732 amdvi_writeq(s, addr, val);
733 }
734 }
735
736 static void amdvi_mmio_write(void *opaque, hwaddr addr, uint64_t val,
737 unsigned size)
738 {
739 AMDVIState *s = opaque;
740 unsigned long offset = addr & 0x07;
741
742 if (addr + size > AMDVI_MMIO_SIZE) {
743 trace_amdvi_mmio_write("error: addr outside region: max ",
744 (uint64_t)AMDVI_MMIO_SIZE, size, val, offset);
745 return;
746 }
747
748 amdvi_mmio_trace(addr, size);
749 switch (addr & ~0x07) {
750 case AMDVI_MMIO_CONTROL:
751 amdvi_mmio_reg_write(s, size, val, addr);
752 amdvi_handle_control_write(s);
753 break;
754 case AMDVI_MMIO_DEVICE_TABLE:
755 amdvi_mmio_reg_write(s, size, val, addr);
756 /* set device table address
757 * This also suffers from inability to tell whether software
758 * is done writing
759 */
760 if (offset || (size == 8)) {
761 amdvi_handle_devtab_write(s);
762 }
763 break;
764 case AMDVI_MMIO_COMMAND_HEAD:
765 amdvi_mmio_reg_write(s, size, val, addr);
766 amdvi_handle_cmdhead_write(s);
767 break;
768 case AMDVI_MMIO_COMMAND_BASE:
769 amdvi_mmio_reg_write(s, size, val, addr);
770 /* FIXME - make sure System Software has finished writing incase
771 * it writes in chucks less than 8 bytes in a robust way.As for
772 * now, this hacks works for the linux driver
773 */
774 if (offset || (size == 8)) {
775 amdvi_handle_cmdbase_write(s);
776 }
777 break;
778 case AMDVI_MMIO_COMMAND_TAIL:
779 amdvi_mmio_reg_write(s, size, val, addr);
780 amdvi_handle_cmdtail_write(s);
781 break;
782 case AMDVI_MMIO_EVENT_BASE:
783 amdvi_mmio_reg_write(s, size, val, addr);
784 amdvi_handle_evtbase_write(s);
785 break;
786 case AMDVI_MMIO_EVENT_HEAD:
787 amdvi_mmio_reg_write(s, size, val, addr);
788 amdvi_handle_evthead_write(s);
789 break;
790 case AMDVI_MMIO_EVENT_TAIL:
791 amdvi_mmio_reg_write(s, size, val, addr);
792 amdvi_handle_evttail_write(s);
793 break;
794 case AMDVI_MMIO_EXCL_LIMIT:
795 amdvi_mmio_reg_write(s, size, val, addr);
796 amdvi_handle_excllim_write(s);
797 break;
798 /* PPR log base - unused for now */
799 case AMDVI_MMIO_PPR_BASE:
800 amdvi_mmio_reg_write(s, size, val, addr);
801 amdvi_handle_pprbase_write(s);
802 break;
803 /* PPR log head - also unused for now */
804 case AMDVI_MMIO_PPR_HEAD:
805 amdvi_mmio_reg_write(s, size, val, addr);
806 amdvi_handle_pprhead_write(s);
807 break;
808 /* PPR log tail - unused for now */
809 case AMDVI_MMIO_PPR_TAIL:
810 amdvi_mmio_reg_write(s, size, val, addr);
811 amdvi_handle_pprtail_write(s);
812 break;
813 }
814 }
815
816 static inline uint64_t amdvi_get_perms(uint64_t entry)
817 {
818 return (entry & (AMDVI_DEV_PERM_READ | AMDVI_DEV_PERM_WRITE)) >>
819 AMDVI_DEV_PERM_SHIFT;
820 }
821
822 /* validate that reserved bits are honoured */
823 static bool amdvi_validate_dte(AMDVIState *s, uint16_t devid,
824 uint64_t *dte)
825 {
826 if ((dte[0] & AMDVI_DTE_LOWER_QUAD_RESERVED)
827 || (dte[1] & AMDVI_DTE_MIDDLE_QUAD_RESERVED)
828 || (dte[2] & AMDVI_DTE_UPPER_QUAD_RESERVED) || dte[3]) {
829 amdvi_log_illegaldevtab_error(s, devid,
830 s->devtab +
831 devid * AMDVI_DEVTAB_ENTRY_SIZE, 0);
832 return false;
833 }
834
835 return true;
836 }
837
838 /* get a device table entry given the devid */
839 static bool amdvi_get_dte(AMDVIState *s, int devid, uint64_t *entry)
840 {
841 uint32_t offset = devid * AMDVI_DEVTAB_ENTRY_SIZE;
842
843 if (dma_memory_read(&address_space_memory, s->devtab + offset, entry,
844 AMDVI_DEVTAB_ENTRY_SIZE, MEMTXATTRS_UNSPECIFIED)) {
845 trace_amdvi_dte_get_fail(s->devtab, offset);
846 /* log error accessing dte */
847 amdvi_log_devtab_error(s, devid, s->devtab + offset, 0);
848 return false;
849 }
850
851 *entry = le64_to_cpu(*entry);
852 if (!amdvi_validate_dte(s, devid, entry)) {
853 trace_amdvi_invalid_dte(entry[0]);
854 return false;
855 }
856
857 return true;
858 }
859
860 /* get pte translation mode */
861 static inline uint8_t get_pte_translation_mode(uint64_t pte)
862 {
863 return (pte >> AMDVI_DEV_MODE_RSHIFT) & AMDVI_DEV_MODE_MASK;
864 }
865
866 static inline uint64_t pte_override_page_mask(uint64_t pte)
867 {
868 uint8_t page_mask = 13;
869 uint64_t addr = (pte & AMDVI_DEV_PT_ROOT_MASK) >> 12;
870 /* find the first zero bit */
871 while (addr & 1) {
872 page_mask++;
873 addr = addr >> 1;
874 }
875
876 return ~((1ULL << page_mask) - 1);
877 }
878
879 static inline uint64_t pte_get_page_mask(uint64_t oldlevel)
880 {
881 return ~((1UL << ((oldlevel * 9) + 3)) - 1);
882 }
883
884 static inline uint64_t amdvi_get_pte_entry(AMDVIState *s, uint64_t pte_addr,
885 uint16_t devid)
886 {
887 uint64_t pte;
888
889 if (dma_memory_read(&address_space_memory, pte_addr,
890 &pte, sizeof(pte), MEMTXATTRS_UNSPECIFIED)) {
891 trace_amdvi_get_pte_hwerror(pte_addr);
892 amdvi_log_pagetab_error(s, devid, pte_addr, 0);
893 pte = 0;
894 return pte;
895 }
896
897 pte = le64_to_cpu(pte);
898 return pte;
899 }
900
901 static void amdvi_page_walk(AMDVIAddressSpace *as, uint64_t *dte,
902 IOMMUTLBEntry *ret, unsigned perms,
903 hwaddr addr)
904 {
905 unsigned level, present, pte_perms, oldlevel;
906 uint64_t pte = dte[0], pte_addr, page_mask;
907
908 /* make sure the DTE has TV = 1 */
909 if (pte & AMDVI_DEV_TRANSLATION_VALID) {
910 level = get_pte_translation_mode(pte);
911 if (level >= 7) {
912 trace_amdvi_mode_invalid(level, addr);
913 return;
914 }
915 if (level == 0) {
916 goto no_remap;
917 }
918
919 /* we are at the leaf page table or page table encodes a huge page */
920 do {
921 pte_perms = amdvi_get_perms(pte);
922 present = pte & 1;
923 if (!present || perms != (perms & pte_perms)) {
924 amdvi_page_fault(as->iommu_state, as->devfn, addr, perms);
925 trace_amdvi_page_fault(addr);
926 return;
927 }
928
929 /* go to the next lower level */
930 pte_addr = pte & AMDVI_DEV_PT_ROOT_MASK;
931 /* add offset and load pte */
932 pte_addr += ((addr >> (3 + 9 * level)) & 0x1FF) << 3;
933 pte = amdvi_get_pte_entry(as->iommu_state, pte_addr, as->devfn);
934 if (!pte) {
935 return;
936 }
937 oldlevel = level;
938 level = get_pte_translation_mode(pte);
939 } while (level > 0 && level < 7);
940
941 if (level == 0x7) {
942 page_mask = pte_override_page_mask(pte);
943 } else {
944 page_mask = pte_get_page_mask(oldlevel);
945 }
946
947 /* get access permissions from pte */
948 ret->iova = addr & page_mask;
949 ret->translated_addr = (pte & AMDVI_DEV_PT_ROOT_MASK) & page_mask;
950 ret->addr_mask = ~page_mask;
951 ret->perm = amdvi_get_perms(pte);
952 return;
953 }
954 no_remap:
955 ret->iova = addr & AMDVI_PAGE_MASK_4K;
956 ret->translated_addr = addr & AMDVI_PAGE_MASK_4K;
957 ret->addr_mask = ~AMDVI_PAGE_MASK_4K;
958 ret->perm = amdvi_get_perms(pte);
959 }
960
961 static void amdvi_do_translate(AMDVIAddressSpace *as, hwaddr addr,
962 bool is_write, IOMMUTLBEntry *ret)
963 {
964 AMDVIState *s = as->iommu_state;
965 uint16_t devid = PCI_BUILD_BDF(as->bus_num, as->devfn);
966 AMDVIIOTLBEntry *iotlb_entry = amdvi_iotlb_lookup(s, addr, devid);
967 uint64_t entry[4];
968
969 if (iotlb_entry) {
970 trace_amdvi_iotlb_hit(PCI_BUS_NUM(devid), PCI_SLOT(devid),
971 PCI_FUNC(devid), addr, iotlb_entry->translated_addr);
972 ret->iova = addr & ~iotlb_entry->page_mask;
973 ret->translated_addr = iotlb_entry->translated_addr;
974 ret->addr_mask = iotlb_entry->page_mask;
975 ret->perm = iotlb_entry->perms;
976 return;
977 }
978
979 if (!amdvi_get_dte(s, devid, entry)) {
980 return;
981 }
982
983 /* devices with V = 0 are not translated */
984 if (!(entry[0] & AMDVI_DEV_VALID)) {
985 goto out;
986 }
987
988 amdvi_page_walk(as, entry, ret,
989 is_write ? AMDVI_PERM_WRITE : AMDVI_PERM_READ, addr);
990
991 amdvi_update_iotlb(s, devid, addr, *ret,
992 entry[1] & AMDVI_DEV_DOMID_ID_MASK);
993 return;
994
995 out:
996 ret->iova = addr & AMDVI_PAGE_MASK_4K;
997 ret->translated_addr = addr & AMDVI_PAGE_MASK_4K;
998 ret->addr_mask = ~AMDVI_PAGE_MASK_4K;
999 ret->perm = IOMMU_RW;
1000 }
1001
1002 static inline bool amdvi_is_interrupt_addr(hwaddr addr)
1003 {
1004 return addr >= AMDVI_INT_ADDR_FIRST && addr <= AMDVI_INT_ADDR_LAST;
1005 }
1006
1007 static IOMMUTLBEntry amdvi_translate(IOMMUMemoryRegion *iommu, hwaddr addr,
1008 IOMMUAccessFlags flag, int iommu_idx)
1009 {
1010 AMDVIAddressSpace *as = container_of(iommu, AMDVIAddressSpace, iommu);
1011 AMDVIState *s = as->iommu_state;
1012 IOMMUTLBEntry ret = {
1013 .target_as = &address_space_memory,
1014 .iova = addr,
1015 .translated_addr = 0,
1016 .addr_mask = ~(hwaddr)0,
1017 .perm = IOMMU_NONE
1018 };
1019
1020 if (!s->enabled) {
1021 /* AMDVI disabled - corresponds to iommu=off not
1022 * failure to provide any parameter
1023 */
1024 ret.iova = addr & AMDVI_PAGE_MASK_4K;
1025 ret.translated_addr = addr & AMDVI_PAGE_MASK_4K;
1026 ret.addr_mask = ~AMDVI_PAGE_MASK_4K;
1027 ret.perm = IOMMU_RW;
1028 return ret;
1029 } else if (amdvi_is_interrupt_addr(addr)) {
1030 ret.iova = addr & AMDVI_PAGE_MASK_4K;
1031 ret.translated_addr = addr & AMDVI_PAGE_MASK_4K;
1032 ret.addr_mask = ~AMDVI_PAGE_MASK_4K;
1033 ret.perm = IOMMU_WO;
1034 return ret;
1035 }
1036
1037 amdvi_do_translate(as, addr, flag & IOMMU_WO, &ret);
1038 trace_amdvi_translation_result(as->bus_num, PCI_SLOT(as->devfn),
1039 PCI_FUNC(as->devfn), addr, ret.translated_addr);
1040 return ret;
1041 }
1042
1043 static int amdvi_get_irte(AMDVIState *s, MSIMessage *origin, uint64_t *dte,
1044 union irte *irte, uint16_t devid)
1045 {
1046 uint64_t irte_root, offset;
1047
1048 irte_root = dte[2] & AMDVI_IR_PHYS_ADDR_MASK;
1049 offset = (origin->data & AMDVI_IRTE_OFFSET) << 2;
1050
1051 trace_amdvi_ir_irte(irte_root, offset);
1052
1053 if (dma_memory_read(&address_space_memory, irte_root + offset,
1054 irte, sizeof(*irte), MEMTXATTRS_UNSPECIFIED)) {
1055 trace_amdvi_ir_err("failed to get irte");
1056 return -AMDVI_IR_GET_IRTE;
1057 }
1058
1059 trace_amdvi_ir_irte_val(irte->val);
1060
1061 return 0;
1062 }
1063
1064 static int amdvi_int_remap_legacy(AMDVIState *iommu,
1065 MSIMessage *origin,
1066 MSIMessage *translated,
1067 uint64_t *dte,
1068 X86IOMMUIrq *irq,
1069 uint16_t sid)
1070 {
1071 int ret;
1072 union irte irte;
1073
1074 /* get interrupt remapping table */
1075 ret = amdvi_get_irte(iommu, origin, dte, &irte, sid);
1076 if (ret < 0) {
1077 return ret;
1078 }
1079
1080 if (!irte.fields.valid) {
1081 trace_amdvi_ir_target_abort("RemapEn is disabled");
1082 return -AMDVI_IR_TARGET_ABORT;
1083 }
1084
1085 if (irte.fields.guest_mode) {
1086 error_report_once("guest mode is not zero");
1087 return -AMDVI_IR_ERR;
1088 }
1089
1090 if (irte.fields.int_type > AMDVI_IOAPIC_INT_TYPE_ARBITRATED) {
1091 error_report_once("reserved int_type");
1092 return -AMDVI_IR_ERR;
1093 }
1094
1095 irq->delivery_mode = irte.fields.int_type;
1096 irq->vector = irte.fields.vector;
1097 irq->dest_mode = irte.fields.dm;
1098 irq->redir_hint = irte.fields.rq_eoi;
1099 irq->dest = irte.fields.destination;
1100
1101 return 0;
1102 }
1103
1104 static int amdvi_get_irte_ga(AMDVIState *s, MSIMessage *origin, uint64_t *dte,
1105 struct irte_ga *irte, uint16_t devid)
1106 {
1107 uint64_t irte_root, offset;
1108
1109 irte_root = dte[2] & AMDVI_IR_PHYS_ADDR_MASK;
1110 offset = (origin->data & AMDVI_IRTE_OFFSET) << 4;
1111 trace_amdvi_ir_irte(irte_root, offset);
1112
1113 if (dma_memory_read(&address_space_memory, irte_root + offset,
1114 irte, sizeof(*irte), MEMTXATTRS_UNSPECIFIED)) {
1115 trace_amdvi_ir_err("failed to get irte_ga");
1116 return -AMDVI_IR_GET_IRTE;
1117 }
1118
1119 trace_amdvi_ir_irte_ga_val(irte->hi.val, irte->lo.val);
1120 return 0;
1121 }
1122
1123 static int amdvi_int_remap_ga(AMDVIState *iommu,
1124 MSIMessage *origin,
1125 MSIMessage *translated,
1126 uint64_t *dte,
1127 X86IOMMUIrq *irq,
1128 uint16_t sid)
1129 {
1130 int ret;
1131 struct irte_ga irte;
1132
1133 /* get interrupt remapping table */
1134 ret = amdvi_get_irte_ga(iommu, origin, dte, &irte, sid);
1135 if (ret < 0) {
1136 return ret;
1137 }
1138
1139 if (!irte.lo.fields_remap.valid) {
1140 trace_amdvi_ir_target_abort("RemapEn is disabled");
1141 return -AMDVI_IR_TARGET_ABORT;
1142 }
1143
1144 if (irte.lo.fields_remap.guest_mode) {
1145 error_report_once("guest mode is not zero");
1146 return -AMDVI_IR_ERR;
1147 }
1148
1149 if (irte.lo.fields_remap.int_type > AMDVI_IOAPIC_INT_TYPE_ARBITRATED) {
1150 error_report_once("reserved int_type is set");
1151 return -AMDVI_IR_ERR;
1152 }
1153
1154 irq->delivery_mode = irte.lo.fields_remap.int_type;
1155 irq->vector = irte.hi.fields.vector;
1156 irq->dest_mode = irte.lo.fields_remap.dm;
1157 irq->redir_hint = irte.lo.fields_remap.rq_eoi;
1158 irq->dest = irte.lo.fields_remap.destination;
1159
1160 return 0;
1161 }
1162
1163 static int __amdvi_int_remap_msi(AMDVIState *iommu,
1164 MSIMessage *origin,
1165 MSIMessage *translated,
1166 uint64_t *dte,
1167 X86IOMMUIrq *irq,
1168 uint16_t sid)
1169 {
1170 int ret;
1171 uint8_t int_ctl;
1172
1173 int_ctl = (dte[2] >> AMDVI_IR_INTCTL_SHIFT) & 3;
1174 trace_amdvi_ir_intctl(int_ctl);
1175
1176 switch (int_ctl) {
1177 case AMDVI_IR_INTCTL_PASS:
1178 memcpy(translated, origin, sizeof(*origin));
1179 return 0;
1180 case AMDVI_IR_INTCTL_REMAP:
1181 break;
1182 case AMDVI_IR_INTCTL_ABORT:
1183 trace_amdvi_ir_target_abort("int_ctl abort");
1184 return -AMDVI_IR_TARGET_ABORT;
1185 default:
1186 trace_amdvi_ir_err("int_ctl reserved");
1187 return -AMDVI_IR_ERR;
1188 }
1189
1190 if (iommu->ga_enabled) {
1191 ret = amdvi_int_remap_ga(iommu, origin, translated, dte, irq, sid);
1192 } else {
1193 ret = amdvi_int_remap_legacy(iommu, origin, translated, dte, irq, sid);
1194 }
1195
1196 return ret;
1197 }
1198
1199 /* Interrupt remapping for MSI/MSI-X entry */
1200 static int amdvi_int_remap_msi(AMDVIState *iommu,
1201 MSIMessage *origin,
1202 MSIMessage *translated,
1203 uint16_t sid)
1204 {
1205 int ret = 0;
1206 uint64_t pass = 0;
1207 uint64_t dte[4] = { 0 };
1208 X86IOMMUIrq irq = { 0 };
1209 uint8_t dest_mode, delivery_mode;
1210
1211 assert(origin && translated);
1212
1213 /*
1214 * When IOMMU is enabled, interrupt remap request will come either from
1215 * IO-APIC or PCI device. If interrupt is from PCI device then it will
1216 * have a valid requester id but if the interrupt is from IO-APIC
1217 * then requester id will be invalid.
1218 */
1219 if (sid == X86_IOMMU_SID_INVALID) {
1220 sid = AMDVI_IOAPIC_SB_DEVID;
1221 }
1222
1223 trace_amdvi_ir_remap_msi_req(origin->address, origin->data, sid);
1224
1225 /* check if device table entry is set before we go further. */
1226 if (!iommu || !iommu->devtab_len) {
1227 memcpy(translated, origin, sizeof(*origin));
1228 goto out;
1229 }
1230
1231 if (!amdvi_get_dte(iommu, sid, dte)) {
1232 return -AMDVI_IR_ERR;
1233 }
1234
1235 /* Check if IR is enabled in DTE */
1236 if (!(dte[2] & AMDVI_IR_REMAP_ENABLE)) {
1237 memcpy(translated, origin, sizeof(*origin));
1238 goto out;
1239 }
1240
1241 /* validate that we are configure with intremap=on */
1242 if (!x86_iommu_ir_supported(X86_IOMMU_DEVICE(iommu))) {
1243 trace_amdvi_err("Interrupt remapping is enabled in the guest but "
1244 "not in the host. Use intremap=on to enable interrupt "
1245 "remapping in amd-iommu.");
1246 return -AMDVI_IR_ERR;
1247 }
1248
1249 if (origin->address & AMDVI_MSI_ADDR_HI_MASK) {
1250 trace_amdvi_err("MSI address high 32 bits non-zero when "
1251 "Interrupt Remapping enabled.");
1252 return -AMDVI_IR_ERR;
1253 }
1254
1255 if ((origin->address & AMDVI_MSI_ADDR_LO_MASK) != APIC_DEFAULT_ADDRESS) {
1256 trace_amdvi_err("MSI is not from IOAPIC.");
1257 return -AMDVI_IR_ERR;
1258 }
1259
1260 /*
1261 * The MSI data register [10:8] are used to get the upstream interrupt type.
1262 *
1263 * See MSI/MSI-X format:
1264 * https://pdfs.semanticscholar.org/presentation/9420/c279e942eca568157711ef5c92b800c40a79.pdf
1265 * (page 5)
1266 */
1267 delivery_mode = (origin->data >> MSI_DATA_DELIVERY_MODE_SHIFT) & 7;
1268
1269 switch (delivery_mode) {
1270 case AMDVI_IOAPIC_INT_TYPE_FIXED:
1271 case AMDVI_IOAPIC_INT_TYPE_ARBITRATED:
1272 trace_amdvi_ir_delivery_mode("fixed/arbitrated");
1273 ret = __amdvi_int_remap_msi(iommu, origin, translated, dte, &irq, sid);
1274 if (ret < 0) {
1275 goto remap_fail;
1276 } else {
1277 /* Translate IRQ to MSI messages */
1278 x86_iommu_irq_to_msi_message(&irq, translated);
1279 goto out;
1280 }
1281 break;
1282 case AMDVI_IOAPIC_INT_TYPE_SMI:
1283 error_report("SMI is not supported!");
1284 ret = -AMDVI_IR_ERR;
1285 break;
1286 case AMDVI_IOAPIC_INT_TYPE_NMI:
1287 pass = dte[3] & AMDVI_DEV_NMI_PASS_MASK;
1288 trace_amdvi_ir_delivery_mode("nmi");
1289 break;
1290 case AMDVI_IOAPIC_INT_TYPE_INIT:
1291 pass = dte[3] & AMDVI_DEV_INT_PASS_MASK;
1292 trace_amdvi_ir_delivery_mode("init");
1293 break;
1294 case AMDVI_IOAPIC_INT_TYPE_EINT:
1295 pass = dte[3] & AMDVI_DEV_EINT_PASS_MASK;
1296 trace_amdvi_ir_delivery_mode("eint");
1297 break;
1298 default:
1299 trace_amdvi_ir_delivery_mode("unsupported delivery_mode");
1300 ret = -AMDVI_IR_ERR;
1301 break;
1302 }
1303
1304 if (ret < 0) {
1305 goto remap_fail;
1306 }
1307
1308 /*
1309 * The MSI address register bit[2] is used to get the destination
1310 * mode. The dest_mode 1 is valid for fixed and arbitrated interrupts
1311 * only.
1312 */
1313 dest_mode = (origin->address >> MSI_ADDR_DEST_MODE_SHIFT) & 1;
1314 if (dest_mode) {
1315 trace_amdvi_ir_err("invalid dest_mode");
1316 ret = -AMDVI_IR_ERR;
1317 goto remap_fail;
1318 }
1319
1320 if (pass) {
1321 memcpy(translated, origin, sizeof(*origin));
1322 } else {
1323 trace_amdvi_ir_err("passthrough is not enabled");
1324 ret = -AMDVI_IR_ERR;
1325 goto remap_fail;
1326 }
1327
1328 out:
1329 trace_amdvi_ir_remap_msi(origin->address, origin->data,
1330 translated->address, translated->data);
1331 return 0;
1332
1333 remap_fail:
1334 return ret;
1335 }
1336
1337 static int amdvi_int_remap(X86IOMMUState *iommu,
1338 MSIMessage *origin,
1339 MSIMessage *translated,
1340 uint16_t sid)
1341 {
1342 return amdvi_int_remap_msi(AMD_IOMMU_DEVICE(iommu), origin,
1343 translated, sid);
1344 }
1345
1346 static MemTxResult amdvi_mem_ir_write(void *opaque, hwaddr addr,
1347 uint64_t value, unsigned size,
1348 MemTxAttrs attrs)
1349 {
1350 int ret;
1351 MSIMessage from = { 0, 0 }, to = { 0, 0 };
1352 uint16_t sid = AMDVI_IOAPIC_SB_DEVID;
1353
1354 from.address = (uint64_t) addr + AMDVI_INT_ADDR_FIRST;
1355 from.data = (uint32_t) value;
1356
1357 trace_amdvi_mem_ir_write_req(addr, value, size);
1358
1359 if (!attrs.unspecified) {
1360 /* We have explicit Source ID */
1361 sid = attrs.requester_id;
1362 }
1363
1364 ret = amdvi_int_remap_msi(opaque, &from, &to, sid);
1365 if (ret < 0) {
1366 /* TODO: log the event using IOMMU log event interface */
1367 error_report_once("failed to remap interrupt from devid 0x%x", sid);
1368 return MEMTX_ERROR;
1369 }
1370
1371 apic_get_class()->send_msi(&to);
1372
1373 trace_amdvi_mem_ir_write(to.address, to.data);
1374 return MEMTX_OK;
1375 }
1376
1377 static MemTxResult amdvi_mem_ir_read(void *opaque, hwaddr addr,
1378 uint64_t *data, unsigned size,
1379 MemTxAttrs attrs)
1380 {
1381 return MEMTX_OK;
1382 }
1383
1384 static const MemoryRegionOps amdvi_ir_ops = {
1385 .read_with_attrs = amdvi_mem_ir_read,
1386 .write_with_attrs = amdvi_mem_ir_write,
1387 .endianness = DEVICE_LITTLE_ENDIAN,
1388 .impl = {
1389 .min_access_size = 4,
1390 .max_access_size = 4,
1391 },
1392 .valid = {
1393 .min_access_size = 4,
1394 .max_access_size = 4,
1395 }
1396 };
1397
1398 static AddressSpace *amdvi_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
1399 {
1400 char name[128];
1401 AMDVIState *s = opaque;
1402 AMDVIAddressSpace **iommu_as, *amdvi_dev_as;
1403 int bus_num = pci_bus_num(bus);
1404
1405 iommu_as = s->address_spaces[bus_num];
1406
1407 /* allocate memory during the first run */
1408 if (!iommu_as) {
1409 iommu_as = g_new0(AMDVIAddressSpace *, PCI_DEVFN_MAX);
1410 s->address_spaces[bus_num] = iommu_as;
1411 }
1412
1413 /* set up AMD-Vi region */
1414 if (!iommu_as[devfn]) {
1415 snprintf(name, sizeof(name), "amd_iommu_devfn_%d", devfn);
1416
1417 iommu_as[devfn] = g_new0(AMDVIAddressSpace, 1);
1418 iommu_as[devfn]->bus_num = (uint8_t)bus_num;
1419 iommu_as[devfn]->devfn = (uint8_t)devfn;
1420 iommu_as[devfn]->iommu_state = s;
1421
1422 amdvi_dev_as = iommu_as[devfn];
1423
1424 /*
1425 * Memory region relationships looks like (Address range shows
1426 * only lower 32 bits to make it short in length...):
1427 *
1428 * |-----------------+-------------------+----------|
1429 * | Name | Address range | Priority |
1430 * |-----------------+-------------------+----------+
1431 * | amdvi_root | 00000000-ffffffff | 0 |
1432 * | amdvi_iommu | 00000000-ffffffff | 1 |
1433 * | amdvi_iommu_ir | fee00000-feefffff | 64 |
1434 * |-----------------+-------------------+----------|
1435 */
1436 memory_region_init_iommu(&amdvi_dev_as->iommu,
1437 sizeof(amdvi_dev_as->iommu),
1438 TYPE_AMD_IOMMU_MEMORY_REGION,
1439 OBJECT(s),
1440 "amd_iommu", UINT64_MAX);
1441 memory_region_init(&amdvi_dev_as->root, OBJECT(s),
1442 "amdvi_root", UINT64_MAX);
1443 address_space_init(&amdvi_dev_as->as, &amdvi_dev_as->root, name);
1444 memory_region_init_io(&amdvi_dev_as->iommu_ir, OBJECT(s),
1445 &amdvi_ir_ops, s, "amd_iommu_ir",
1446 AMDVI_INT_ADDR_SIZE);
1447 memory_region_add_subregion_overlap(&amdvi_dev_as->root,
1448 AMDVI_INT_ADDR_FIRST,
1449 &amdvi_dev_as->iommu_ir,
1450 64);
1451 memory_region_add_subregion_overlap(&amdvi_dev_as->root, 0,
1452 MEMORY_REGION(&amdvi_dev_as->iommu),
1453 1);
1454 }
1455 return &iommu_as[devfn]->as;
1456 }
1457
1458 static const MemoryRegionOps mmio_mem_ops = {
1459 .read = amdvi_mmio_read,
1460 .write = amdvi_mmio_write,
1461 .endianness = DEVICE_LITTLE_ENDIAN,
1462 .impl = {
1463 .min_access_size = 1,
1464 .max_access_size = 8,
1465 .unaligned = false,
1466 },
1467 .valid = {
1468 .min_access_size = 1,
1469 .max_access_size = 8,
1470 }
1471 };
1472
1473 static int amdvi_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu,
1474 IOMMUNotifierFlag old,
1475 IOMMUNotifierFlag new,
1476 Error **errp)
1477 {
1478 AMDVIAddressSpace *as = container_of(iommu, AMDVIAddressSpace, iommu);
1479
1480 if (new & IOMMU_NOTIFIER_MAP) {
1481 error_setg(errp,
1482 "device %02x.%02x.%x requires iommu notifier which is not "
1483 "currently supported", as->bus_num, PCI_SLOT(as->devfn),
1484 PCI_FUNC(as->devfn));
1485 return -EINVAL;
1486 }
1487 return 0;
1488 }
1489
1490 static void amdvi_init(AMDVIState *s)
1491 {
1492 amdvi_iotlb_reset(s);
1493
1494 s->devtab_len = 0;
1495 s->cmdbuf_len = 0;
1496 s->cmdbuf_head = 0;
1497 s->cmdbuf_tail = 0;
1498 s->evtlog_head = 0;
1499 s->evtlog_tail = 0;
1500 s->excl_enabled = false;
1501 s->excl_allow = false;
1502 s->mmio_enabled = false;
1503 s->enabled = false;
1504 s->ats_enabled = false;
1505 s->cmdbuf_enabled = false;
1506
1507 /* reset MMIO */
1508 memset(s->mmior, 0, AMDVI_MMIO_SIZE);
1509 amdvi_set_quad(s, AMDVI_MMIO_EXT_FEATURES, AMDVI_EXT_FEATURES,
1510 0xffffffffffffffef, 0);
1511 amdvi_set_quad(s, AMDVI_MMIO_STATUS, 0, 0x98, 0x67);
1512
1513 /* reset device ident */
1514 pci_config_set_vendor_id(s->pci.dev.config, PCI_VENDOR_ID_AMD);
1515 pci_config_set_prog_interface(s->pci.dev.config, 00);
1516 pci_config_set_device_id(s->pci.dev.config, s->devid);
1517 pci_config_set_class(s->pci.dev.config, 0x0806);
1518
1519 /* reset AMDVI specific capabilities, all r/o */
1520 pci_set_long(s->pci.dev.config + s->capab_offset, AMDVI_CAPAB_FEATURES);
1521 pci_set_long(s->pci.dev.config + s->capab_offset + AMDVI_CAPAB_BAR_LOW,
1522 s->mmio.addr & ~(0xffff0000));
1523 pci_set_long(s->pci.dev.config + s->capab_offset + AMDVI_CAPAB_BAR_HIGH,
1524 (s->mmio.addr & ~(0xffff)) >> 16);
1525 pci_set_long(s->pci.dev.config + s->capab_offset + AMDVI_CAPAB_RANGE,
1526 0xff000000);
1527 pci_set_long(s->pci.dev.config + s->capab_offset + AMDVI_CAPAB_MISC, 0);
1528 pci_set_long(s->pci.dev.config + s->capab_offset + AMDVI_CAPAB_MISC,
1529 AMDVI_MAX_PH_ADDR | AMDVI_MAX_GVA_ADDR | AMDVI_MAX_VA_ADDR);
1530 }
1531
1532 static void amdvi_sysbus_reset(DeviceState *dev)
1533 {
1534 AMDVIState *s = AMD_IOMMU_DEVICE(dev);
1535
1536 msi_reset(&s->pci.dev);
1537 amdvi_init(s);
1538 }
1539
1540 static void amdvi_sysbus_realize(DeviceState *dev, Error **errp)
1541 {
1542 int ret = 0;
1543 AMDVIState *s = AMD_IOMMU_DEVICE(dev);
1544 MachineState *ms = MACHINE(qdev_get_machine());
1545 PCMachineState *pcms = PC_MACHINE(ms);
1546 X86MachineState *x86ms = X86_MACHINE(ms);
1547 PCIBus *bus = pcms->bus;
1548
1549 s->iotlb = g_hash_table_new_full(amdvi_uint64_hash,
1550 amdvi_uint64_equal, g_free, g_free);
1551
1552 /* This device should take care of IOMMU PCI properties */
1553 if (!qdev_realize(DEVICE(&s->pci), &bus->qbus, errp)) {
1554 return;
1555 }
1556 ret = pci_add_capability(&s->pci.dev, AMDVI_CAPAB_ID_SEC, 0,
1557 AMDVI_CAPAB_SIZE, errp);
1558 if (ret < 0) {
1559 return;
1560 }
1561 s->capab_offset = ret;
1562
1563 ret = pci_add_capability(&s->pci.dev, PCI_CAP_ID_MSI, 0,
1564 AMDVI_CAPAB_REG_SIZE, errp);
1565 if (ret < 0) {
1566 return;
1567 }
1568 ret = pci_add_capability(&s->pci.dev, PCI_CAP_ID_HT, 0,
1569 AMDVI_CAPAB_REG_SIZE, errp);
1570 if (ret < 0) {
1571 return;
1572 }
1573
1574 /* Pseudo address space under root PCI bus. */
1575 x86ms->ioapic_as = amdvi_host_dma_iommu(bus, s, AMDVI_IOAPIC_SB_DEVID);
1576
1577 /* set up MMIO */
1578 memory_region_init_io(&s->mmio, OBJECT(s), &mmio_mem_ops, s, "amdvi-mmio",
1579 AMDVI_MMIO_SIZE);
1580
1581 sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->mmio);
1582 sysbus_mmio_map(SYS_BUS_DEVICE(s), 0, AMDVI_BASE_ADDR);
1583 pci_setup_iommu(bus, amdvi_host_dma_iommu, s);
1584 s->devid = object_property_get_int(OBJECT(&s->pci), "addr", &error_abort);
1585 msi_init(&s->pci.dev, 0, 1, true, false, errp);
1586 amdvi_init(s);
1587 }
1588
1589 static const VMStateDescription vmstate_amdvi_sysbus = {
1590 .name = "amd-iommu",
1591 .unmigratable = 1
1592 };
1593
1594 static void amdvi_sysbus_instance_init(Object *klass)
1595 {
1596 AMDVIState *s = AMD_IOMMU_DEVICE(klass);
1597
1598 object_initialize(&s->pci, sizeof(s->pci), TYPE_AMD_IOMMU_PCI);
1599 }
1600
1601 static void amdvi_sysbus_class_init(ObjectClass *klass, void *data)
1602 {
1603 DeviceClass *dc = DEVICE_CLASS(klass);
1604 X86IOMMUClass *dc_class = X86_IOMMU_DEVICE_CLASS(klass);
1605
1606 dc->reset = amdvi_sysbus_reset;
1607 dc->vmsd = &vmstate_amdvi_sysbus;
1608 dc->hotpluggable = false;
1609 dc_class->realize = amdvi_sysbus_realize;
1610 dc_class->int_remap = amdvi_int_remap;
1611 /* Supported by the pc-q35-* machine types */
1612 dc->user_creatable = true;
1613 set_bit(DEVICE_CATEGORY_MISC, dc->categories);
1614 dc->desc = "AMD IOMMU (AMD-Vi) DMA Remapping device";
1615 }
1616
1617 static const TypeInfo amdvi_sysbus = {
1618 .name = TYPE_AMD_IOMMU_DEVICE,
1619 .parent = TYPE_X86_IOMMU_DEVICE,
1620 .instance_size = sizeof(AMDVIState),
1621 .instance_init = amdvi_sysbus_instance_init,
1622 .class_init = amdvi_sysbus_class_init
1623 };
1624
1625 static void amdvi_pci_class_init(ObjectClass *klass, void *data)
1626 {
1627 DeviceClass *dc = DEVICE_CLASS(klass);
1628
1629 set_bit(DEVICE_CATEGORY_MISC, dc->categories);
1630 dc->desc = "AMD IOMMU (AMD-Vi) DMA Remapping device";
1631 }
1632
1633 static const TypeInfo amdvi_pci = {
1634 .name = TYPE_AMD_IOMMU_PCI,
1635 .parent = TYPE_PCI_DEVICE,
1636 .instance_size = sizeof(AMDVIPCIState),
1637 .class_init = amdvi_pci_class_init,
1638 .interfaces = (InterfaceInfo[]) {
1639 { INTERFACE_CONVENTIONAL_PCI_DEVICE },
1640 { },
1641 },
1642 };
1643
1644 static void amdvi_iommu_memory_region_class_init(ObjectClass *klass, void *data)
1645 {
1646 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
1647
1648 imrc->translate = amdvi_translate;
1649 imrc->notify_flag_changed = amdvi_iommu_notify_flag_changed;
1650 }
1651
1652 static const TypeInfo amdvi_iommu_memory_region_info = {
1653 .parent = TYPE_IOMMU_MEMORY_REGION,
1654 .name = TYPE_AMD_IOMMU_MEMORY_REGION,
1655 .class_init = amdvi_iommu_memory_region_class_init,
1656 };
1657
1658 static void amdvi_register_types(void)
1659 {
1660 type_register_static(&amdvi_pci);
1661 type_register_static(&amdvi_sysbus);
1662 type_register_static(&amdvi_iommu_memory_region_info);
1663 }
1664
1665 type_init(amdvi_register_types);
QEmu