2 * QEMU emulation of AMD IOMMU (AMD-Vi)
4 * Copyright (C) 2011 Eduard - Gabriel Munteanu
5 * Copyright (C) 2015, 2016 David Kiarie Kahurani
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License along
18 * with this program; if not, see <http://www.gnu.org/licenses/>.
20 * Cache implementation inspired by hw/i386/intel_iommu.c
23 #include "qemu/osdep.h"
24 #include "hw/i386/pc.h"
25 #include "hw/pci/msi.h"
26 #include "hw/pci/pci_bus.h"
27 #include "migration/vmstate.h"
28 #include "amd_iommu.h"
29 #include "qapi/error.h"
30 #include "qemu/error-report.h"
31 #include "hw/i386/apic_internal.h"
33 #include "hw/i386/apic-msidef.h"
35 /* used AMD-Vi MMIO registers */
36 const char *amdvi_mmio_low
[] = {
37 "AMDVI_MMIO_DEVTAB_BASE",
38 "AMDVI_MMIO_CMDBUF_BASE",
39 "AMDVI_MMIO_EVTLOG_BASE",
41 "AMDVI_MMIO_EXCL_BASE",
42 "AMDVI_MMIO_EXCL_LIMIT",
43 "AMDVI_MMIO_EXT_FEATURES",
44 "AMDVI_MMIO_PPR_BASE",
47 const char *amdvi_mmio_high
[] = {
48 "AMDVI_MMIO_COMMAND_HEAD",
49 "AMDVI_MMIO_COMMAND_TAIL",
50 "AMDVI_MMIO_EVTLOG_HEAD",
51 "AMDVI_MMIO_EVTLOG_TAIL",
53 "AMDVI_MMIO_PPR_HEAD",
54 "AMDVI_MMIO_PPR_TAIL",
58 struct AMDVIAddressSpace
{
59 uint8_t bus_num
; /* bus number */
60 uint8_t devfn
; /* device function */
61 AMDVIState
*iommu_state
; /* AMDVI - one per machine */
62 MemoryRegion root
; /* AMDVI Root memory map region */
63 IOMMUMemoryRegion iommu
; /* Device's address translation region */
64 MemoryRegion iommu_ir
; /* Device's interrupt remapping region */
65 AddressSpace as
; /* device's corresponding address space */
68 /* AMDVI cache entry */
69 typedef struct AMDVIIOTLBEntry
{
70 uint16_t domid
; /* assigned domain id */
71 uint16_t devid
; /* device owning entry */
72 uint64_t perms
; /* access permissions */
73 uint64_t translated_addr
; /* translated address */
74 uint64_t page_mask
; /* physical page size */
77 /* configure MMIO registers at startup/reset */
78 static void amdvi_set_quad(AMDVIState
*s
, hwaddr addr
, uint64_t val
,
79 uint64_t romask
, uint64_t w1cmask
)
81 stq_le_p(&s
->mmior
[addr
], val
);
82 stq_le_p(&s
->romask
[addr
], romask
);
83 stq_le_p(&s
->w1cmask
[addr
], w1cmask
);
86 static uint16_t amdvi_readw(AMDVIState
*s
, hwaddr addr
)
88 return lduw_le_p(&s
->mmior
[addr
]);
91 static uint32_t amdvi_readl(AMDVIState
*s
, hwaddr addr
)
93 return ldl_le_p(&s
->mmior
[addr
]);
96 static uint64_t amdvi_readq(AMDVIState
*s
, hwaddr addr
)
98 return ldq_le_p(&s
->mmior
[addr
]);
102 static void amdvi_writeq_raw(AMDVIState
*s
, hwaddr addr
, uint64_t val
)
104 stq_le_p(&s
->mmior
[addr
], val
);
108 static void amdvi_writew(AMDVIState
*s
, hwaddr addr
, uint16_t val
)
110 uint16_t romask
= lduw_le_p(&s
->romask
[addr
]);
111 uint16_t w1cmask
= lduw_le_p(&s
->w1cmask
[addr
]);
112 uint16_t oldval
= lduw_le_p(&s
->mmior
[addr
]);
113 stw_le_p(&s
->mmior
[addr
],
114 ((oldval
& romask
) | (val
& ~romask
)) & ~(val
& w1cmask
));
117 static void amdvi_writel(AMDVIState
*s
, hwaddr addr
, uint32_t val
)
119 uint32_t romask
= ldl_le_p(&s
->romask
[addr
]);
120 uint32_t w1cmask
= ldl_le_p(&s
->w1cmask
[addr
]);
121 uint32_t oldval
= ldl_le_p(&s
->mmior
[addr
]);
122 stl_le_p(&s
->mmior
[addr
],
123 ((oldval
& romask
) | (val
& ~romask
)) & ~(val
& w1cmask
));
126 static void amdvi_writeq(AMDVIState
*s
, hwaddr addr
, uint64_t val
)
128 uint64_t romask
= ldq_le_p(&s
->romask
[addr
]);
129 uint64_t w1cmask
= ldq_le_p(&s
->w1cmask
[addr
]);
130 uint32_t oldval
= ldq_le_p(&s
->mmior
[addr
]);
131 stq_le_p(&s
->mmior
[addr
],
132 ((oldval
& romask
) | (val
& ~romask
)) & ~(val
& w1cmask
));
135 /* OR a 64-bit register with a 64-bit value */
136 static bool amdvi_test_mask(AMDVIState
*s
, hwaddr addr
, uint64_t val
)
138 return amdvi_readq(s
, addr
) | val
;
141 /* OR a 64-bit register with a 64-bit value storing result in the register */
142 static void amdvi_assign_orq(AMDVIState
*s
, hwaddr addr
, uint64_t val
)
144 amdvi_writeq_raw(s
, addr
, amdvi_readq(s
, addr
) | val
);
147 /* AND a 64-bit register with a 64-bit value storing result in the register */
148 static void amdvi_assign_andq(AMDVIState
*s
, hwaddr addr
, uint64_t val
)
150 amdvi_writeq_raw(s
, addr
, amdvi_readq(s
, addr
) & val
);
153 static void amdvi_generate_msi_interrupt(AMDVIState
*s
)
157 .requester_id
= pci_requester_id(&s
->pci
.dev
)
160 if (msi_enabled(&s
->pci
.dev
)) {
161 msg
= msi_get_message(&s
->pci
.dev
, 0);
162 address_space_stl_le(&address_space_memory
, msg
.address
, msg
.data
,
167 static void amdvi_log_event(AMDVIState
*s
, uint64_t *evt
)
169 /* event logging not enabled */
170 if (!s
->evtlog_enabled
|| amdvi_test_mask(s
, AMDVI_MMIO_STATUS
,
171 AMDVI_MMIO_STATUS_EVT_OVF
)) {
175 /* event log buffer full */
176 if (s
->evtlog_tail
>= s
->evtlog_len
) {
177 amdvi_assign_orq(s
, AMDVI_MMIO_STATUS
, AMDVI_MMIO_STATUS_EVT_OVF
);
178 /* generate interrupt */
179 amdvi_generate_msi_interrupt(s
);
183 if (dma_memory_write(&address_space_memory
, s
->evtlog
+ s
->evtlog_tail
,
184 evt
, AMDVI_EVENT_LEN
, MEMTXATTRS_UNSPECIFIED
)) {
185 trace_amdvi_evntlog_fail(s
->evtlog
, s
->evtlog_tail
);
188 s
->evtlog_tail
+= AMDVI_EVENT_LEN
;
189 amdvi_assign_orq(s
, AMDVI_MMIO_STATUS
, AMDVI_MMIO_STATUS_COMP_INT
);
190 amdvi_generate_msi_interrupt(s
);
193 static void amdvi_setevent_bits(uint64_t *buffer
, uint64_t value
, int start
,
196 int index
= start
/ 64, bitpos
= start
% 64;
197 uint64_t mask
= MAKE_64BIT_MASK(start
, length
);
198 buffer
[index
] &= ~mask
;
199 buffer
[index
] |= (value
<< bitpos
) & mask
;
202 * AMDVi event structure
204 * 48:63 -> event type + miscellaneous info
205 * 64:127 -> related address
207 static void amdvi_encode_event(uint64_t *evt
, uint16_t devid
, uint64_t addr
,
213 amdvi_setevent_bits(evt
, devid
, 0, 16);
214 amdvi_setevent_bits(evt
, info
, 48, 16);
215 amdvi_setevent_bits(evt
, addr
, 64, 64);
217 /* log an error encountered during a page walk
219 * @addr: virtual address in translation request
221 static void amdvi_page_fault(AMDVIState
*s
, uint16_t devid
,
222 hwaddr addr
, uint16_t info
)
226 info
|= AMDVI_EVENT_IOPF_I
| AMDVI_EVENT_IOPF
;
227 amdvi_encode_event(evt
, devid
, addr
, info
);
228 amdvi_log_event(s
, evt
);
229 pci_word_test_and_set_mask(s
->pci
.dev
.config
+ PCI_STATUS
,
230 PCI_STATUS_SIG_TARGET_ABORT
);
233 * log a master abort accessing device table
234 * @devtab : address of device table entry
235 * @info : error flags
237 static void amdvi_log_devtab_error(AMDVIState
*s
, uint16_t devid
,
238 hwaddr devtab
, uint16_t info
)
242 info
|= AMDVI_EVENT_DEV_TAB_HW_ERROR
;
244 amdvi_encode_event(evt
, devid
, devtab
, info
);
245 amdvi_log_event(s
, evt
);
246 pci_word_test_and_set_mask(s
->pci
.dev
.config
+ PCI_STATUS
,
247 PCI_STATUS_SIG_TARGET_ABORT
);
249 /* log an event trying to access command buffer
250 * @addr : address that couldn't be accessed
252 static void amdvi_log_command_error(AMDVIState
*s
, hwaddr addr
)
255 uint16_t info
= AMDVI_EVENT_COMMAND_HW_ERROR
;
257 amdvi_encode_event(evt
, 0, addr
, info
);
258 amdvi_log_event(s
, evt
);
259 pci_word_test_and_set_mask(s
->pci
.dev
.config
+ PCI_STATUS
,
260 PCI_STATUS_SIG_TARGET_ABORT
);
262 /* log an illegal command event
263 * @addr : address of illegal command
265 static void amdvi_log_illegalcom_error(AMDVIState
*s
, uint16_t info
,
270 info
|= AMDVI_EVENT_ILLEGAL_COMMAND_ERROR
;
271 amdvi_encode_event(evt
, 0, addr
, info
);
272 amdvi_log_event(s
, evt
);
274 /* log an error accessing device table
276 * @devid : device owning the table entry
277 * @devtab : address of device table entry
278 * @info : error flags
280 static void amdvi_log_illegaldevtab_error(AMDVIState
*s
, uint16_t devid
,
281 hwaddr addr
, uint16_t info
)
285 info
|= AMDVI_EVENT_ILLEGAL_DEVTAB_ENTRY
;
286 amdvi_encode_event(evt
, devid
, addr
, info
);
287 amdvi_log_event(s
, evt
);
289 /* log an error accessing a PTE entry
290 * @addr : address that couldn't be accessed
292 static void amdvi_log_pagetab_error(AMDVIState
*s
, uint16_t devid
,
293 hwaddr addr
, uint16_t info
)
297 info
|= AMDVI_EVENT_PAGE_TAB_HW_ERROR
;
298 amdvi_encode_event(evt
, devid
, addr
, info
);
299 amdvi_log_event(s
, evt
);
300 pci_word_test_and_set_mask(s
->pci
.dev
.config
+ PCI_STATUS
,
301 PCI_STATUS_SIG_TARGET_ABORT
);
304 static gboolean
amdvi_uint64_equal(gconstpointer v1
, gconstpointer v2
)
306 return *((const uint64_t *)v1
) == *((const uint64_t *)v2
);
309 static guint
amdvi_uint64_hash(gconstpointer v
)
311 return (guint
)*(const uint64_t *)v
;
314 static AMDVIIOTLBEntry
*amdvi_iotlb_lookup(AMDVIState
*s
, hwaddr addr
,
317 uint64_t key
= (addr
>> AMDVI_PAGE_SHIFT_4K
) |
318 ((uint64_t)(devid
) << AMDVI_DEVID_SHIFT
);
319 return g_hash_table_lookup(s
->iotlb
, &key
);
322 static void amdvi_iotlb_reset(AMDVIState
*s
)
325 trace_amdvi_iotlb_reset();
326 g_hash_table_remove_all(s
->iotlb
);
329 static gboolean
amdvi_iotlb_remove_by_devid(gpointer key
, gpointer value
,
332 AMDVIIOTLBEntry
*entry
= (AMDVIIOTLBEntry
*)value
;
333 uint16_t devid
= *(uint16_t *)user_data
;
334 return entry
->devid
== devid
;
337 static void amdvi_iotlb_remove_page(AMDVIState
*s
, hwaddr addr
,
340 uint64_t key
= (addr
>> AMDVI_PAGE_SHIFT_4K
) |
341 ((uint64_t)(devid
) << AMDVI_DEVID_SHIFT
);
342 g_hash_table_remove(s
->iotlb
, &key
);
345 static void amdvi_update_iotlb(AMDVIState
*s
, uint16_t devid
,
346 uint64_t gpa
, IOMMUTLBEntry to_cache
,
349 AMDVIIOTLBEntry
*entry
= g_new(AMDVIIOTLBEntry
, 1);
350 uint64_t *key
= g_new(uint64_t, 1);
351 uint64_t gfn
= gpa
>> AMDVI_PAGE_SHIFT_4K
;
353 /* don't cache erroneous translations */
354 if (to_cache
.perm
!= IOMMU_NONE
) {
355 trace_amdvi_cache_update(domid
, PCI_BUS_NUM(devid
), PCI_SLOT(devid
),
356 PCI_FUNC(devid
), gpa
, to_cache
.translated_addr
);
358 if (g_hash_table_size(s
->iotlb
) >= AMDVI_IOTLB_MAX_SIZE
) {
359 amdvi_iotlb_reset(s
);
362 entry
->domid
= domid
;
363 entry
->perms
= to_cache
.perm
;
364 entry
->translated_addr
= to_cache
.translated_addr
;
365 entry
->page_mask
= to_cache
.addr_mask
;
366 *key
= gfn
| ((uint64_t)(devid
) << AMDVI_DEVID_SHIFT
);
367 g_hash_table_replace(s
->iotlb
, key
, entry
);
371 static void amdvi_completion_wait(AMDVIState
*s
, uint64_t *cmd
)
373 /* pad the last 3 bits */
374 hwaddr addr
= cpu_to_le64(extract64(cmd
[0], 3, 49)) << 3;
375 uint64_t data
= cpu_to_le64(cmd
[1]);
377 if (extract64(cmd
[0], 52, 8)) {
378 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
379 s
->cmdbuf
+ s
->cmdbuf_head
);
381 if (extract64(cmd
[0], 0, 1)) {
382 if (dma_memory_write(&address_space_memory
, addr
, &data
,
383 AMDVI_COMPLETION_DATA_SIZE
,
384 MEMTXATTRS_UNSPECIFIED
)) {
385 trace_amdvi_completion_wait_fail(addr
);
388 /* set completion interrupt */
389 if (extract64(cmd
[0], 1, 1)) {
390 amdvi_assign_orq(s
, AMDVI_MMIO_STATUS
, AMDVI_MMIO_STATUS_COMP_INT
);
391 /* generate interrupt */
392 amdvi_generate_msi_interrupt(s
);
394 trace_amdvi_completion_wait(addr
, data
);
397 /* log error without aborting since linux seems to be using reserved bits */
398 static void amdvi_inval_devtab_entry(AMDVIState
*s
, uint64_t *cmd
)
400 uint16_t devid
= cpu_to_le16((uint16_t)extract64(cmd
[0], 0, 16));
402 /* This command should invalidate internal caches of which there isn't */
403 if (extract64(cmd
[0], 16, 44) || cmd
[1]) {
404 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
405 s
->cmdbuf
+ s
->cmdbuf_head
);
407 trace_amdvi_devtab_inval(PCI_BUS_NUM(devid
), PCI_SLOT(devid
),
411 static void amdvi_complete_ppr(AMDVIState
*s
, uint64_t *cmd
)
413 if (extract64(cmd
[0], 16, 16) || extract64(cmd
[0], 52, 8) ||
414 extract64(cmd
[1], 0, 2) || extract64(cmd
[1], 3, 29)
415 || extract64(cmd
[1], 48, 16)) {
416 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
417 s
->cmdbuf
+ s
->cmdbuf_head
);
419 trace_amdvi_ppr_exec();
422 static void amdvi_inval_all(AMDVIState
*s
, uint64_t *cmd
)
424 if (extract64(cmd
[0], 0, 60) || cmd
[1]) {
425 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
426 s
->cmdbuf
+ s
->cmdbuf_head
);
429 amdvi_iotlb_reset(s
);
430 trace_amdvi_all_inval();
433 static gboolean
amdvi_iotlb_remove_by_domid(gpointer key
, gpointer value
,
436 AMDVIIOTLBEntry
*entry
= (AMDVIIOTLBEntry
*)value
;
437 uint16_t domid
= *(uint16_t *)user_data
;
438 return entry
->domid
== domid
;
441 /* we don't have devid - we can't remove pages by address */
442 static void amdvi_inval_pages(AMDVIState
*s
, uint64_t *cmd
)
444 uint16_t domid
= cpu_to_le16((uint16_t)extract64(cmd
[0], 32, 16));
446 if (extract64(cmd
[0], 20, 12) || extract64(cmd
[0], 48, 12) ||
447 extract64(cmd
[1], 3, 9)) {
448 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
449 s
->cmdbuf
+ s
->cmdbuf_head
);
452 g_hash_table_foreach_remove(s
->iotlb
, amdvi_iotlb_remove_by_domid
,
454 trace_amdvi_pages_inval(domid
);
457 static void amdvi_prefetch_pages(AMDVIState
*s
, uint64_t *cmd
)
459 if (extract64(cmd
[0], 16, 8) || extract64(cmd
[0], 52, 8) ||
460 extract64(cmd
[1], 1, 1) || extract64(cmd
[1], 3, 1) ||
461 extract64(cmd
[1], 5, 7)) {
462 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
463 s
->cmdbuf
+ s
->cmdbuf_head
);
466 trace_amdvi_prefetch_pages();
469 static void amdvi_inval_inttable(AMDVIState
*s
, uint64_t *cmd
)
471 if (extract64(cmd
[0], 16, 44) || cmd
[1]) {
472 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
473 s
->cmdbuf
+ s
->cmdbuf_head
);
477 trace_amdvi_intr_inval();
480 /* FIXME: Try to work with the specified size instead of all the pages
481 * when the S bit is on
483 static void iommu_inval_iotlb(AMDVIState
*s
, uint64_t *cmd
)
486 uint16_t devid
= extract64(cmd
[0], 0, 16);
487 if (extract64(cmd
[1], 1, 1) || extract64(cmd
[1], 3, 1) ||
488 extract64(cmd
[1], 6, 6)) {
489 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
490 s
->cmdbuf
+ s
->cmdbuf_head
);
494 if (extract64(cmd
[1], 0, 1)) {
495 g_hash_table_foreach_remove(s
->iotlb
, amdvi_iotlb_remove_by_devid
,
498 amdvi_iotlb_remove_page(s
, cpu_to_le64(extract64(cmd
[1], 12, 52)) << 12,
499 cpu_to_le16(extract64(cmd
[1], 0, 16)));
501 trace_amdvi_iotlb_inval();
504 /* not honouring reserved bits is regarded as an illegal command */
505 static void amdvi_cmdbuf_exec(AMDVIState
*s
)
509 if (dma_memory_read(&address_space_memory
, s
->cmdbuf
+ s
->cmdbuf_head
,
510 cmd
, AMDVI_COMMAND_SIZE
, MEMTXATTRS_UNSPECIFIED
)) {
511 trace_amdvi_command_read_fail(s
->cmdbuf
, s
->cmdbuf_head
);
512 amdvi_log_command_error(s
, s
->cmdbuf
+ s
->cmdbuf_head
);
516 switch (extract64(cmd
[0], 60, 4)) {
517 case AMDVI_CMD_COMPLETION_WAIT
:
518 amdvi_completion_wait(s
, cmd
);
520 case AMDVI_CMD_INVAL_DEVTAB_ENTRY
:
521 amdvi_inval_devtab_entry(s
, cmd
);
523 case AMDVI_CMD_INVAL_AMDVI_PAGES
:
524 amdvi_inval_pages(s
, cmd
);
526 case AMDVI_CMD_INVAL_IOTLB_PAGES
:
527 iommu_inval_iotlb(s
, cmd
);
529 case AMDVI_CMD_INVAL_INTR_TABLE
:
530 amdvi_inval_inttable(s
, cmd
);
532 case AMDVI_CMD_PREFETCH_AMDVI_PAGES
:
533 amdvi_prefetch_pages(s
, cmd
);
535 case AMDVI_CMD_COMPLETE_PPR_REQUEST
:
536 amdvi_complete_ppr(s
, cmd
);
538 case AMDVI_CMD_INVAL_AMDVI_ALL
:
539 amdvi_inval_all(s
, cmd
);
542 trace_amdvi_unhandled_command(extract64(cmd
[1], 60, 4));
543 /* log illegal command */
544 amdvi_log_illegalcom_error(s
, extract64(cmd
[1], 60, 4),
545 s
->cmdbuf
+ s
->cmdbuf_head
);
549 static void amdvi_cmdbuf_run(AMDVIState
*s
)
551 if (!s
->cmdbuf_enabled
) {
552 trace_amdvi_command_error(amdvi_readq(s
, AMDVI_MMIO_CONTROL
));
556 /* check if there is work to do. */
557 while (s
->cmdbuf_head
!= s
->cmdbuf_tail
) {
558 trace_amdvi_command_exec(s
->cmdbuf_head
, s
->cmdbuf_tail
, s
->cmdbuf
);
559 amdvi_cmdbuf_exec(s
);
560 s
->cmdbuf_head
+= AMDVI_COMMAND_SIZE
;
561 amdvi_writeq_raw(s
, AMDVI_MMIO_COMMAND_HEAD
, s
->cmdbuf_head
);
563 /* wrap head pointer */
564 if (s
->cmdbuf_head
>= s
->cmdbuf_len
* AMDVI_COMMAND_SIZE
) {
570 static void amdvi_mmio_trace(hwaddr addr
, unsigned size
)
572 uint8_t index
= (addr
& ~0x2000) / 8;
574 if ((addr
& 0x2000)) {
576 index
= index
>= AMDVI_MMIO_REGS_HIGH
? AMDVI_MMIO_REGS_HIGH
: index
;
577 trace_amdvi_mmio_read(amdvi_mmio_high
[index
], addr
, size
, addr
& ~0x07);
579 index
= index
>= AMDVI_MMIO_REGS_LOW
? AMDVI_MMIO_REGS_LOW
: index
;
580 trace_amdvi_mmio_read(amdvi_mmio_low
[index
], addr
, size
, addr
& ~0x07);
584 static uint64_t amdvi_mmio_read(void *opaque
, hwaddr addr
, unsigned size
)
586 AMDVIState
*s
= opaque
;
589 if (addr
+ size
> AMDVI_MMIO_SIZE
) {
590 trace_amdvi_mmio_read_invalid(AMDVI_MMIO_SIZE
, addr
, size
);
595 val
= amdvi_readw(s
, addr
);
596 } else if (size
== 4) {
597 val
= amdvi_readl(s
, addr
);
598 } else if (size
== 8) {
599 val
= amdvi_readq(s
, addr
);
601 amdvi_mmio_trace(addr
, size
);
606 static void amdvi_handle_control_write(AMDVIState
*s
)
608 unsigned long control
= amdvi_readq(s
, AMDVI_MMIO_CONTROL
);
609 s
->enabled
= !!(control
& AMDVI_MMIO_CONTROL_AMDVIEN
);
611 s
->ats_enabled
= !!(control
& AMDVI_MMIO_CONTROL_HTTUNEN
);
612 s
->evtlog_enabled
= s
->enabled
&& !!(control
&
613 AMDVI_MMIO_CONTROL_EVENTLOGEN
);
615 s
->evtlog_intr
= !!(control
& AMDVI_MMIO_CONTROL_EVENTINTEN
);
616 s
->completion_wait_intr
= !!(control
& AMDVI_MMIO_CONTROL_COMWAITINTEN
);
617 s
->cmdbuf_enabled
= s
->enabled
&& !!(control
&
618 AMDVI_MMIO_CONTROL_CMDBUFLEN
);
619 s
->ga_enabled
= !!(control
& AMDVI_MMIO_CONTROL_GAEN
);
621 /* update the flags depending on the control register */
622 if (s
->cmdbuf_enabled
) {
623 amdvi_assign_orq(s
, AMDVI_MMIO_STATUS
, AMDVI_MMIO_STATUS_CMDBUF_RUN
);
625 amdvi_assign_andq(s
, AMDVI_MMIO_STATUS
, ~AMDVI_MMIO_STATUS_CMDBUF_RUN
);
627 if (s
->evtlog_enabled
) {
628 amdvi_assign_orq(s
, AMDVI_MMIO_STATUS
, AMDVI_MMIO_STATUS_EVT_RUN
);
630 amdvi_assign_andq(s
, AMDVI_MMIO_STATUS
, ~AMDVI_MMIO_STATUS_EVT_RUN
);
633 trace_amdvi_control_status(control
);
637 static inline void amdvi_handle_devtab_write(AMDVIState
*s
)
640 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_DEVICE_TABLE
);
641 s
->devtab
= (val
& AMDVI_MMIO_DEVTAB_BASE_MASK
);
643 /* set device table length */
644 s
->devtab_len
= ((val
& AMDVI_MMIO_DEVTAB_SIZE_MASK
) + 1 *
645 (AMDVI_MMIO_DEVTAB_SIZE_UNIT
/
646 AMDVI_MMIO_DEVTAB_ENTRY_SIZE
));
649 static inline void amdvi_handle_cmdhead_write(AMDVIState
*s
)
651 s
->cmdbuf_head
= amdvi_readq(s
, AMDVI_MMIO_COMMAND_HEAD
)
652 & AMDVI_MMIO_CMDBUF_HEAD_MASK
;
656 static inline void amdvi_handle_cmdbase_write(AMDVIState
*s
)
658 s
->cmdbuf
= amdvi_readq(s
, AMDVI_MMIO_COMMAND_BASE
)
659 & AMDVI_MMIO_CMDBUF_BASE_MASK
;
660 s
->cmdbuf_len
= 1UL << (amdvi_readq(s
, AMDVI_MMIO_CMDBUF_SIZE_BYTE
)
661 & AMDVI_MMIO_CMDBUF_SIZE_MASK
);
662 s
->cmdbuf_head
= s
->cmdbuf_tail
= 0;
665 static inline void amdvi_handle_cmdtail_write(AMDVIState
*s
)
667 s
->cmdbuf_tail
= amdvi_readq(s
, AMDVI_MMIO_COMMAND_TAIL
)
668 & AMDVI_MMIO_CMDBUF_TAIL_MASK
;
672 static inline void amdvi_handle_excllim_write(AMDVIState
*s
)
674 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_EXCL_LIMIT
);
675 s
->excl_limit
= (val
& AMDVI_MMIO_EXCL_LIMIT_MASK
) |
676 AMDVI_MMIO_EXCL_LIMIT_LOW
;
679 static inline void amdvi_handle_evtbase_write(AMDVIState
*s
)
681 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_EVENT_BASE
);
682 s
->evtlog
= val
& AMDVI_MMIO_EVTLOG_BASE_MASK
;
683 s
->evtlog_len
= 1UL << (amdvi_readq(s
, AMDVI_MMIO_EVTLOG_SIZE_BYTE
)
684 & AMDVI_MMIO_EVTLOG_SIZE_MASK
);
687 static inline void amdvi_handle_evttail_write(AMDVIState
*s
)
689 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_EVENT_TAIL
);
690 s
->evtlog_tail
= val
& AMDVI_MMIO_EVTLOG_TAIL_MASK
;
693 static inline void amdvi_handle_evthead_write(AMDVIState
*s
)
695 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_EVENT_HEAD
);
696 s
->evtlog_head
= val
& AMDVI_MMIO_EVTLOG_HEAD_MASK
;
699 static inline void amdvi_handle_pprbase_write(AMDVIState
*s
)
701 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_PPR_BASE
);
702 s
->ppr_log
= val
& AMDVI_MMIO_PPRLOG_BASE_MASK
;
703 s
->pprlog_len
= 1UL << (amdvi_readq(s
, AMDVI_MMIO_PPRLOG_SIZE_BYTE
)
704 & AMDVI_MMIO_PPRLOG_SIZE_MASK
);
707 static inline void amdvi_handle_pprhead_write(AMDVIState
*s
)
709 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_PPR_HEAD
);
710 s
->pprlog_head
= val
& AMDVI_MMIO_PPRLOG_HEAD_MASK
;
713 static inline void amdvi_handle_pprtail_write(AMDVIState
*s
)
715 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_PPR_TAIL
);
716 s
->pprlog_tail
= val
& AMDVI_MMIO_PPRLOG_TAIL_MASK
;
719 /* FIXME: something might go wrong if System Software writes in chunks
720 * of one byte but linux writes in chunks of 4 bytes so currently it
721 * works correctly with linux but will definitely be busted if software
722 * reads/writes 8 bytes
724 static void amdvi_mmio_reg_write(AMDVIState
*s
, unsigned size
, uint64_t val
,
728 amdvi_writew(s
, addr
, val
);
729 } else if (size
== 4) {
730 amdvi_writel(s
, addr
, val
);
731 } else if (size
== 8) {
732 amdvi_writeq(s
, addr
, val
);
736 static void amdvi_mmio_write(void *opaque
, hwaddr addr
, uint64_t val
,
739 AMDVIState
*s
= opaque
;
740 unsigned long offset
= addr
& 0x07;
742 if (addr
+ size
> AMDVI_MMIO_SIZE
) {
743 trace_amdvi_mmio_write("error: addr outside region: max ",
744 (uint64_t)AMDVI_MMIO_SIZE
, size
, val
, offset
);
748 amdvi_mmio_trace(addr
, size
);
749 switch (addr
& ~0x07) {
750 case AMDVI_MMIO_CONTROL
:
751 amdvi_mmio_reg_write(s
, size
, val
, addr
);
752 amdvi_handle_control_write(s
);
754 case AMDVI_MMIO_DEVICE_TABLE
:
755 amdvi_mmio_reg_write(s
, size
, val
, addr
);
756 /* set device table address
757 * This also suffers from inability to tell whether software
760 if (offset
|| (size
== 8)) {
761 amdvi_handle_devtab_write(s
);
764 case AMDVI_MMIO_COMMAND_HEAD
:
765 amdvi_mmio_reg_write(s
, size
, val
, addr
);
766 amdvi_handle_cmdhead_write(s
);
768 case AMDVI_MMIO_COMMAND_BASE
:
769 amdvi_mmio_reg_write(s
, size
, val
, addr
);
770 /* FIXME - make sure System Software has finished writing in case
771 * it writes in chucks less than 8 bytes in a robust way.As for
772 * now, this hacks works for the linux driver
774 if (offset
|| (size
== 8)) {
775 amdvi_handle_cmdbase_write(s
);
778 case AMDVI_MMIO_COMMAND_TAIL
:
779 amdvi_mmio_reg_write(s
, size
, val
, addr
);
780 amdvi_handle_cmdtail_write(s
);
782 case AMDVI_MMIO_EVENT_BASE
:
783 amdvi_mmio_reg_write(s
, size
, val
, addr
);
784 amdvi_handle_evtbase_write(s
);
786 case AMDVI_MMIO_EVENT_HEAD
:
787 amdvi_mmio_reg_write(s
, size
, val
, addr
);
788 amdvi_handle_evthead_write(s
);
790 case AMDVI_MMIO_EVENT_TAIL
:
791 amdvi_mmio_reg_write(s
, size
, val
, addr
);
792 amdvi_handle_evttail_write(s
);
794 case AMDVI_MMIO_EXCL_LIMIT
:
795 amdvi_mmio_reg_write(s
, size
, val
, addr
);
796 amdvi_handle_excllim_write(s
);
798 /* PPR log base - unused for now */
799 case AMDVI_MMIO_PPR_BASE
:
800 amdvi_mmio_reg_write(s
, size
, val
, addr
);
801 amdvi_handle_pprbase_write(s
);
803 /* PPR log head - also unused for now */
804 case AMDVI_MMIO_PPR_HEAD
:
805 amdvi_mmio_reg_write(s
, size
, val
, addr
);
806 amdvi_handle_pprhead_write(s
);
808 /* PPR log tail - unused for now */
809 case AMDVI_MMIO_PPR_TAIL
:
810 amdvi_mmio_reg_write(s
, size
, val
, addr
);
811 amdvi_handle_pprtail_write(s
);
816 static inline uint64_t amdvi_get_perms(uint64_t entry
)
818 return (entry
& (AMDVI_DEV_PERM_READ
| AMDVI_DEV_PERM_WRITE
)) >>
819 AMDVI_DEV_PERM_SHIFT
;
822 /* validate that reserved bits are honoured */
823 static bool amdvi_validate_dte(AMDVIState
*s
, uint16_t devid
,
826 if ((dte
[0] & AMDVI_DTE_LOWER_QUAD_RESERVED
)
827 || (dte
[1] & AMDVI_DTE_MIDDLE_QUAD_RESERVED
)
828 || (dte
[2] & AMDVI_DTE_UPPER_QUAD_RESERVED
) || dte
[3]) {
829 amdvi_log_illegaldevtab_error(s
, devid
,
831 devid
* AMDVI_DEVTAB_ENTRY_SIZE
, 0);
838 /* get a device table entry given the devid */
839 static bool amdvi_get_dte(AMDVIState
*s
, int devid
, uint64_t *entry
)
841 uint32_t offset
= devid
* AMDVI_DEVTAB_ENTRY_SIZE
;
843 if (dma_memory_read(&address_space_memory
, s
->devtab
+ offset
, entry
,
844 AMDVI_DEVTAB_ENTRY_SIZE
, MEMTXATTRS_UNSPECIFIED
)) {
845 trace_amdvi_dte_get_fail(s
->devtab
, offset
);
846 /* log error accessing dte */
847 amdvi_log_devtab_error(s
, devid
, s
->devtab
+ offset
, 0);
851 *entry
= le64_to_cpu(*entry
);
852 if (!amdvi_validate_dte(s
, devid
, entry
)) {
853 trace_amdvi_invalid_dte(entry
[0]);
860 /* get pte translation mode */
861 static inline uint8_t get_pte_translation_mode(uint64_t pte
)
863 return (pte
>> AMDVI_DEV_MODE_RSHIFT
) & AMDVI_DEV_MODE_MASK
;
866 static inline uint64_t pte_override_page_mask(uint64_t pte
)
868 uint8_t page_mask
= 13;
869 uint64_t addr
= (pte
& AMDVI_DEV_PT_ROOT_MASK
) >> 12;
870 /* find the first zero bit */
876 return ~((1ULL << page_mask
) - 1);
879 static inline uint64_t pte_get_page_mask(uint64_t oldlevel
)
881 return ~((1UL << ((oldlevel
* 9) + 3)) - 1);
884 static inline uint64_t amdvi_get_pte_entry(AMDVIState
*s
, uint64_t pte_addr
,
889 if (dma_memory_read(&address_space_memory
, pte_addr
,
890 &pte
, sizeof(pte
), MEMTXATTRS_UNSPECIFIED
)) {
891 trace_amdvi_get_pte_hwerror(pte_addr
);
892 amdvi_log_pagetab_error(s
, devid
, pte_addr
, 0);
897 pte
= le64_to_cpu(pte
);
901 static void amdvi_page_walk(AMDVIAddressSpace
*as
, uint64_t *dte
,
902 IOMMUTLBEntry
*ret
, unsigned perms
,
905 unsigned level
, present
, pte_perms
, oldlevel
;
906 uint64_t pte
= dte
[0], pte_addr
, page_mask
;
908 /* make sure the DTE has TV = 1 */
909 if (pte
& AMDVI_DEV_TRANSLATION_VALID
) {
910 level
= get_pte_translation_mode(pte
);
912 trace_amdvi_mode_invalid(level
, addr
);
919 /* we are at the leaf page table or page table encodes a huge page */
921 pte_perms
= amdvi_get_perms(pte
);
923 if (!present
|| perms
!= (perms
& pte_perms
)) {
924 amdvi_page_fault(as
->iommu_state
, as
->devfn
, addr
, perms
);
925 trace_amdvi_page_fault(addr
);
929 /* go to the next lower level */
930 pte_addr
= pte
& AMDVI_DEV_PT_ROOT_MASK
;
931 /* add offset and load pte */
932 pte_addr
+= ((addr
>> (3 + 9 * level
)) & 0x1FF) << 3;
933 pte
= amdvi_get_pte_entry(as
->iommu_state
, pte_addr
, as
->devfn
);
938 level
= get_pte_translation_mode(pte
);
939 } while (level
> 0 && level
< 7);
942 page_mask
= pte_override_page_mask(pte
);
944 page_mask
= pte_get_page_mask(oldlevel
);
947 /* get access permissions from pte */
948 ret
->iova
= addr
& page_mask
;
949 ret
->translated_addr
= (pte
& AMDVI_DEV_PT_ROOT_MASK
) & page_mask
;
950 ret
->addr_mask
= ~page_mask
;
951 ret
->perm
= amdvi_get_perms(pte
);
955 ret
->iova
= addr
& AMDVI_PAGE_MASK_4K
;
956 ret
->translated_addr
= addr
& AMDVI_PAGE_MASK_4K
;
957 ret
->addr_mask
= ~AMDVI_PAGE_MASK_4K
;
958 ret
->perm
= amdvi_get_perms(pte
);
961 static void amdvi_do_translate(AMDVIAddressSpace
*as
, hwaddr addr
,
962 bool is_write
, IOMMUTLBEntry
*ret
)
964 AMDVIState
*s
= as
->iommu_state
;
965 uint16_t devid
= PCI_BUILD_BDF(as
->bus_num
, as
->devfn
);
966 AMDVIIOTLBEntry
*iotlb_entry
= amdvi_iotlb_lookup(s
, addr
, devid
);
970 trace_amdvi_iotlb_hit(PCI_BUS_NUM(devid
), PCI_SLOT(devid
),
971 PCI_FUNC(devid
), addr
, iotlb_entry
->translated_addr
);
972 ret
->iova
= addr
& ~iotlb_entry
->page_mask
;
973 ret
->translated_addr
= iotlb_entry
->translated_addr
;
974 ret
->addr_mask
= iotlb_entry
->page_mask
;
975 ret
->perm
= iotlb_entry
->perms
;
979 if (!amdvi_get_dte(s
, devid
, entry
)) {
983 /* devices with V = 0 are not translated */
984 if (!(entry
[0] & AMDVI_DEV_VALID
)) {
988 amdvi_page_walk(as
, entry
, ret
,
989 is_write
? AMDVI_PERM_WRITE
: AMDVI_PERM_READ
, addr
);
991 amdvi_update_iotlb(s
, devid
, addr
, *ret
,
992 entry
[1] & AMDVI_DEV_DOMID_ID_MASK
);
996 ret
->iova
= addr
& AMDVI_PAGE_MASK_4K
;
997 ret
->translated_addr
= addr
& AMDVI_PAGE_MASK_4K
;
998 ret
->addr_mask
= ~AMDVI_PAGE_MASK_4K
;
999 ret
->perm
= IOMMU_RW
;
1002 static inline bool amdvi_is_interrupt_addr(hwaddr addr
)
1004 return addr
>= AMDVI_INT_ADDR_FIRST
&& addr
<= AMDVI_INT_ADDR_LAST
;
1007 static IOMMUTLBEntry
amdvi_translate(IOMMUMemoryRegion
*iommu
, hwaddr addr
,
1008 IOMMUAccessFlags flag
, int iommu_idx
)
1010 AMDVIAddressSpace
*as
= container_of(iommu
, AMDVIAddressSpace
, iommu
);
1011 AMDVIState
*s
= as
->iommu_state
;
1012 IOMMUTLBEntry ret
= {
1013 .target_as
= &address_space_memory
,
1015 .translated_addr
= 0,
1016 .addr_mask
= ~(hwaddr
)0,
1021 /* AMDVI disabled - corresponds to iommu=off not
1022 * failure to provide any parameter
1024 ret
.iova
= addr
& AMDVI_PAGE_MASK_4K
;
1025 ret
.translated_addr
= addr
& AMDVI_PAGE_MASK_4K
;
1026 ret
.addr_mask
= ~AMDVI_PAGE_MASK_4K
;
1027 ret
.perm
= IOMMU_RW
;
1029 } else if (amdvi_is_interrupt_addr(addr
)) {
1030 ret
.iova
= addr
& AMDVI_PAGE_MASK_4K
;
1031 ret
.translated_addr
= addr
& AMDVI_PAGE_MASK_4K
;
1032 ret
.addr_mask
= ~AMDVI_PAGE_MASK_4K
;
1033 ret
.perm
= IOMMU_WO
;
1037 amdvi_do_translate(as
, addr
, flag
& IOMMU_WO
, &ret
);
1038 trace_amdvi_translation_result(as
->bus_num
, PCI_SLOT(as
->devfn
),
1039 PCI_FUNC(as
->devfn
), addr
, ret
.translated_addr
);
1043 static int amdvi_get_irte(AMDVIState
*s
, MSIMessage
*origin
, uint64_t *dte
,
1044 union irte
*irte
, uint16_t devid
)
1046 uint64_t irte_root
, offset
;
1048 irte_root
= dte
[2] & AMDVI_IR_PHYS_ADDR_MASK
;
1049 offset
= (origin
->data
& AMDVI_IRTE_OFFSET
) << 2;
1051 trace_amdvi_ir_irte(irte_root
, offset
);
1053 if (dma_memory_read(&address_space_memory
, irte_root
+ offset
,
1054 irte
, sizeof(*irte
), MEMTXATTRS_UNSPECIFIED
)) {
1055 trace_amdvi_ir_err("failed to get irte");
1056 return -AMDVI_IR_GET_IRTE
;
1059 trace_amdvi_ir_irte_val(irte
->val
);
1064 static int amdvi_int_remap_legacy(AMDVIState
*iommu
,
1066 MSIMessage
*translated
,
1074 /* get interrupt remapping table */
1075 ret
= amdvi_get_irte(iommu
, origin
, dte
, &irte
, sid
);
1080 if (!irte
.fields
.valid
) {
1081 trace_amdvi_ir_target_abort("RemapEn is disabled");
1082 return -AMDVI_IR_TARGET_ABORT
;
1085 if (irte
.fields
.guest_mode
) {
1086 error_report_once("guest mode is not zero");
1087 return -AMDVI_IR_ERR
;
1090 if (irte
.fields
.int_type
> AMDVI_IOAPIC_INT_TYPE_ARBITRATED
) {
1091 error_report_once("reserved int_type");
1092 return -AMDVI_IR_ERR
;
1095 irq
->delivery_mode
= irte
.fields
.int_type
;
1096 irq
->vector
= irte
.fields
.vector
;
1097 irq
->dest_mode
= irte
.fields
.dm
;
1098 irq
->redir_hint
= irte
.fields
.rq_eoi
;
1099 irq
->dest
= irte
.fields
.destination
;
1104 static int amdvi_get_irte_ga(AMDVIState
*s
, MSIMessage
*origin
, uint64_t *dte
,
1105 struct irte_ga
*irte
, uint16_t devid
)
1107 uint64_t irte_root
, offset
;
1109 irte_root
= dte
[2] & AMDVI_IR_PHYS_ADDR_MASK
;
1110 offset
= (origin
->data
& AMDVI_IRTE_OFFSET
) << 4;
1111 trace_amdvi_ir_irte(irte_root
, offset
);
1113 if (dma_memory_read(&address_space_memory
, irte_root
+ offset
,
1114 irte
, sizeof(*irte
), MEMTXATTRS_UNSPECIFIED
)) {
1115 trace_amdvi_ir_err("failed to get irte_ga");
1116 return -AMDVI_IR_GET_IRTE
;
1119 trace_amdvi_ir_irte_ga_val(irte
->hi
.val
, irte
->lo
.val
);
1123 static int amdvi_int_remap_ga(AMDVIState
*iommu
,
1125 MSIMessage
*translated
,
1131 struct irte_ga irte
;
1133 /* get interrupt remapping table */
1134 ret
= amdvi_get_irte_ga(iommu
, origin
, dte
, &irte
, sid
);
1139 if (!irte
.lo
.fields_remap
.valid
) {
1140 trace_amdvi_ir_target_abort("RemapEn is disabled");
1141 return -AMDVI_IR_TARGET_ABORT
;
1144 if (irte
.lo
.fields_remap
.guest_mode
) {
1145 error_report_once("guest mode is not zero");
1146 return -AMDVI_IR_ERR
;
1149 if (irte
.lo
.fields_remap
.int_type
> AMDVI_IOAPIC_INT_TYPE_ARBITRATED
) {
1150 error_report_once("reserved int_type is set");
1151 return -AMDVI_IR_ERR
;
1154 irq
->delivery_mode
= irte
.lo
.fields_remap
.int_type
;
1155 irq
->vector
= irte
.hi
.fields
.vector
;
1156 irq
->dest_mode
= irte
.lo
.fields_remap
.dm
;
1157 irq
->redir_hint
= irte
.lo
.fields_remap
.rq_eoi
;
1158 irq
->dest
= irte
.lo
.fields_remap
.destination
;
1163 static int __amdvi_int_remap_msi(AMDVIState
*iommu
,
1165 MSIMessage
*translated
,
1173 int_ctl
= (dte
[2] >> AMDVI_IR_INTCTL_SHIFT
) & 3;
1174 trace_amdvi_ir_intctl(int_ctl
);
1177 case AMDVI_IR_INTCTL_PASS
:
1178 memcpy(translated
, origin
, sizeof(*origin
));
1180 case AMDVI_IR_INTCTL_REMAP
:
1182 case AMDVI_IR_INTCTL_ABORT
:
1183 trace_amdvi_ir_target_abort("int_ctl abort");
1184 return -AMDVI_IR_TARGET_ABORT
;
1186 trace_amdvi_ir_err("int_ctl reserved");
1187 return -AMDVI_IR_ERR
;
1190 if (iommu
->ga_enabled
) {
1191 ret
= amdvi_int_remap_ga(iommu
, origin
, translated
, dte
, irq
, sid
);
1193 ret
= amdvi_int_remap_legacy(iommu
, origin
, translated
, dte
, irq
, sid
);
1199 /* Interrupt remapping for MSI/MSI-X entry */
1200 static int amdvi_int_remap_msi(AMDVIState
*iommu
,
1202 MSIMessage
*translated
,
1207 uint64_t dte
[4] = { 0 };
1208 X86IOMMUIrq irq
= { 0 };
1209 uint8_t dest_mode
, delivery_mode
;
1211 assert(origin
&& translated
);
1214 * When IOMMU is enabled, interrupt remap request will come either from
1215 * IO-APIC or PCI device. If interrupt is from PCI device then it will
1216 * have a valid requester id but if the interrupt is from IO-APIC
1217 * then requester id will be invalid.
1219 if (sid
== X86_IOMMU_SID_INVALID
) {
1220 sid
= AMDVI_IOAPIC_SB_DEVID
;
1223 trace_amdvi_ir_remap_msi_req(origin
->address
, origin
->data
, sid
);
1225 /* check if device table entry is set before we go further. */
1226 if (!iommu
|| !iommu
->devtab_len
) {
1227 memcpy(translated
, origin
, sizeof(*origin
));
1231 if (!amdvi_get_dte(iommu
, sid
, dte
)) {
1232 return -AMDVI_IR_ERR
;
1235 /* Check if IR is enabled in DTE */
1236 if (!(dte
[2] & AMDVI_IR_REMAP_ENABLE
)) {
1237 memcpy(translated
, origin
, sizeof(*origin
));
1241 /* validate that we are configure with intremap=on */
1242 if (!x86_iommu_ir_supported(X86_IOMMU_DEVICE(iommu
))) {
1243 trace_amdvi_err("Interrupt remapping is enabled in the guest but "
1244 "not in the host. Use intremap=on to enable interrupt "
1245 "remapping in amd-iommu.");
1246 return -AMDVI_IR_ERR
;
1249 if (origin
->address
< AMDVI_INT_ADDR_FIRST
||
1250 origin
->address
+ sizeof(origin
->data
) > AMDVI_INT_ADDR_LAST
+ 1) {
1251 trace_amdvi_err("MSI is not from IOAPIC.");
1252 return -AMDVI_IR_ERR
;
1256 * The MSI data register [10:8] are used to get the upstream interrupt type.
1258 * See MSI/MSI-X format:
1259 * https://pdfs.semanticscholar.org/presentation/9420/c279e942eca568157711ef5c92b800c40a79.pdf
1262 delivery_mode
= (origin
->data
>> MSI_DATA_DELIVERY_MODE_SHIFT
) & 7;
1264 switch (delivery_mode
) {
1265 case AMDVI_IOAPIC_INT_TYPE_FIXED
:
1266 case AMDVI_IOAPIC_INT_TYPE_ARBITRATED
:
1267 trace_amdvi_ir_delivery_mode("fixed/arbitrated");
1268 ret
= __amdvi_int_remap_msi(iommu
, origin
, translated
, dte
, &irq
, sid
);
1272 /* Translate IRQ to MSI messages */
1273 x86_iommu_irq_to_msi_message(&irq
, translated
);
1277 case AMDVI_IOAPIC_INT_TYPE_SMI
:
1278 error_report("SMI is not supported!");
1279 ret
= -AMDVI_IR_ERR
;
1281 case AMDVI_IOAPIC_INT_TYPE_NMI
:
1282 pass
= dte
[3] & AMDVI_DEV_NMI_PASS_MASK
;
1283 trace_amdvi_ir_delivery_mode("nmi");
1285 case AMDVI_IOAPIC_INT_TYPE_INIT
:
1286 pass
= dte
[3] & AMDVI_DEV_INT_PASS_MASK
;
1287 trace_amdvi_ir_delivery_mode("init");
1289 case AMDVI_IOAPIC_INT_TYPE_EINT
:
1290 pass
= dte
[3] & AMDVI_DEV_EINT_PASS_MASK
;
1291 trace_amdvi_ir_delivery_mode("eint");
1294 trace_amdvi_ir_delivery_mode("unsupported delivery_mode");
1295 ret
= -AMDVI_IR_ERR
;
1304 * The MSI address register bit[2] is used to get the destination
1305 * mode. The dest_mode 1 is valid for fixed and arbitrated interrupts
1308 dest_mode
= (origin
->address
>> MSI_ADDR_DEST_MODE_SHIFT
) & 1;
1310 trace_amdvi_ir_err("invalid dest_mode");
1311 ret
= -AMDVI_IR_ERR
;
1316 memcpy(translated
, origin
, sizeof(*origin
));
1318 trace_amdvi_ir_err("passthrough is not enabled");
1319 ret
= -AMDVI_IR_ERR
;
1324 trace_amdvi_ir_remap_msi(origin
->address
, origin
->data
,
1325 translated
->address
, translated
->data
);
1332 static int amdvi_int_remap(X86IOMMUState
*iommu
,
1334 MSIMessage
*translated
,
1337 return amdvi_int_remap_msi(AMD_IOMMU_DEVICE(iommu
), origin
,
1341 static MemTxResult
amdvi_mem_ir_write(void *opaque
, hwaddr addr
,
1342 uint64_t value
, unsigned size
,
1346 MSIMessage from
= { 0, 0 }, to
= { 0, 0 };
1347 uint16_t sid
= AMDVI_IOAPIC_SB_DEVID
;
1349 from
.address
= (uint64_t) addr
+ AMDVI_INT_ADDR_FIRST
;
1350 from
.data
= (uint32_t) value
;
1352 trace_amdvi_mem_ir_write_req(addr
, value
, size
);
1354 if (!attrs
.unspecified
) {
1355 /* We have explicit Source ID */
1356 sid
= attrs
.requester_id
;
1359 ret
= amdvi_int_remap_msi(opaque
, &from
, &to
, sid
);
1361 /* TODO: log the event using IOMMU log event interface */
1362 error_report_once("failed to remap interrupt from devid 0x%x", sid
);
1366 apic_get_class(NULL
)->send_msi(&to
);
1368 trace_amdvi_mem_ir_write(to
.address
, to
.data
);
1372 static MemTxResult
amdvi_mem_ir_read(void *opaque
, hwaddr addr
,
1373 uint64_t *data
, unsigned size
,
1379 static const MemoryRegionOps amdvi_ir_ops
= {
1380 .read_with_attrs
= amdvi_mem_ir_read
,
1381 .write_with_attrs
= amdvi_mem_ir_write
,
1382 .endianness
= DEVICE_LITTLE_ENDIAN
,
1384 .min_access_size
= 4,
1385 .max_access_size
= 4,
1388 .min_access_size
= 4,
1389 .max_access_size
= 4,
1393 static AddressSpace
*amdvi_host_dma_iommu(PCIBus
*bus
, void *opaque
, int devfn
)
1396 AMDVIState
*s
= opaque
;
1397 AMDVIAddressSpace
**iommu_as
, *amdvi_dev_as
;
1398 int bus_num
= pci_bus_num(bus
);
1400 iommu_as
= s
->address_spaces
[bus_num
];
1402 /* allocate memory during the first run */
1404 iommu_as
= g_new0(AMDVIAddressSpace
*, PCI_DEVFN_MAX
);
1405 s
->address_spaces
[bus_num
] = iommu_as
;
1408 /* set up AMD-Vi region */
1409 if (!iommu_as
[devfn
]) {
1410 snprintf(name
, sizeof(name
), "amd_iommu_devfn_%d", devfn
);
1412 iommu_as
[devfn
] = g_new0(AMDVIAddressSpace
, 1);
1413 iommu_as
[devfn
]->bus_num
= (uint8_t)bus_num
;
1414 iommu_as
[devfn
]->devfn
= (uint8_t)devfn
;
1415 iommu_as
[devfn
]->iommu_state
= s
;
1417 amdvi_dev_as
= iommu_as
[devfn
];
1420 * Memory region relationships looks like (Address range shows
1421 * only lower 32 bits to make it short in length...):
1423 * |-----------------+-------------------+----------|
1424 * | Name | Address range | Priority |
1425 * |-----------------+-------------------+----------+
1426 * | amdvi_root | 00000000-ffffffff | 0 |
1427 * | amdvi_iommu | 00000000-ffffffff | 1 |
1428 * | amdvi_iommu_ir | fee00000-feefffff | 64 |
1429 * |-----------------+-------------------+----------|
1431 memory_region_init_iommu(&amdvi_dev_as
->iommu
,
1432 sizeof(amdvi_dev_as
->iommu
),
1433 TYPE_AMD_IOMMU_MEMORY_REGION
,
1435 "amd_iommu", UINT64_MAX
);
1436 memory_region_init(&amdvi_dev_as
->root
, OBJECT(s
),
1437 "amdvi_root", UINT64_MAX
);
1438 address_space_init(&amdvi_dev_as
->as
, &amdvi_dev_as
->root
, name
);
1439 memory_region_init_io(&amdvi_dev_as
->iommu_ir
, OBJECT(s
),
1440 &amdvi_ir_ops
, s
, "amd_iommu_ir",
1441 AMDVI_INT_ADDR_SIZE
);
1442 memory_region_add_subregion_overlap(&amdvi_dev_as
->root
,
1443 AMDVI_INT_ADDR_FIRST
,
1444 &amdvi_dev_as
->iommu_ir
,
1446 memory_region_add_subregion_overlap(&amdvi_dev_as
->root
, 0,
1447 MEMORY_REGION(&amdvi_dev_as
->iommu
),
1450 return &iommu_as
[devfn
]->as
;
1453 static const PCIIOMMUOps amdvi_iommu_ops
= {
1454 .get_address_space
= amdvi_host_dma_iommu
,
1457 static const MemoryRegionOps mmio_mem_ops
= {
1458 .read
= amdvi_mmio_read
,
1459 .write
= amdvi_mmio_write
,
1460 .endianness
= DEVICE_LITTLE_ENDIAN
,
1462 .min_access_size
= 1,
1463 .max_access_size
= 8,
1467 .min_access_size
= 1,
1468 .max_access_size
= 8,
1472 static int amdvi_iommu_notify_flag_changed(IOMMUMemoryRegion
*iommu
,
1473 IOMMUNotifierFlag old
,
1474 IOMMUNotifierFlag
new,
1477 AMDVIAddressSpace
*as
= container_of(iommu
, AMDVIAddressSpace
, iommu
);
1479 if (new & IOMMU_NOTIFIER_MAP
) {
1481 "device %02x.%02x.%x requires iommu notifier which is not "
1482 "currently supported", as
->bus_num
, PCI_SLOT(as
->devfn
),
1483 PCI_FUNC(as
->devfn
));
1489 static void amdvi_init(AMDVIState
*s
)
1491 amdvi_iotlb_reset(s
);
1499 s
->excl_enabled
= false;
1500 s
->excl_allow
= false;
1501 s
->mmio_enabled
= false;
1503 s
->ats_enabled
= false;
1504 s
->cmdbuf_enabled
= false;
1507 memset(s
->mmior
, 0, AMDVI_MMIO_SIZE
);
1508 amdvi_set_quad(s
, AMDVI_MMIO_EXT_FEATURES
, AMDVI_EXT_FEATURES
,
1509 0xffffffffffffffef, 0);
1510 amdvi_set_quad(s
, AMDVI_MMIO_STATUS
, 0, 0x98, 0x67);
1513 static void amdvi_pci_realize(PCIDevice
*pdev
, Error
**errp
)
1515 AMDVIPCIState
*s
= AMD_IOMMU_PCI(pdev
);
1518 ret
= pci_add_capability(pdev
, AMDVI_CAPAB_ID_SEC
, 0,
1519 AMDVI_CAPAB_SIZE
, errp
);
1523 s
->capab_offset
= ret
;
1525 ret
= pci_add_capability(pdev
, PCI_CAP_ID_MSI
, 0,
1526 AMDVI_CAPAB_REG_SIZE
, errp
);
1530 ret
= pci_add_capability(pdev
, PCI_CAP_ID_HT
, 0,
1531 AMDVI_CAPAB_REG_SIZE
, errp
);
1536 if (msi_init(pdev
, 0, 1, true, false, errp
) < 0) {
1540 /* reset device ident */
1541 pci_config_set_prog_interface(pdev
->config
, 0);
1543 /* reset AMDVI specific capabilities, all r/o */
1544 pci_set_long(pdev
->config
+ s
->capab_offset
, AMDVI_CAPAB_FEATURES
);
1545 pci_set_long(pdev
->config
+ s
->capab_offset
+ AMDVI_CAPAB_BAR_LOW
,
1546 AMDVI_BASE_ADDR
& ~(0xffff0000));
1547 pci_set_long(pdev
->config
+ s
->capab_offset
+ AMDVI_CAPAB_BAR_HIGH
,
1548 (AMDVI_BASE_ADDR
& ~(0xffff)) >> 16);
1549 pci_set_long(pdev
->config
+ s
->capab_offset
+ AMDVI_CAPAB_RANGE
,
1551 pci_set_long(pdev
->config
+ s
->capab_offset
+ AMDVI_CAPAB_MISC
, 0);
1552 pci_set_long(pdev
->config
+ s
->capab_offset
+ AMDVI_CAPAB_MISC
,
1553 AMDVI_MAX_PH_ADDR
| AMDVI_MAX_GVA_ADDR
| AMDVI_MAX_VA_ADDR
);
1556 static void amdvi_sysbus_reset(DeviceState
*dev
)
1558 AMDVIState
*s
= AMD_IOMMU_DEVICE(dev
);
1560 msi_reset(&s
->pci
.dev
);
1564 static void amdvi_sysbus_realize(DeviceState
*dev
, Error
**errp
)
1566 AMDVIState
*s
= AMD_IOMMU_DEVICE(dev
);
1567 MachineState
*ms
= MACHINE(qdev_get_machine());
1568 PCMachineState
*pcms
= PC_MACHINE(ms
);
1569 X86MachineState
*x86ms
= X86_MACHINE(ms
);
1570 PCIBus
*bus
= pcms
->bus
;
1572 s
->iotlb
= g_hash_table_new_full(amdvi_uint64_hash
,
1573 amdvi_uint64_equal
, g_free
, g_free
);
1575 /* This device should take care of IOMMU PCI properties */
1576 if (!qdev_realize(DEVICE(&s
->pci
), &bus
->qbus
, errp
)) {
1580 /* Pseudo address space under root PCI bus. */
1581 x86ms
->ioapic_as
= amdvi_host_dma_iommu(bus
, s
, AMDVI_IOAPIC_SB_DEVID
);
1584 memory_region_init_io(&s
->mmio
, OBJECT(s
), &mmio_mem_ops
, s
, "amdvi-mmio",
1586 memory_region_add_subregion(get_system_memory(), AMDVI_BASE_ADDR
,
1588 pci_setup_iommu(bus
, &amdvi_iommu_ops
, s
);
1592 static const VMStateDescription vmstate_amdvi_sysbus
= {
1593 .name
= "amd-iommu",
1597 static void amdvi_sysbus_instance_init(Object
*klass
)
1599 AMDVIState
*s
= AMD_IOMMU_DEVICE(klass
);
1601 object_initialize(&s
->pci
, sizeof(s
->pci
), TYPE_AMD_IOMMU_PCI
);
1604 static void amdvi_sysbus_class_init(ObjectClass
*klass
, void *data
)
1606 DeviceClass
*dc
= DEVICE_CLASS(klass
);
1607 X86IOMMUClass
*dc_class
= X86_IOMMU_DEVICE_CLASS(klass
);
1609 dc
->reset
= amdvi_sysbus_reset
;
1610 dc
->vmsd
= &vmstate_amdvi_sysbus
;
1611 dc
->hotpluggable
= false;
1612 dc_class
->realize
= amdvi_sysbus_realize
;
1613 dc_class
->int_remap
= amdvi_int_remap
;
1614 /* Supported by the pc-q35-* machine types */
1615 dc
->user_creatable
= true;
1616 set_bit(DEVICE_CATEGORY_MISC
, dc
->categories
);
1617 dc
->desc
= "AMD IOMMU (AMD-Vi) DMA Remapping device";
1620 static const TypeInfo amdvi_sysbus
= {
1621 .name
= TYPE_AMD_IOMMU_DEVICE
,
1622 .parent
= TYPE_X86_IOMMU_DEVICE
,
1623 .instance_size
= sizeof(AMDVIState
),
1624 .instance_init
= amdvi_sysbus_instance_init
,
1625 .class_init
= amdvi_sysbus_class_init
1628 static void amdvi_pci_class_init(ObjectClass
*klass
, void *data
)
1630 DeviceClass
*dc
= DEVICE_CLASS(klass
);
1631 PCIDeviceClass
*k
= PCI_DEVICE_CLASS(klass
);
1633 k
->vendor_id
= PCI_VENDOR_ID_AMD
;
1634 k
->class_id
= 0x0806;
1635 k
->realize
= amdvi_pci_realize
;
1637 set_bit(DEVICE_CATEGORY_MISC
, dc
->categories
);
1638 dc
->desc
= "AMD IOMMU (AMD-Vi) DMA Remapping device";
1641 static const TypeInfo amdvi_pci
= {
1642 .name
= TYPE_AMD_IOMMU_PCI
,
1643 .parent
= TYPE_PCI_DEVICE
,
1644 .instance_size
= sizeof(AMDVIPCIState
),
1645 .class_init
= amdvi_pci_class_init
,
1646 .interfaces
= (InterfaceInfo
[]) {
1647 { INTERFACE_CONVENTIONAL_PCI_DEVICE
},
1652 static void amdvi_iommu_memory_region_class_init(ObjectClass
*klass
, void *data
)
1654 IOMMUMemoryRegionClass
*imrc
= IOMMU_MEMORY_REGION_CLASS(klass
);
1656 imrc
->translate
= amdvi_translate
;
1657 imrc
->notify_flag_changed
= amdvi_iommu_notify_flag_changed
;
1660 static const TypeInfo amdvi_iommu_memory_region_info
= {
1661 .parent
= TYPE_IOMMU_MEMORY_REGION
,
1662 .name
= TYPE_AMD_IOMMU_MEMORY_REGION
,
1663 .class_init
= amdvi_iommu_memory_region_class_init
,
1666 static void amdvi_register_types(void)
1668 type_register_static(&amdvi_pci
);
1669 type_register_static(&amdvi_sysbus
);
1670 type_register_static(&amdvi_iommu_memory_region_info
);
1673 type_init(amdvi_register_types
);