2 * QEMU emulation of AMD IOMMU (AMD-Vi)
4 * Copyright (C) 2011 Eduard - Gabriel Munteanu
5 * Copyright (C) 2015, 2016 David Kiarie Kahurani
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License along
18 * with this program; if not, see <http://www.gnu.org/licenses/>.
20 * Cache implementation inspired by hw/i386/intel_iommu.c
23 #include "qemu/osdep.h"
24 #include "hw/i386/pc.h"
25 #include "hw/pci/msi.h"
26 #include "hw/pci/pci_bus.h"
27 #include "migration/vmstate.h"
28 #include "amd_iommu.h"
29 #include "qapi/error.h"
30 #include "qemu/error-report.h"
31 #include "hw/i386/apic_internal.h"
33 #include "hw/i386/apic-msidef.h"
35 /* used AMD-Vi MMIO registers */
36 const char *amdvi_mmio_low
[] = {
37 "AMDVI_MMIO_DEVTAB_BASE",
38 "AMDVI_MMIO_CMDBUF_BASE",
39 "AMDVI_MMIO_EVTLOG_BASE",
41 "AMDVI_MMIO_EXCL_BASE",
42 "AMDVI_MMIO_EXCL_LIMIT",
43 "AMDVI_MMIO_EXT_FEATURES",
44 "AMDVI_MMIO_PPR_BASE",
47 const char *amdvi_mmio_high
[] = {
48 "AMDVI_MMIO_COMMAND_HEAD",
49 "AMDVI_MMIO_COMMAND_TAIL",
50 "AMDVI_MMIO_EVTLOG_HEAD",
51 "AMDVI_MMIO_EVTLOG_TAIL",
53 "AMDVI_MMIO_PPR_HEAD",
54 "AMDVI_MMIO_PPR_TAIL",
58 struct AMDVIAddressSpace
{
59 uint8_t bus_num
; /* bus number */
60 uint8_t devfn
; /* device function */
61 AMDVIState
*iommu_state
; /* AMDVI - one per machine */
62 MemoryRegion root
; /* AMDVI Root memory map region */
63 IOMMUMemoryRegion iommu
; /* Device's address translation region */
64 MemoryRegion iommu_ir
; /* Device's interrupt remapping region */
65 AddressSpace as
; /* device's corresponding address space */
68 /* AMDVI cache entry */
69 typedef struct AMDVIIOTLBEntry
{
70 uint16_t domid
; /* assigned domain id */
71 uint16_t devid
; /* device owning entry */
72 uint64_t perms
; /* access permissions */
73 uint64_t translated_addr
; /* translated address */
74 uint64_t page_mask
; /* physical page size */
77 /* configure MMIO registers at startup/reset */
78 static void amdvi_set_quad(AMDVIState
*s
, hwaddr addr
, uint64_t val
,
79 uint64_t romask
, uint64_t w1cmask
)
81 stq_le_p(&s
->mmior
[addr
], val
);
82 stq_le_p(&s
->romask
[addr
], romask
);
83 stq_le_p(&s
->w1cmask
[addr
], w1cmask
);
86 static uint16_t amdvi_readw(AMDVIState
*s
, hwaddr addr
)
88 return lduw_le_p(&s
->mmior
[addr
]);
91 static uint32_t amdvi_readl(AMDVIState
*s
, hwaddr addr
)
93 return ldl_le_p(&s
->mmior
[addr
]);
96 static uint64_t amdvi_readq(AMDVIState
*s
, hwaddr addr
)
98 return ldq_le_p(&s
->mmior
[addr
]);
102 static void amdvi_writeq_raw(AMDVIState
*s
, hwaddr addr
, uint64_t val
)
104 stq_le_p(&s
->mmior
[addr
], val
);
108 static void amdvi_writew(AMDVIState
*s
, hwaddr addr
, uint16_t val
)
110 uint16_t romask
= lduw_le_p(&s
->romask
[addr
]);
111 uint16_t w1cmask
= lduw_le_p(&s
->w1cmask
[addr
]);
112 uint16_t oldval
= lduw_le_p(&s
->mmior
[addr
]);
113 stw_le_p(&s
->mmior
[addr
],
114 ((oldval
& romask
) | (val
& ~romask
)) & ~(val
& w1cmask
));
117 static void amdvi_writel(AMDVIState
*s
, hwaddr addr
, uint32_t val
)
119 uint32_t romask
= ldl_le_p(&s
->romask
[addr
]);
120 uint32_t w1cmask
= ldl_le_p(&s
->w1cmask
[addr
]);
121 uint32_t oldval
= ldl_le_p(&s
->mmior
[addr
]);
122 stl_le_p(&s
->mmior
[addr
],
123 ((oldval
& romask
) | (val
& ~romask
)) & ~(val
& w1cmask
));
126 static void amdvi_writeq(AMDVIState
*s
, hwaddr addr
, uint64_t val
)
128 uint64_t romask
= ldq_le_p(&s
->romask
[addr
]);
129 uint64_t w1cmask
= ldq_le_p(&s
->w1cmask
[addr
]);
130 uint32_t oldval
= ldq_le_p(&s
->mmior
[addr
]);
131 stq_le_p(&s
->mmior
[addr
],
132 ((oldval
& romask
) | (val
& ~romask
)) & ~(val
& w1cmask
));
135 /* OR a 64-bit register with a 64-bit value */
136 static bool amdvi_test_mask(AMDVIState
*s
, hwaddr addr
, uint64_t val
)
138 return amdvi_readq(s
, addr
) | val
;
141 /* OR a 64-bit register with a 64-bit value storing result in the register */
142 static void amdvi_assign_orq(AMDVIState
*s
, hwaddr addr
, uint64_t val
)
144 amdvi_writeq_raw(s
, addr
, amdvi_readq(s
, addr
) | val
);
147 /* AND a 64-bit register with a 64-bit value storing result in the register */
148 static void amdvi_assign_andq(AMDVIState
*s
, hwaddr addr
, uint64_t val
)
150 amdvi_writeq_raw(s
, addr
, amdvi_readq(s
, addr
) & val
);
153 static void amdvi_generate_msi_interrupt(AMDVIState
*s
)
157 .requester_id
= pci_requester_id(&s
->pci
.dev
)
160 if (msi_enabled(&s
->pci
.dev
)) {
161 msg
= msi_get_message(&s
->pci
.dev
, 0);
162 address_space_stl_le(&address_space_memory
, msg
.address
, msg
.data
,
167 static void amdvi_log_event(AMDVIState
*s
, uint64_t *evt
)
169 /* event logging not enabled */
170 if (!s
->evtlog_enabled
|| amdvi_test_mask(s
, AMDVI_MMIO_STATUS
,
171 AMDVI_MMIO_STATUS_EVT_OVF
)) {
175 /* event log buffer full */
176 if (s
->evtlog_tail
>= s
->evtlog_len
) {
177 amdvi_assign_orq(s
, AMDVI_MMIO_STATUS
, AMDVI_MMIO_STATUS_EVT_OVF
);
178 /* generate interrupt */
179 amdvi_generate_msi_interrupt(s
);
183 if (dma_memory_write(&address_space_memory
, s
->evtlog
+ s
->evtlog_tail
,
184 evt
, AMDVI_EVENT_LEN
, MEMTXATTRS_UNSPECIFIED
)) {
185 trace_amdvi_evntlog_fail(s
->evtlog
, s
->evtlog_tail
);
188 s
->evtlog_tail
+= AMDVI_EVENT_LEN
;
189 amdvi_assign_orq(s
, AMDVI_MMIO_STATUS
, AMDVI_MMIO_STATUS_COMP_INT
);
190 amdvi_generate_msi_interrupt(s
);
193 static void amdvi_setevent_bits(uint64_t *buffer
, uint64_t value
, int start
,
196 int index
= start
/ 64, bitpos
= start
% 64;
197 uint64_t mask
= MAKE_64BIT_MASK(start
, length
);
198 buffer
[index
] &= ~mask
;
199 buffer
[index
] |= (value
<< bitpos
) & mask
;
202 * AMDVi event structure
204 * 55:63 -> event type + miscellaneous info
205 * 63:127 -> related address
207 static void amdvi_encode_event(uint64_t *evt
, uint16_t devid
, uint64_t addr
,
210 amdvi_setevent_bits(evt
, devid
, 0, 16);
211 amdvi_setevent_bits(evt
, info
, 55, 8);
212 amdvi_setevent_bits(evt
, addr
, 63, 64);
214 /* log an error encountered during a page walk
216 * @addr: virtual address in translation request
218 static void amdvi_page_fault(AMDVIState
*s
, uint16_t devid
,
219 hwaddr addr
, uint16_t info
)
223 info
|= AMDVI_EVENT_IOPF_I
| AMDVI_EVENT_IOPF
;
224 amdvi_encode_event(evt
, devid
, addr
, info
);
225 amdvi_log_event(s
, evt
);
226 pci_word_test_and_set_mask(s
->pci
.dev
.config
+ PCI_STATUS
,
227 PCI_STATUS_SIG_TARGET_ABORT
);
230 * log a master abort accessing device table
231 * @devtab : address of device table entry
232 * @info : error flags
234 static void amdvi_log_devtab_error(AMDVIState
*s
, uint16_t devid
,
235 hwaddr devtab
, uint16_t info
)
239 info
|= AMDVI_EVENT_DEV_TAB_HW_ERROR
;
241 amdvi_encode_event(evt
, devid
, devtab
, info
);
242 amdvi_log_event(s
, evt
);
243 pci_word_test_and_set_mask(s
->pci
.dev
.config
+ PCI_STATUS
,
244 PCI_STATUS_SIG_TARGET_ABORT
);
246 /* log an event trying to access command buffer
247 * @addr : address that couldn't be accessed
249 static void amdvi_log_command_error(AMDVIState
*s
, hwaddr addr
)
251 uint64_t evt
[4], info
= AMDVI_EVENT_COMMAND_HW_ERROR
;
253 amdvi_encode_event(evt
, 0, addr
, info
);
254 amdvi_log_event(s
, evt
);
255 pci_word_test_and_set_mask(s
->pci
.dev
.config
+ PCI_STATUS
,
256 PCI_STATUS_SIG_TARGET_ABORT
);
258 /* log an illegal comand event
259 * @addr : address of illegal command
261 static void amdvi_log_illegalcom_error(AMDVIState
*s
, uint16_t info
,
266 info
|= AMDVI_EVENT_ILLEGAL_COMMAND_ERROR
;
267 amdvi_encode_event(evt
, 0, addr
, info
);
268 amdvi_log_event(s
, evt
);
270 /* log an error accessing device table
272 * @devid : device owning the table entry
273 * @devtab : address of device table entry
274 * @info : error flags
276 static void amdvi_log_illegaldevtab_error(AMDVIState
*s
, uint16_t devid
,
277 hwaddr addr
, uint16_t info
)
281 info
|= AMDVI_EVENT_ILLEGAL_DEVTAB_ENTRY
;
282 amdvi_encode_event(evt
, devid
, addr
, info
);
283 amdvi_log_event(s
, evt
);
285 /* log an error accessing a PTE entry
286 * @addr : address that couldn't be accessed
288 static void amdvi_log_pagetab_error(AMDVIState
*s
, uint16_t devid
,
289 hwaddr addr
, uint16_t info
)
293 info
|= AMDVI_EVENT_PAGE_TAB_HW_ERROR
;
294 amdvi_encode_event(evt
, devid
, addr
, info
);
295 amdvi_log_event(s
, evt
);
296 pci_word_test_and_set_mask(s
->pci
.dev
.config
+ PCI_STATUS
,
297 PCI_STATUS_SIG_TARGET_ABORT
);
300 static gboolean
amdvi_uint64_equal(gconstpointer v1
, gconstpointer v2
)
302 return *((const uint64_t *)v1
) == *((const uint64_t *)v2
);
305 static guint
amdvi_uint64_hash(gconstpointer v
)
307 return (guint
)*(const uint64_t *)v
;
310 static AMDVIIOTLBEntry
*amdvi_iotlb_lookup(AMDVIState
*s
, hwaddr addr
,
313 uint64_t key
= (addr
>> AMDVI_PAGE_SHIFT_4K
) |
314 ((uint64_t)(devid
) << AMDVI_DEVID_SHIFT
);
315 return g_hash_table_lookup(s
->iotlb
, &key
);
318 static void amdvi_iotlb_reset(AMDVIState
*s
)
321 trace_amdvi_iotlb_reset();
322 g_hash_table_remove_all(s
->iotlb
);
325 static gboolean
amdvi_iotlb_remove_by_devid(gpointer key
, gpointer value
,
328 AMDVIIOTLBEntry
*entry
= (AMDVIIOTLBEntry
*)value
;
329 uint16_t devid
= *(uint16_t *)user_data
;
330 return entry
->devid
== devid
;
333 static void amdvi_iotlb_remove_page(AMDVIState
*s
, hwaddr addr
,
336 uint64_t key
= (addr
>> AMDVI_PAGE_SHIFT_4K
) |
337 ((uint64_t)(devid
) << AMDVI_DEVID_SHIFT
);
338 g_hash_table_remove(s
->iotlb
, &key
);
341 static void amdvi_update_iotlb(AMDVIState
*s
, uint16_t devid
,
342 uint64_t gpa
, IOMMUTLBEntry to_cache
,
345 AMDVIIOTLBEntry
*entry
= g_new(AMDVIIOTLBEntry
, 1);
346 uint64_t *key
= g_new(uint64_t, 1);
347 uint64_t gfn
= gpa
>> AMDVI_PAGE_SHIFT_4K
;
349 /* don't cache erroneous translations */
350 if (to_cache
.perm
!= IOMMU_NONE
) {
351 trace_amdvi_cache_update(domid
, PCI_BUS_NUM(devid
), PCI_SLOT(devid
),
352 PCI_FUNC(devid
), gpa
, to_cache
.translated_addr
);
354 if (g_hash_table_size(s
->iotlb
) >= AMDVI_IOTLB_MAX_SIZE
) {
355 amdvi_iotlb_reset(s
);
358 entry
->domid
= domid
;
359 entry
->perms
= to_cache
.perm
;
360 entry
->translated_addr
= to_cache
.translated_addr
;
361 entry
->page_mask
= to_cache
.addr_mask
;
362 *key
= gfn
| ((uint64_t)(devid
) << AMDVI_DEVID_SHIFT
);
363 g_hash_table_replace(s
->iotlb
, key
, entry
);
367 static void amdvi_completion_wait(AMDVIState
*s
, uint64_t *cmd
)
369 /* pad the last 3 bits */
370 hwaddr addr
= cpu_to_le64(extract64(cmd
[0], 3, 49)) << 3;
371 uint64_t data
= cpu_to_le64(cmd
[1]);
373 if (extract64(cmd
[0], 52, 8)) {
374 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
375 s
->cmdbuf
+ s
->cmdbuf_head
);
377 if (extract64(cmd
[0], 0, 1)) {
378 if (dma_memory_write(&address_space_memory
, addr
, &data
,
379 AMDVI_COMPLETION_DATA_SIZE
,
380 MEMTXATTRS_UNSPECIFIED
)) {
381 trace_amdvi_completion_wait_fail(addr
);
384 /* set completion interrupt */
385 if (extract64(cmd
[0], 1, 1)) {
386 amdvi_assign_orq(s
, AMDVI_MMIO_STATUS
, AMDVI_MMIO_STATUS_COMP_INT
);
387 /* generate interrupt */
388 amdvi_generate_msi_interrupt(s
);
390 trace_amdvi_completion_wait(addr
, data
);
393 /* log error without aborting since linux seems to be using reserved bits */
394 static void amdvi_inval_devtab_entry(AMDVIState
*s
, uint64_t *cmd
)
396 uint16_t devid
= cpu_to_le16((uint16_t)extract64(cmd
[0], 0, 16));
398 /* This command should invalidate internal caches of which there isn't */
399 if (extract64(cmd
[0], 16, 44) || cmd
[1]) {
400 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
401 s
->cmdbuf
+ s
->cmdbuf_head
);
403 trace_amdvi_devtab_inval(PCI_BUS_NUM(devid
), PCI_SLOT(devid
),
407 static void amdvi_complete_ppr(AMDVIState
*s
, uint64_t *cmd
)
409 if (extract64(cmd
[0], 16, 16) || extract64(cmd
[0], 52, 8) ||
410 extract64(cmd
[1], 0, 2) || extract64(cmd
[1], 3, 29)
411 || extract64(cmd
[1], 48, 16)) {
412 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
413 s
->cmdbuf
+ s
->cmdbuf_head
);
415 trace_amdvi_ppr_exec();
418 static void amdvi_inval_all(AMDVIState
*s
, uint64_t *cmd
)
420 if (extract64(cmd
[0], 0, 60) || cmd
[1]) {
421 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
422 s
->cmdbuf
+ s
->cmdbuf_head
);
425 amdvi_iotlb_reset(s
);
426 trace_amdvi_all_inval();
429 static gboolean
amdvi_iotlb_remove_by_domid(gpointer key
, gpointer value
,
432 AMDVIIOTLBEntry
*entry
= (AMDVIIOTLBEntry
*)value
;
433 uint16_t domid
= *(uint16_t *)user_data
;
434 return entry
->domid
== domid
;
437 /* we don't have devid - we can't remove pages by address */
438 static void amdvi_inval_pages(AMDVIState
*s
, uint64_t *cmd
)
440 uint16_t domid
= cpu_to_le16((uint16_t)extract64(cmd
[0], 32, 16));
442 if (extract64(cmd
[0], 20, 12) || extract64(cmd
[0], 48, 12) ||
443 extract64(cmd
[1], 3, 9)) {
444 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
445 s
->cmdbuf
+ s
->cmdbuf_head
);
448 g_hash_table_foreach_remove(s
->iotlb
, amdvi_iotlb_remove_by_domid
,
450 trace_amdvi_pages_inval(domid
);
453 static void amdvi_prefetch_pages(AMDVIState
*s
, uint64_t *cmd
)
455 if (extract64(cmd
[0], 16, 8) || extract64(cmd
[0], 52, 8) ||
456 extract64(cmd
[1], 1, 1) || extract64(cmd
[1], 3, 1) ||
457 extract64(cmd
[1], 5, 7)) {
458 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
459 s
->cmdbuf
+ s
->cmdbuf_head
);
462 trace_amdvi_prefetch_pages();
465 static void amdvi_inval_inttable(AMDVIState
*s
, uint64_t *cmd
)
467 if (extract64(cmd
[0], 16, 44) || cmd
[1]) {
468 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
469 s
->cmdbuf
+ s
->cmdbuf_head
);
473 trace_amdvi_intr_inval();
476 /* FIXME: Try to work with the specified size instead of all the pages
477 * when the S bit is on
479 static void iommu_inval_iotlb(AMDVIState
*s
, uint64_t *cmd
)
482 uint16_t devid
= extract64(cmd
[0], 0, 16);
483 if (extract64(cmd
[1], 1, 1) || extract64(cmd
[1], 3, 1) ||
484 extract64(cmd
[1], 6, 6)) {
485 amdvi_log_illegalcom_error(s
, extract64(cmd
[0], 60, 4),
486 s
->cmdbuf
+ s
->cmdbuf_head
);
490 if (extract64(cmd
[1], 0, 1)) {
491 g_hash_table_foreach_remove(s
->iotlb
, amdvi_iotlb_remove_by_devid
,
494 amdvi_iotlb_remove_page(s
, cpu_to_le64(extract64(cmd
[1], 12, 52)) << 12,
495 cpu_to_le16(extract64(cmd
[1], 0, 16)));
497 trace_amdvi_iotlb_inval();
500 /* not honouring reserved bits is regarded as an illegal command */
501 static void amdvi_cmdbuf_exec(AMDVIState
*s
)
505 if (dma_memory_read(&address_space_memory
, s
->cmdbuf
+ s
->cmdbuf_head
,
506 cmd
, AMDVI_COMMAND_SIZE
, MEMTXATTRS_UNSPECIFIED
)) {
507 trace_amdvi_command_read_fail(s
->cmdbuf
, s
->cmdbuf_head
);
508 amdvi_log_command_error(s
, s
->cmdbuf
+ s
->cmdbuf_head
);
512 switch (extract64(cmd
[0], 60, 4)) {
513 case AMDVI_CMD_COMPLETION_WAIT
:
514 amdvi_completion_wait(s
, cmd
);
516 case AMDVI_CMD_INVAL_DEVTAB_ENTRY
:
517 amdvi_inval_devtab_entry(s
, cmd
);
519 case AMDVI_CMD_INVAL_AMDVI_PAGES
:
520 amdvi_inval_pages(s
, cmd
);
522 case AMDVI_CMD_INVAL_IOTLB_PAGES
:
523 iommu_inval_iotlb(s
, cmd
);
525 case AMDVI_CMD_INVAL_INTR_TABLE
:
526 amdvi_inval_inttable(s
, cmd
);
528 case AMDVI_CMD_PREFETCH_AMDVI_PAGES
:
529 amdvi_prefetch_pages(s
, cmd
);
531 case AMDVI_CMD_COMPLETE_PPR_REQUEST
:
532 amdvi_complete_ppr(s
, cmd
);
534 case AMDVI_CMD_INVAL_AMDVI_ALL
:
535 amdvi_inval_all(s
, cmd
);
538 trace_amdvi_unhandled_command(extract64(cmd
[1], 60, 4));
539 /* log illegal command */
540 amdvi_log_illegalcom_error(s
, extract64(cmd
[1], 60, 4),
541 s
->cmdbuf
+ s
->cmdbuf_head
);
545 static void amdvi_cmdbuf_run(AMDVIState
*s
)
547 if (!s
->cmdbuf_enabled
) {
548 trace_amdvi_command_error(amdvi_readq(s
, AMDVI_MMIO_CONTROL
));
552 /* check if there is work to do. */
553 while (s
->cmdbuf_head
!= s
->cmdbuf_tail
) {
554 trace_amdvi_command_exec(s
->cmdbuf_head
, s
->cmdbuf_tail
, s
->cmdbuf
);
555 amdvi_cmdbuf_exec(s
);
556 s
->cmdbuf_head
+= AMDVI_COMMAND_SIZE
;
557 amdvi_writeq_raw(s
, AMDVI_MMIO_COMMAND_HEAD
, s
->cmdbuf_head
);
559 /* wrap head pointer */
560 if (s
->cmdbuf_head
>= s
->cmdbuf_len
* AMDVI_COMMAND_SIZE
) {
566 static void amdvi_mmio_trace(hwaddr addr
, unsigned size
)
568 uint8_t index
= (addr
& ~0x2000) / 8;
570 if ((addr
& 0x2000)) {
572 index
= index
>= AMDVI_MMIO_REGS_HIGH
? AMDVI_MMIO_REGS_HIGH
: index
;
573 trace_amdvi_mmio_read(amdvi_mmio_high
[index
], addr
, size
, addr
& ~0x07);
575 index
= index
>= AMDVI_MMIO_REGS_LOW
? AMDVI_MMIO_REGS_LOW
: index
;
576 trace_amdvi_mmio_read(amdvi_mmio_low
[index
], addr
, size
, addr
& ~0x07);
580 static uint64_t amdvi_mmio_read(void *opaque
, hwaddr addr
, unsigned size
)
582 AMDVIState
*s
= opaque
;
585 if (addr
+ size
> AMDVI_MMIO_SIZE
) {
586 trace_amdvi_mmio_read_invalid(AMDVI_MMIO_SIZE
, addr
, size
);
591 val
= amdvi_readw(s
, addr
);
592 } else if (size
== 4) {
593 val
= amdvi_readl(s
, addr
);
594 } else if (size
== 8) {
595 val
= amdvi_readq(s
, addr
);
597 amdvi_mmio_trace(addr
, size
);
602 static void amdvi_handle_control_write(AMDVIState
*s
)
604 unsigned long control
= amdvi_readq(s
, AMDVI_MMIO_CONTROL
);
605 s
->enabled
= !!(control
& AMDVI_MMIO_CONTROL_AMDVIEN
);
607 s
->ats_enabled
= !!(control
& AMDVI_MMIO_CONTROL_HTTUNEN
);
608 s
->evtlog_enabled
= s
->enabled
&& !!(control
&
609 AMDVI_MMIO_CONTROL_EVENTLOGEN
);
611 s
->evtlog_intr
= !!(control
& AMDVI_MMIO_CONTROL_EVENTINTEN
);
612 s
->completion_wait_intr
= !!(control
& AMDVI_MMIO_CONTROL_COMWAITINTEN
);
613 s
->cmdbuf_enabled
= s
->enabled
&& !!(control
&
614 AMDVI_MMIO_CONTROL_CMDBUFLEN
);
615 s
->ga_enabled
= !!(control
& AMDVI_MMIO_CONTROL_GAEN
);
617 /* update the flags depending on the control register */
618 if (s
->cmdbuf_enabled
) {
619 amdvi_assign_orq(s
, AMDVI_MMIO_STATUS
, AMDVI_MMIO_STATUS_CMDBUF_RUN
);
621 amdvi_assign_andq(s
, AMDVI_MMIO_STATUS
, ~AMDVI_MMIO_STATUS_CMDBUF_RUN
);
623 if (s
->evtlog_enabled
) {
624 amdvi_assign_orq(s
, AMDVI_MMIO_STATUS
, AMDVI_MMIO_STATUS_EVT_RUN
);
626 amdvi_assign_andq(s
, AMDVI_MMIO_STATUS
, ~AMDVI_MMIO_STATUS_EVT_RUN
);
629 trace_amdvi_control_status(control
);
633 static inline void amdvi_handle_devtab_write(AMDVIState
*s
)
636 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_DEVICE_TABLE
);
637 s
->devtab
= (val
& AMDVI_MMIO_DEVTAB_BASE_MASK
);
639 /* set device table length */
640 s
->devtab_len
= ((val
& AMDVI_MMIO_DEVTAB_SIZE_MASK
) + 1 *
641 (AMDVI_MMIO_DEVTAB_SIZE_UNIT
/
642 AMDVI_MMIO_DEVTAB_ENTRY_SIZE
));
645 static inline void amdvi_handle_cmdhead_write(AMDVIState
*s
)
647 s
->cmdbuf_head
= amdvi_readq(s
, AMDVI_MMIO_COMMAND_HEAD
)
648 & AMDVI_MMIO_CMDBUF_HEAD_MASK
;
652 static inline void amdvi_handle_cmdbase_write(AMDVIState
*s
)
654 s
->cmdbuf
= amdvi_readq(s
, AMDVI_MMIO_COMMAND_BASE
)
655 & AMDVI_MMIO_CMDBUF_BASE_MASK
;
656 s
->cmdbuf_len
= 1UL << (amdvi_readq(s
, AMDVI_MMIO_CMDBUF_SIZE_BYTE
)
657 & AMDVI_MMIO_CMDBUF_SIZE_MASK
);
658 s
->cmdbuf_head
= s
->cmdbuf_tail
= 0;
661 static inline void amdvi_handle_cmdtail_write(AMDVIState
*s
)
663 s
->cmdbuf_tail
= amdvi_readq(s
, AMDVI_MMIO_COMMAND_TAIL
)
664 & AMDVI_MMIO_CMDBUF_TAIL_MASK
;
668 static inline void amdvi_handle_excllim_write(AMDVIState
*s
)
670 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_EXCL_LIMIT
);
671 s
->excl_limit
= (val
& AMDVI_MMIO_EXCL_LIMIT_MASK
) |
672 AMDVI_MMIO_EXCL_LIMIT_LOW
;
675 static inline void amdvi_handle_evtbase_write(AMDVIState
*s
)
677 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_EVENT_BASE
);
678 s
->evtlog
= val
& AMDVI_MMIO_EVTLOG_BASE_MASK
;
679 s
->evtlog_len
= 1UL << (amdvi_readq(s
, AMDVI_MMIO_EVTLOG_SIZE_BYTE
)
680 & AMDVI_MMIO_EVTLOG_SIZE_MASK
);
683 static inline void amdvi_handle_evttail_write(AMDVIState
*s
)
685 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_EVENT_TAIL
);
686 s
->evtlog_tail
= val
& AMDVI_MMIO_EVTLOG_TAIL_MASK
;
689 static inline void amdvi_handle_evthead_write(AMDVIState
*s
)
691 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_EVENT_HEAD
);
692 s
->evtlog_head
= val
& AMDVI_MMIO_EVTLOG_HEAD_MASK
;
695 static inline void amdvi_handle_pprbase_write(AMDVIState
*s
)
697 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_PPR_BASE
);
698 s
->ppr_log
= val
& AMDVI_MMIO_PPRLOG_BASE_MASK
;
699 s
->pprlog_len
= 1UL << (amdvi_readq(s
, AMDVI_MMIO_PPRLOG_SIZE_BYTE
)
700 & AMDVI_MMIO_PPRLOG_SIZE_MASK
);
703 static inline void amdvi_handle_pprhead_write(AMDVIState
*s
)
705 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_PPR_HEAD
);
706 s
->pprlog_head
= val
& AMDVI_MMIO_PPRLOG_HEAD_MASK
;
709 static inline void amdvi_handle_pprtail_write(AMDVIState
*s
)
711 uint64_t val
= amdvi_readq(s
, AMDVI_MMIO_PPR_TAIL
);
712 s
->pprlog_tail
= val
& AMDVI_MMIO_PPRLOG_TAIL_MASK
;
715 /* FIXME: something might go wrong if System Software writes in chunks
716 * of one byte but linux writes in chunks of 4 bytes so currently it
717 * works correctly with linux but will definitely be busted if software
718 * reads/writes 8 bytes
720 static void amdvi_mmio_reg_write(AMDVIState
*s
, unsigned size
, uint64_t val
,
724 amdvi_writew(s
, addr
, val
);
725 } else if (size
== 4) {
726 amdvi_writel(s
, addr
, val
);
727 } else if (size
== 8) {
728 amdvi_writeq(s
, addr
, val
);
732 static void amdvi_mmio_write(void *opaque
, hwaddr addr
, uint64_t val
,
735 AMDVIState
*s
= opaque
;
736 unsigned long offset
= addr
& 0x07;
738 if (addr
+ size
> AMDVI_MMIO_SIZE
) {
739 trace_amdvi_mmio_write("error: addr outside region: max ",
740 (uint64_t)AMDVI_MMIO_SIZE
, size
, val
, offset
);
744 amdvi_mmio_trace(addr
, size
);
745 switch (addr
& ~0x07) {
746 case AMDVI_MMIO_CONTROL
:
747 amdvi_mmio_reg_write(s
, size
, val
, addr
);
748 amdvi_handle_control_write(s
);
750 case AMDVI_MMIO_DEVICE_TABLE
:
751 amdvi_mmio_reg_write(s
, size
, val
, addr
);
752 /* set device table address
753 * This also suffers from inability to tell whether software
756 if (offset
|| (size
== 8)) {
757 amdvi_handle_devtab_write(s
);
760 case AMDVI_MMIO_COMMAND_HEAD
:
761 amdvi_mmio_reg_write(s
, size
, val
, addr
);
762 amdvi_handle_cmdhead_write(s
);
764 case AMDVI_MMIO_COMMAND_BASE
:
765 amdvi_mmio_reg_write(s
, size
, val
, addr
);
766 /* FIXME - make sure System Software has finished writing incase
767 * it writes in chucks less than 8 bytes in a robust way.As for
768 * now, this hacks works for the linux driver
770 if (offset
|| (size
== 8)) {
771 amdvi_handle_cmdbase_write(s
);
774 case AMDVI_MMIO_COMMAND_TAIL
:
775 amdvi_mmio_reg_write(s
, size
, val
, addr
);
776 amdvi_handle_cmdtail_write(s
);
778 case AMDVI_MMIO_EVENT_BASE
:
779 amdvi_mmio_reg_write(s
, size
, val
, addr
);
780 amdvi_handle_evtbase_write(s
);
782 case AMDVI_MMIO_EVENT_HEAD
:
783 amdvi_mmio_reg_write(s
, size
, val
, addr
);
784 amdvi_handle_evthead_write(s
);
786 case AMDVI_MMIO_EVENT_TAIL
:
787 amdvi_mmio_reg_write(s
, size
, val
, addr
);
788 amdvi_handle_evttail_write(s
);
790 case AMDVI_MMIO_EXCL_LIMIT
:
791 amdvi_mmio_reg_write(s
, size
, val
, addr
);
792 amdvi_handle_excllim_write(s
);
794 /* PPR log base - unused for now */
795 case AMDVI_MMIO_PPR_BASE
:
796 amdvi_mmio_reg_write(s
, size
, val
, addr
);
797 amdvi_handle_pprbase_write(s
);
799 /* PPR log head - also unused for now */
800 case AMDVI_MMIO_PPR_HEAD
:
801 amdvi_mmio_reg_write(s
, size
, val
, addr
);
802 amdvi_handle_pprhead_write(s
);
804 /* PPR log tail - unused for now */
805 case AMDVI_MMIO_PPR_TAIL
:
806 amdvi_mmio_reg_write(s
, size
, val
, addr
);
807 amdvi_handle_pprtail_write(s
);
812 static inline uint64_t amdvi_get_perms(uint64_t entry
)
814 return (entry
& (AMDVI_DEV_PERM_READ
| AMDVI_DEV_PERM_WRITE
)) >>
815 AMDVI_DEV_PERM_SHIFT
;
818 /* validate that reserved bits are honoured */
819 static bool amdvi_validate_dte(AMDVIState
*s
, uint16_t devid
,
822 if ((dte
[0] & AMDVI_DTE_LOWER_QUAD_RESERVED
)
823 || (dte
[1] & AMDVI_DTE_MIDDLE_QUAD_RESERVED
)
824 || (dte
[2] & AMDVI_DTE_UPPER_QUAD_RESERVED
) || dte
[3]) {
825 amdvi_log_illegaldevtab_error(s
, devid
,
827 devid
* AMDVI_DEVTAB_ENTRY_SIZE
, 0);
834 /* get a device table entry given the devid */
835 static bool amdvi_get_dte(AMDVIState
*s
, int devid
, uint64_t *entry
)
837 uint32_t offset
= devid
* AMDVI_DEVTAB_ENTRY_SIZE
;
839 if (dma_memory_read(&address_space_memory
, s
->devtab
+ offset
, entry
,
840 AMDVI_DEVTAB_ENTRY_SIZE
, MEMTXATTRS_UNSPECIFIED
)) {
841 trace_amdvi_dte_get_fail(s
->devtab
, offset
);
842 /* log error accessing dte */
843 amdvi_log_devtab_error(s
, devid
, s
->devtab
+ offset
, 0);
847 *entry
= le64_to_cpu(*entry
);
848 if (!amdvi_validate_dte(s
, devid
, entry
)) {
849 trace_amdvi_invalid_dte(entry
[0]);
856 /* get pte translation mode */
857 static inline uint8_t get_pte_translation_mode(uint64_t pte
)
859 return (pte
>> AMDVI_DEV_MODE_RSHIFT
) & AMDVI_DEV_MODE_MASK
;
862 static inline uint64_t pte_override_page_mask(uint64_t pte
)
864 uint8_t page_mask
= 13;
865 uint64_t addr
= (pte
& AMDVI_DEV_PT_ROOT_MASK
) >> 12;
866 /* find the first zero bit */
872 return ~((1ULL << page_mask
) - 1);
875 static inline uint64_t pte_get_page_mask(uint64_t oldlevel
)
877 return ~((1UL << ((oldlevel
* 9) + 3)) - 1);
880 static inline uint64_t amdvi_get_pte_entry(AMDVIState
*s
, uint64_t pte_addr
,
885 if (dma_memory_read(&address_space_memory
, pte_addr
,
886 &pte
, sizeof(pte
), MEMTXATTRS_UNSPECIFIED
)) {
887 trace_amdvi_get_pte_hwerror(pte_addr
);
888 amdvi_log_pagetab_error(s
, devid
, pte_addr
, 0);
893 pte
= le64_to_cpu(pte
);
897 static void amdvi_page_walk(AMDVIAddressSpace
*as
, uint64_t *dte
,
898 IOMMUTLBEntry
*ret
, unsigned perms
,
901 unsigned level
, present
, pte_perms
, oldlevel
;
902 uint64_t pte
= dte
[0], pte_addr
, page_mask
;
904 /* make sure the DTE has TV = 1 */
905 if (pte
& AMDVI_DEV_TRANSLATION_VALID
) {
906 level
= get_pte_translation_mode(pte
);
908 trace_amdvi_mode_invalid(level
, addr
);
915 /* we are at the leaf page table or page table encodes a huge page */
917 pte_perms
= amdvi_get_perms(pte
);
919 if (!present
|| perms
!= (perms
& pte_perms
)) {
920 amdvi_page_fault(as
->iommu_state
, as
->devfn
, addr
, perms
);
921 trace_amdvi_page_fault(addr
);
925 /* go to the next lower level */
926 pte_addr
= pte
& AMDVI_DEV_PT_ROOT_MASK
;
927 /* add offset and load pte */
928 pte_addr
+= ((addr
>> (3 + 9 * level
)) & 0x1FF) << 3;
929 pte
= amdvi_get_pte_entry(as
->iommu_state
, pte_addr
, as
->devfn
);
934 level
= get_pte_translation_mode(pte
);
941 page_mask
= pte_override_page_mask(pte
);
943 page_mask
= pte_get_page_mask(oldlevel
);
946 /* get access permissions from pte */
947 ret
->iova
= addr
& page_mask
;
948 ret
->translated_addr
= (pte
& AMDVI_DEV_PT_ROOT_MASK
) & page_mask
;
949 ret
->addr_mask
= ~page_mask
;
950 ret
->perm
= amdvi_get_perms(pte
);
954 ret
->iova
= addr
& AMDVI_PAGE_MASK_4K
;
955 ret
->translated_addr
= addr
& AMDVI_PAGE_MASK_4K
;
956 ret
->addr_mask
= ~AMDVI_PAGE_MASK_4K
;
957 ret
->perm
= amdvi_get_perms(pte
);
960 static void amdvi_do_translate(AMDVIAddressSpace
*as
, hwaddr addr
,
961 bool is_write
, IOMMUTLBEntry
*ret
)
963 AMDVIState
*s
= as
->iommu_state
;
964 uint16_t devid
= PCI_BUILD_BDF(as
->bus_num
, as
->devfn
);
965 AMDVIIOTLBEntry
*iotlb_entry
= amdvi_iotlb_lookup(s
, addr
, devid
);
969 trace_amdvi_iotlb_hit(PCI_BUS_NUM(devid
), PCI_SLOT(devid
),
970 PCI_FUNC(devid
), addr
, iotlb_entry
->translated_addr
);
971 ret
->iova
= addr
& ~iotlb_entry
->page_mask
;
972 ret
->translated_addr
= iotlb_entry
->translated_addr
;
973 ret
->addr_mask
= iotlb_entry
->page_mask
;
974 ret
->perm
= iotlb_entry
->perms
;
978 if (!amdvi_get_dte(s
, devid
, entry
)) {
982 /* devices with V = 0 are not translated */
983 if (!(entry
[0] & AMDVI_DEV_VALID
)) {
987 amdvi_page_walk(as
, entry
, ret
,
988 is_write
? AMDVI_PERM_WRITE
: AMDVI_PERM_READ
, addr
);
990 amdvi_update_iotlb(s
, devid
, addr
, *ret
,
991 entry
[1] & AMDVI_DEV_DOMID_ID_MASK
);
995 ret
->iova
= addr
& AMDVI_PAGE_MASK_4K
;
996 ret
->translated_addr
= addr
& AMDVI_PAGE_MASK_4K
;
997 ret
->addr_mask
= ~AMDVI_PAGE_MASK_4K
;
998 ret
->perm
= IOMMU_RW
;
1001 static inline bool amdvi_is_interrupt_addr(hwaddr addr
)
1003 return addr
>= AMDVI_INT_ADDR_FIRST
&& addr
<= AMDVI_INT_ADDR_LAST
;
1006 static IOMMUTLBEntry
amdvi_translate(IOMMUMemoryRegion
*iommu
, hwaddr addr
,
1007 IOMMUAccessFlags flag
, int iommu_idx
)
1009 AMDVIAddressSpace
*as
= container_of(iommu
, AMDVIAddressSpace
, iommu
);
1010 AMDVIState
*s
= as
->iommu_state
;
1011 IOMMUTLBEntry ret
= {
1012 .target_as
= &address_space_memory
,
1014 .translated_addr
= 0,
1015 .addr_mask
= ~(hwaddr
)0,
1020 /* AMDVI disabled - corresponds to iommu=off not
1021 * failure to provide any parameter
1023 ret
.iova
= addr
& AMDVI_PAGE_MASK_4K
;
1024 ret
.translated_addr
= addr
& AMDVI_PAGE_MASK_4K
;
1025 ret
.addr_mask
= ~AMDVI_PAGE_MASK_4K
;
1026 ret
.perm
= IOMMU_RW
;
1028 } else if (amdvi_is_interrupt_addr(addr
)) {
1029 ret
.iova
= addr
& AMDVI_PAGE_MASK_4K
;
1030 ret
.translated_addr
= addr
& AMDVI_PAGE_MASK_4K
;
1031 ret
.addr_mask
= ~AMDVI_PAGE_MASK_4K
;
1032 ret
.perm
= IOMMU_WO
;
1036 amdvi_do_translate(as
, addr
, flag
& IOMMU_WO
, &ret
);
1037 trace_amdvi_translation_result(as
->bus_num
, PCI_SLOT(as
->devfn
),
1038 PCI_FUNC(as
->devfn
), addr
, ret
.translated_addr
);
1042 static int amdvi_get_irte(AMDVIState
*s
, MSIMessage
*origin
, uint64_t *dte
,
1043 union irte
*irte
, uint16_t devid
)
1045 uint64_t irte_root
, offset
;
1047 irte_root
= dte
[2] & AMDVI_IR_PHYS_ADDR_MASK
;
1048 offset
= (origin
->data
& AMDVI_IRTE_OFFSET
) << 2;
1050 trace_amdvi_ir_irte(irte_root
, offset
);
1052 if (dma_memory_read(&address_space_memory
, irte_root
+ offset
,
1053 irte
, sizeof(*irte
), MEMTXATTRS_UNSPECIFIED
)) {
1054 trace_amdvi_ir_err("failed to get irte");
1055 return -AMDVI_IR_GET_IRTE
;
1058 trace_amdvi_ir_irte_val(irte
->val
);
1063 static int amdvi_int_remap_legacy(AMDVIState
*iommu
,
1065 MSIMessage
*translated
,
1073 /* get interrupt remapping table */
1074 ret
= amdvi_get_irte(iommu
, origin
, dte
, &irte
, sid
);
1079 if (!irte
.fields
.valid
) {
1080 trace_amdvi_ir_target_abort("RemapEn is disabled");
1081 return -AMDVI_IR_TARGET_ABORT
;
1084 if (irte
.fields
.guest_mode
) {
1085 error_report_once("guest mode is not zero");
1086 return -AMDVI_IR_ERR
;
1089 if (irte
.fields
.int_type
> AMDVI_IOAPIC_INT_TYPE_ARBITRATED
) {
1090 error_report_once("reserved int_type");
1091 return -AMDVI_IR_ERR
;
1094 irq
->delivery_mode
= irte
.fields
.int_type
;
1095 irq
->vector
= irte
.fields
.vector
;
1096 irq
->dest_mode
= irte
.fields
.dm
;
1097 irq
->redir_hint
= irte
.fields
.rq_eoi
;
1098 irq
->dest
= irte
.fields
.destination
;
1103 static int amdvi_get_irte_ga(AMDVIState
*s
, MSIMessage
*origin
, uint64_t *dte
,
1104 struct irte_ga
*irte
, uint16_t devid
)
1106 uint64_t irte_root
, offset
;
1108 irte_root
= dte
[2] & AMDVI_IR_PHYS_ADDR_MASK
;
1109 offset
= (origin
->data
& AMDVI_IRTE_OFFSET
) << 4;
1110 trace_amdvi_ir_irte(irte_root
, offset
);
1112 if (dma_memory_read(&address_space_memory
, irte_root
+ offset
,
1113 irte
, sizeof(*irte
), MEMTXATTRS_UNSPECIFIED
)) {
1114 trace_amdvi_ir_err("failed to get irte_ga");
1115 return -AMDVI_IR_GET_IRTE
;
1118 trace_amdvi_ir_irte_ga_val(irte
->hi
.val
, irte
->lo
.val
);
1122 static int amdvi_int_remap_ga(AMDVIState
*iommu
,
1124 MSIMessage
*translated
,
1130 struct irte_ga irte
;
1132 /* get interrupt remapping table */
1133 ret
= amdvi_get_irte_ga(iommu
, origin
, dte
, &irte
, sid
);
1138 if (!irte
.lo
.fields_remap
.valid
) {
1139 trace_amdvi_ir_target_abort("RemapEn is disabled");
1140 return -AMDVI_IR_TARGET_ABORT
;
1143 if (irte
.lo
.fields_remap
.guest_mode
) {
1144 error_report_once("guest mode is not zero");
1145 return -AMDVI_IR_ERR
;
1148 if (irte
.lo
.fields_remap
.int_type
> AMDVI_IOAPIC_INT_TYPE_ARBITRATED
) {
1149 error_report_once("reserved int_type is set");
1150 return -AMDVI_IR_ERR
;
1153 irq
->delivery_mode
= irte
.lo
.fields_remap
.int_type
;
1154 irq
->vector
= irte
.hi
.fields
.vector
;
1155 irq
->dest_mode
= irte
.lo
.fields_remap
.dm
;
1156 irq
->redir_hint
= irte
.lo
.fields_remap
.rq_eoi
;
1157 irq
->dest
= irte
.lo
.fields_remap
.destination
;
1162 static int __amdvi_int_remap_msi(AMDVIState
*iommu
,
1164 MSIMessage
*translated
,
1172 int_ctl
= (dte
[2] >> AMDVI_IR_INTCTL_SHIFT
) & 3;
1173 trace_amdvi_ir_intctl(int_ctl
);
1176 case AMDVI_IR_INTCTL_PASS
:
1177 memcpy(translated
, origin
, sizeof(*origin
));
1179 case AMDVI_IR_INTCTL_REMAP
:
1181 case AMDVI_IR_INTCTL_ABORT
:
1182 trace_amdvi_ir_target_abort("int_ctl abort");
1183 return -AMDVI_IR_TARGET_ABORT
;
1185 trace_amdvi_ir_err("int_ctl reserved");
1186 return -AMDVI_IR_ERR
;
1189 if (iommu
->ga_enabled
) {
1190 ret
= amdvi_int_remap_ga(iommu
, origin
, translated
, dte
, irq
, sid
);
1192 ret
= amdvi_int_remap_legacy(iommu
, origin
, translated
, dte
, irq
, sid
);
1198 /* Interrupt remapping for MSI/MSI-X entry */
1199 static int amdvi_int_remap_msi(AMDVIState
*iommu
,
1201 MSIMessage
*translated
,
1206 uint64_t dte
[4] = { 0 };
1207 X86IOMMUIrq irq
= { 0 };
1208 uint8_t dest_mode
, delivery_mode
;
1210 assert(origin
&& translated
);
1213 * When IOMMU is enabled, interrupt remap request will come either from
1214 * IO-APIC or PCI device. If interrupt is from PCI device then it will
1215 * have a valid requester id but if the interrupt is from IO-APIC
1216 * then requester id will be invalid.
1218 if (sid
== X86_IOMMU_SID_INVALID
) {
1219 sid
= AMDVI_IOAPIC_SB_DEVID
;
1222 trace_amdvi_ir_remap_msi_req(origin
->address
, origin
->data
, sid
);
1224 /* check if device table entry is set before we go further. */
1225 if (!iommu
|| !iommu
->devtab_len
) {
1226 memcpy(translated
, origin
, sizeof(*origin
));
1230 if (!amdvi_get_dte(iommu
, sid
, dte
)) {
1231 return -AMDVI_IR_ERR
;
1234 /* Check if IR is enabled in DTE */
1235 if (!(dte
[2] & AMDVI_IR_REMAP_ENABLE
)) {
1236 memcpy(translated
, origin
, sizeof(*origin
));
1240 /* validate that we are configure with intremap=on */
1241 if (!x86_iommu_ir_supported(X86_IOMMU_DEVICE(iommu
))) {
1242 trace_amdvi_err("Interrupt remapping is enabled in the guest but "
1243 "not in the host. Use intremap=on to enable interrupt "
1244 "remapping in amd-iommu.");
1245 return -AMDVI_IR_ERR
;
1248 if (origin
->address
& AMDVI_MSI_ADDR_HI_MASK
) {
1249 trace_amdvi_err("MSI address high 32 bits non-zero when "
1250 "Interrupt Remapping enabled.");
1251 return -AMDVI_IR_ERR
;
1254 if ((origin
->address
& AMDVI_MSI_ADDR_LO_MASK
) != APIC_DEFAULT_ADDRESS
) {
1255 trace_amdvi_err("MSI is not from IOAPIC.");
1256 return -AMDVI_IR_ERR
;
1260 * The MSI data register [10:8] are used to get the upstream interrupt type.
1262 * See MSI/MSI-X format:
1263 * https://pdfs.semanticscholar.org/presentation/9420/c279e942eca568157711ef5c92b800c40a79.pdf
1266 delivery_mode
= (origin
->data
>> MSI_DATA_DELIVERY_MODE_SHIFT
) & 7;
1268 switch (delivery_mode
) {
1269 case AMDVI_IOAPIC_INT_TYPE_FIXED
:
1270 case AMDVI_IOAPIC_INT_TYPE_ARBITRATED
:
1271 trace_amdvi_ir_delivery_mode("fixed/arbitrated");
1272 ret
= __amdvi_int_remap_msi(iommu
, origin
, translated
, dte
, &irq
, sid
);
1276 /* Translate IRQ to MSI messages */
1277 x86_iommu_irq_to_msi_message(&irq
, translated
);
1281 case AMDVI_IOAPIC_INT_TYPE_SMI
:
1282 error_report("SMI is not supported!");
1283 ret
= -AMDVI_IR_ERR
;
1285 case AMDVI_IOAPIC_INT_TYPE_NMI
:
1286 pass
= dte
[3] & AMDVI_DEV_NMI_PASS_MASK
;
1287 trace_amdvi_ir_delivery_mode("nmi");
1289 case AMDVI_IOAPIC_INT_TYPE_INIT
:
1290 pass
= dte
[3] & AMDVI_DEV_INT_PASS_MASK
;
1291 trace_amdvi_ir_delivery_mode("init");
1293 case AMDVI_IOAPIC_INT_TYPE_EINT
:
1294 pass
= dte
[3] & AMDVI_DEV_EINT_PASS_MASK
;
1295 trace_amdvi_ir_delivery_mode("eint");
1298 trace_amdvi_ir_delivery_mode("unsupported delivery_mode");
1299 ret
= -AMDVI_IR_ERR
;
1308 * The MSI address register bit[2] is used to get the destination
1309 * mode. The dest_mode 1 is valid for fixed and arbitrated interrupts
1312 dest_mode
= (origin
->address
>> MSI_ADDR_DEST_MODE_SHIFT
) & 1;
1314 trace_amdvi_ir_err("invalid dest_mode");
1315 ret
= -AMDVI_IR_ERR
;
1320 memcpy(translated
, origin
, sizeof(*origin
));
1322 trace_amdvi_ir_err("passthrough is not enabled");
1323 ret
= -AMDVI_IR_ERR
;
1328 trace_amdvi_ir_remap_msi(origin
->address
, origin
->data
,
1329 translated
->address
, translated
->data
);
1336 static int amdvi_int_remap(X86IOMMUState
*iommu
,
1338 MSIMessage
*translated
,
1341 return amdvi_int_remap_msi(AMD_IOMMU_DEVICE(iommu
), origin
,
1345 static MemTxResult
amdvi_mem_ir_write(void *opaque
, hwaddr addr
,
1346 uint64_t value
, unsigned size
,
1350 MSIMessage from
= { 0, 0 }, to
= { 0, 0 };
1351 uint16_t sid
= AMDVI_IOAPIC_SB_DEVID
;
1353 from
.address
= (uint64_t) addr
+ AMDVI_INT_ADDR_FIRST
;
1354 from
.data
= (uint32_t) value
;
1356 trace_amdvi_mem_ir_write_req(addr
, value
, size
);
1358 if (!attrs
.unspecified
) {
1359 /* We have explicit Source ID */
1360 sid
= attrs
.requester_id
;
1363 ret
= amdvi_int_remap_msi(opaque
, &from
, &to
, sid
);
1365 /* TODO: log the event using IOMMU log event interface */
1366 error_report_once("failed to remap interrupt from devid 0x%x", sid
);
1370 apic_get_class()->send_msi(&to
);
1372 trace_amdvi_mem_ir_write(to
.address
, to
.data
);
1376 static MemTxResult
amdvi_mem_ir_read(void *opaque
, hwaddr addr
,
1377 uint64_t *data
, unsigned size
,
1383 static const MemoryRegionOps amdvi_ir_ops
= {
1384 .read_with_attrs
= amdvi_mem_ir_read
,
1385 .write_with_attrs
= amdvi_mem_ir_write
,
1386 .endianness
= DEVICE_LITTLE_ENDIAN
,
1388 .min_access_size
= 4,
1389 .max_access_size
= 4,
1392 .min_access_size
= 4,
1393 .max_access_size
= 4,
1397 static AddressSpace
*amdvi_host_dma_iommu(PCIBus
*bus
, void *opaque
, int devfn
)
1400 AMDVIState
*s
= opaque
;
1401 AMDVIAddressSpace
**iommu_as
, *amdvi_dev_as
;
1402 int bus_num
= pci_bus_num(bus
);
1404 iommu_as
= s
->address_spaces
[bus_num
];
1406 /* allocate memory during the first run */
1408 iommu_as
= g_malloc0(sizeof(AMDVIAddressSpace
*) * PCI_DEVFN_MAX
);
1409 s
->address_spaces
[bus_num
] = iommu_as
;
1412 /* set up AMD-Vi region */
1413 if (!iommu_as
[devfn
]) {
1414 snprintf(name
, sizeof(name
), "amd_iommu_devfn_%d", devfn
);
1416 iommu_as
[devfn
] = g_malloc0(sizeof(AMDVIAddressSpace
));
1417 iommu_as
[devfn
]->bus_num
= (uint8_t)bus_num
;
1418 iommu_as
[devfn
]->devfn
= (uint8_t)devfn
;
1419 iommu_as
[devfn
]->iommu_state
= s
;
1421 amdvi_dev_as
= iommu_as
[devfn
];
1424 * Memory region relationships looks like (Address range shows
1425 * only lower 32 bits to make it short in length...):
1427 * |-----------------+-------------------+----------|
1428 * | Name | Address range | Priority |
1429 * |-----------------+-------------------+----------+
1430 * | amdvi_root | 00000000-ffffffff | 0 |
1431 * | amdvi_iommu | 00000000-ffffffff | 1 |
1432 * | amdvi_iommu_ir | fee00000-feefffff | 64 |
1433 * |-----------------+-------------------+----------|
1435 memory_region_init_iommu(&amdvi_dev_as
->iommu
,
1436 sizeof(amdvi_dev_as
->iommu
),
1437 TYPE_AMD_IOMMU_MEMORY_REGION
,
1439 "amd_iommu", UINT64_MAX
);
1440 memory_region_init(&amdvi_dev_as
->root
, OBJECT(s
),
1441 "amdvi_root", UINT64_MAX
);
1442 address_space_init(&amdvi_dev_as
->as
, &amdvi_dev_as
->root
, name
);
1443 memory_region_init_io(&amdvi_dev_as
->iommu_ir
, OBJECT(s
),
1444 &amdvi_ir_ops
, s
, "amd_iommu_ir",
1445 AMDVI_INT_ADDR_SIZE
);
1446 memory_region_add_subregion_overlap(&amdvi_dev_as
->root
,
1447 AMDVI_INT_ADDR_FIRST
,
1448 &amdvi_dev_as
->iommu_ir
,
1450 memory_region_add_subregion_overlap(&amdvi_dev_as
->root
, 0,
1451 MEMORY_REGION(&amdvi_dev_as
->iommu
),
1454 return &iommu_as
[devfn
]->as
;
1457 static const MemoryRegionOps mmio_mem_ops
= {
1458 .read
= amdvi_mmio_read
,
1459 .write
= amdvi_mmio_write
,
1460 .endianness
= DEVICE_LITTLE_ENDIAN
,
1462 .min_access_size
= 1,
1463 .max_access_size
= 8,
1467 .min_access_size
= 1,
1468 .max_access_size
= 8,
1472 static int amdvi_iommu_notify_flag_changed(IOMMUMemoryRegion
*iommu
,
1473 IOMMUNotifierFlag old
,
1474 IOMMUNotifierFlag
new,
1477 AMDVIAddressSpace
*as
= container_of(iommu
, AMDVIAddressSpace
, iommu
);
1479 if (new & IOMMU_NOTIFIER_MAP
) {
1481 "device %02x.%02x.%x requires iommu notifier which is not "
1482 "currently supported", as
->bus_num
, PCI_SLOT(as
->devfn
),
1483 PCI_FUNC(as
->devfn
));
1489 static void amdvi_init(AMDVIState
*s
)
1491 amdvi_iotlb_reset(s
);
1499 s
->excl_enabled
= false;
1500 s
->excl_allow
= false;
1501 s
->mmio_enabled
= false;
1503 s
->ats_enabled
= false;
1504 s
->cmdbuf_enabled
= false;
1507 memset(s
->mmior
, 0, AMDVI_MMIO_SIZE
);
1508 amdvi_set_quad(s
, AMDVI_MMIO_EXT_FEATURES
, AMDVI_EXT_FEATURES
,
1509 0xffffffffffffffef, 0);
1510 amdvi_set_quad(s
, AMDVI_MMIO_STATUS
, 0, 0x98, 0x67);
1512 /* reset device ident */
1513 pci_config_set_vendor_id(s
->pci
.dev
.config
, PCI_VENDOR_ID_AMD
);
1514 pci_config_set_prog_interface(s
->pci
.dev
.config
, 00);
1515 pci_config_set_device_id(s
->pci
.dev
.config
, s
->devid
);
1516 pci_config_set_class(s
->pci
.dev
.config
, 0x0806);
1518 /* reset AMDVI specific capabilities, all r/o */
1519 pci_set_long(s
->pci
.dev
.config
+ s
->capab_offset
, AMDVI_CAPAB_FEATURES
);
1520 pci_set_long(s
->pci
.dev
.config
+ s
->capab_offset
+ AMDVI_CAPAB_BAR_LOW
,
1521 s
->mmio
.addr
& ~(0xffff0000));
1522 pci_set_long(s
->pci
.dev
.config
+ s
->capab_offset
+ AMDVI_CAPAB_BAR_HIGH
,
1523 (s
->mmio
.addr
& ~(0xffff)) >> 16);
1524 pci_set_long(s
->pci
.dev
.config
+ s
->capab_offset
+ AMDVI_CAPAB_RANGE
,
1526 pci_set_long(s
->pci
.dev
.config
+ s
->capab_offset
+ AMDVI_CAPAB_MISC
, 0);
1527 pci_set_long(s
->pci
.dev
.config
+ s
->capab_offset
+ AMDVI_CAPAB_MISC
,
1528 AMDVI_MAX_PH_ADDR
| AMDVI_MAX_GVA_ADDR
| AMDVI_MAX_VA_ADDR
);
1531 static void amdvi_sysbus_reset(DeviceState
*dev
)
1533 AMDVIState
*s
= AMD_IOMMU_DEVICE(dev
);
1535 msi_reset(&s
->pci
.dev
);
1539 static void amdvi_sysbus_realize(DeviceState
*dev
, Error
**errp
)
1542 AMDVIState
*s
= AMD_IOMMU_DEVICE(dev
);
1543 MachineState
*ms
= MACHINE(qdev_get_machine());
1544 PCMachineState
*pcms
= PC_MACHINE(ms
);
1545 X86MachineState
*x86ms
= X86_MACHINE(ms
);
1546 PCIBus
*bus
= pcms
->bus
;
1548 s
->iotlb
= g_hash_table_new_full(amdvi_uint64_hash
,
1549 amdvi_uint64_equal
, g_free
, g_free
);
1551 /* This device should take care of IOMMU PCI properties */
1552 if (!qdev_realize(DEVICE(&s
->pci
), &bus
->qbus
, errp
)) {
1555 ret
= pci_add_capability(&s
->pci
.dev
, AMDVI_CAPAB_ID_SEC
, 0,
1556 AMDVI_CAPAB_SIZE
, errp
);
1560 s
->capab_offset
= ret
;
1562 ret
= pci_add_capability(&s
->pci
.dev
, PCI_CAP_ID_MSI
, 0,
1563 AMDVI_CAPAB_REG_SIZE
, errp
);
1567 ret
= pci_add_capability(&s
->pci
.dev
, PCI_CAP_ID_HT
, 0,
1568 AMDVI_CAPAB_REG_SIZE
, errp
);
1573 /* Pseudo address space under root PCI bus. */
1574 x86ms
->ioapic_as
= amdvi_host_dma_iommu(bus
, s
, AMDVI_IOAPIC_SB_DEVID
);
1577 memory_region_init_io(&s
->mmio
, OBJECT(s
), &mmio_mem_ops
, s
, "amdvi-mmio",
1580 sysbus_init_mmio(SYS_BUS_DEVICE(s
), &s
->mmio
);
1581 sysbus_mmio_map(SYS_BUS_DEVICE(s
), 0, AMDVI_BASE_ADDR
);
1582 pci_setup_iommu(bus
, amdvi_host_dma_iommu
, s
);
1583 s
->devid
= object_property_get_int(OBJECT(&s
->pci
), "addr", &error_abort
);
1584 msi_init(&s
->pci
.dev
, 0, 1, true, false, errp
);
1588 static const VMStateDescription vmstate_amdvi_sysbus
= {
1589 .name
= "amd-iommu",
1593 static void amdvi_sysbus_instance_init(Object
*klass
)
1595 AMDVIState
*s
= AMD_IOMMU_DEVICE(klass
);
1597 object_initialize(&s
->pci
, sizeof(s
->pci
), TYPE_AMD_IOMMU_PCI
);
1600 static void amdvi_sysbus_class_init(ObjectClass
*klass
, void *data
)
1602 DeviceClass
*dc
= DEVICE_CLASS(klass
);
1603 X86IOMMUClass
*dc_class
= X86_IOMMU_DEVICE_CLASS(klass
);
1605 dc
->reset
= amdvi_sysbus_reset
;
1606 dc
->vmsd
= &vmstate_amdvi_sysbus
;
1607 dc
->hotpluggable
= false;
1608 dc_class
->realize
= amdvi_sysbus_realize
;
1609 dc_class
->int_remap
= amdvi_int_remap
;
1610 /* Supported by the pc-q35-* machine types */
1611 dc
->user_creatable
= true;
1612 set_bit(DEVICE_CATEGORY_MISC
, dc
->categories
);
1613 dc
->desc
= "AMD IOMMU (AMD-Vi) DMA Remapping device";
1616 static const TypeInfo amdvi_sysbus
= {
1617 .name
= TYPE_AMD_IOMMU_DEVICE
,
1618 .parent
= TYPE_X86_IOMMU_DEVICE
,
1619 .instance_size
= sizeof(AMDVIState
),
1620 .instance_init
= amdvi_sysbus_instance_init
,
1621 .class_init
= amdvi_sysbus_class_init
1624 static void amdvi_pci_class_init(ObjectClass
*klass
, void *data
)
1626 DeviceClass
*dc
= DEVICE_CLASS(klass
);
1628 set_bit(DEVICE_CATEGORY_MISC
, dc
->categories
);
1629 dc
->desc
= "AMD IOMMU (AMD-Vi) DMA Remapping device";
1632 static const TypeInfo amdvi_pci
= {
1633 .name
= TYPE_AMD_IOMMU_PCI
,
1634 .parent
= TYPE_PCI_DEVICE
,
1635 .instance_size
= sizeof(AMDVIPCIState
),
1636 .class_init
= amdvi_pci_class_init
,
1637 .interfaces
= (InterfaceInfo
[]) {
1638 { INTERFACE_CONVENTIONAL_PCI_DEVICE
},
1643 static void amdvi_iommu_memory_region_class_init(ObjectClass
*klass
, void *data
)
1645 IOMMUMemoryRegionClass
*imrc
= IOMMU_MEMORY_REGION_CLASS(klass
);
1647 imrc
->translate
= amdvi_translate
;
1648 imrc
->notify_flag_changed
= amdvi_iommu_notify_flag_changed
;
1651 static const TypeInfo amdvi_iommu_memory_region_info
= {
1652 .parent
= TYPE_IOMMU_MEMORY_REGION
,
1653 .name
= TYPE_AMD_IOMMU_MEMORY_REGION
,
1654 .class_init
= amdvi_iommu_memory_region_class_init
,
1657 static void amdvi_register_types(void)
1659 type_register_static(&amdvi_pci
);
1660 type_register_static(&amdvi_sysbus
);
1661 type_register_static(&amdvi_iommu_memory_region_info
);
1664 type_init(amdvi_register_types
);