?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
scsi: Address spurious clang warning
[qemu.git] / hw / i386 / amd_iommu.c
blob353a810e6bc8097e3f2e143339503365b0a5a98d
1 /*
2 * QEMU emulation of AMD IOMMU (AMD-Vi)
3 *
4 * Copyright (C) 2011 Eduard - Gabriel Munteanu
5 * Copyright (C) 2015 David Kiarie, <davidkiarie4@gmail.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16
17 * You should have received a copy of the GNU General Public License along
18 * with this program; if not, see <http://www.gnu.org/licenses/>.
19 *
20 * Cache implementation inspired by hw/i386/intel_iommu.c
21 */
22 #include "qemu/osdep.h"
23 #include "hw/i386/pc.h"
24 #include "hw/pci/msi.h"
25 #include "hw/pci/pci_bus.h"
26 #include "amd_iommu.h"
27 #include "qapi/error.h"
28 #include "qemu/error-report.h"
29 #include "hw/i386/apic_internal.h"
30 #include "trace.h"
31 #include "hw/i386/apic-msidef.h"
32
33 /* used AMD-Vi MMIO registers */
34 const char *amdvi_mmio_low[] = {
35 "AMDVI_MMIO_DEVTAB_BASE",
36 "AMDVI_MMIO_CMDBUF_BASE",
37 "AMDVI_MMIO_EVTLOG_BASE",
38 "AMDVI_MMIO_CONTROL",
39 "AMDVI_MMIO_EXCL_BASE",
40 "AMDVI_MMIO_EXCL_LIMIT",
41 "AMDVI_MMIO_EXT_FEATURES",
42 "AMDVI_MMIO_PPR_BASE",
43 "UNHANDLED"
44 };
45 const char *amdvi_mmio_high[] = {
46 "AMDVI_MMIO_COMMAND_HEAD",
47 "AMDVI_MMIO_COMMAND_TAIL",
48 "AMDVI_MMIO_EVTLOG_HEAD",
49 "AMDVI_MMIO_EVTLOG_TAIL",
50 "AMDVI_MMIO_STATUS",
51 "AMDVI_MMIO_PPR_HEAD",
52 "AMDVI_MMIO_PPR_TAIL",
53 "UNHANDLED"
54 };
55
56 struct AMDVIAddressSpace {
57 uint8_t bus_num; /* bus number */
58 uint8_t devfn; /* device function */
59 AMDVIState *iommu_state; /* AMDVI - one per machine */
60 MemoryRegion root; /* AMDVI Root memory map region */
61 IOMMUMemoryRegion iommu; /* Device's address translation region */
62 MemoryRegion iommu_ir; /* Device's interrupt remapping region */
63 AddressSpace as; /* device's corresponding address space */
64 };
65
66 /* AMDVI cache entry */
67 typedef struct AMDVIIOTLBEntry {
68 uint16_t domid; /* assigned domain id */
69 uint16_t devid; /* device owning entry */
70 uint64_t perms; /* access permissions */
71 uint64_t translated_addr; /* translated address */
72 uint64_t page_mask; /* physical page size */
73 } AMDVIIOTLBEntry;
74
75 /* configure MMIO registers at startup/reset */
76 static void amdvi_set_quad(AMDVIState *s, hwaddr addr, uint64_t val,
77 uint64_t romask, uint64_t w1cmask)
78 {
79 stq_le_p(&s->mmior[addr], val);
80 stq_le_p(&s->romask[addr], romask);
81 stq_le_p(&s->w1cmask[addr], w1cmask);
82 }
83
84 static uint16_t amdvi_readw(AMDVIState *s, hwaddr addr)
85 {
86 return lduw_le_p(&s->mmior[addr]);
87 }
88
89 static uint32_t amdvi_readl(AMDVIState *s, hwaddr addr)
90 {
91 return ldl_le_p(&s->mmior[addr]);
92 }
93
94 static uint64_t amdvi_readq(AMDVIState *s, hwaddr addr)
95 {
96 return ldq_le_p(&s->mmior[addr]);
97 }
98
99 /* internal write */
100 static void amdvi_writeq_raw(AMDVIState *s, uint64_t val, hwaddr addr)
101 {
102 stq_le_p(&s->mmior[addr], val);
103 }
104
105 /* external write */
106 static void amdvi_writew(AMDVIState *s, hwaddr addr, uint16_t val)
107 {
108 uint16_t romask = lduw_le_p(&s->romask[addr]);
109 uint16_t w1cmask = lduw_le_p(&s->w1cmask[addr]);
110 uint16_t oldval = lduw_le_p(&s->mmior[addr]);
111 stw_le_p(&s->mmior[addr],
112 ((oldval & romask) | (val & ~romask)) & ~(val & w1cmask));
113 }
114
115 static void amdvi_writel(AMDVIState *s, hwaddr addr, uint32_t val)
116 {
117 uint32_t romask = ldl_le_p(&s->romask[addr]);
118 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]);
119 uint32_t oldval = ldl_le_p(&s->mmior[addr]);
120 stl_le_p(&s->mmior[addr],
121 ((oldval & romask) | (val & ~romask)) & ~(val & w1cmask));
122 }
123
124 static void amdvi_writeq(AMDVIState *s, hwaddr addr, uint64_t val)
125 {
126 uint64_t romask = ldq_le_p(&s->romask[addr]);
127 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]);
128 uint32_t oldval = ldq_le_p(&s->mmior[addr]);
129 stq_le_p(&s->mmior[addr],
130 ((oldval & romask) | (val & ~romask)) & ~(val & w1cmask));
131 }
132
133 /* OR a 64-bit register with a 64-bit value */
134 static bool amdvi_test_mask(AMDVIState *s, hwaddr addr, uint64_t val)
135 {
136 return amdvi_readq(s, addr) | val;
137 }
138
139 /* OR a 64-bit register with a 64-bit value storing result in the register */
140 static void amdvi_assign_orq(AMDVIState *s, hwaddr addr, uint64_t val)
141 {
142 amdvi_writeq_raw(s, addr, amdvi_readq(s, addr) | val);
143 }
144
145 /* AND a 64-bit register with a 64-bit value storing result in the register */
146 static void amdvi_assign_andq(AMDVIState *s, hwaddr addr, uint64_t val)
147 {
148 amdvi_writeq_raw(s, addr, amdvi_readq(s, addr) & val);
149 }
150
151 static void amdvi_generate_msi_interrupt(AMDVIState *s)
152 {
153 MSIMessage msg = {};
154 MemTxAttrs attrs = {
155 .requester_id = pci_requester_id(&s->pci.dev)
156 };
157
158 if (msi_enabled(&s->pci.dev)) {
159 msg = msi_get_message(&s->pci.dev, 0);
160 address_space_stl_le(&address_space_memory, msg.address, msg.data,
161 attrs, NULL);
162 }
163 }
164
165 static void amdvi_log_event(AMDVIState *s, uint64_t *evt)
166 {
167 /* event logging not enabled */
168 if (!s->evtlog_enabled || amdvi_test_mask(s, AMDVI_MMIO_STATUS,
169 AMDVI_MMIO_STATUS_EVT_OVF)) {
170 return;
171 }
172
173 /* event log buffer full */
174 if (s->evtlog_tail >= s->evtlog_len) {
175 amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_EVT_OVF);
176 /* generate interrupt */
177 amdvi_generate_msi_interrupt(s);
178 return;
179 }
180
181 if (dma_memory_write(&address_space_memory, s->evtlog + s->evtlog_tail,
182 &evt, AMDVI_EVENT_LEN)) {
183 trace_amdvi_evntlog_fail(s->evtlog, s->evtlog_tail);
184 }
185
186 s->evtlog_tail += AMDVI_EVENT_LEN;
187 amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_COMP_INT);
188 amdvi_generate_msi_interrupt(s);
189 }
190
191 static void amdvi_setevent_bits(uint64_t *buffer, uint64_t value, int start,
192 int length)
193 {
194 int index = start / 64, bitpos = start % 64;
195 uint64_t mask = MAKE_64BIT_MASK(start, length);
196 buffer[index] &= ~mask;
197 buffer[index] |= (value << bitpos) & mask;
198 }
199 /*
200 * AMDVi event structure
201 * 0:15 -> DeviceID
202 * 55:63 -> event type + miscellaneous info
203 * 63:127 -> related address
204 */
205 static void amdvi_encode_event(uint64_t *evt, uint16_t devid, uint64_t addr,
206 uint16_t info)
207 {
208 amdvi_setevent_bits(evt, devid, 0, 16);
209 amdvi_setevent_bits(evt, info, 55, 8);
210 amdvi_setevent_bits(evt, addr, 63, 64);
211 }
212 /* log an error encountered during a page walk
213 *
214 * @addr: virtual address in translation request
215 */
216 static void amdvi_page_fault(AMDVIState *s, uint16_t devid,
217 hwaddr addr, uint16_t info)
218 {
219 uint64_t evt[4];
220
221 info |= AMDVI_EVENT_IOPF_I | AMDVI_EVENT_IOPF;
222 amdvi_encode_event(evt, devid, addr, info);
223 amdvi_log_event(s, evt);
224 pci_word_test_and_set_mask(s->pci.dev.config + PCI_STATUS,
225 PCI_STATUS_SIG_TARGET_ABORT);
226 }
227 /*
228 * log a master abort accessing device table
229 * @devtab : address of device table entry
230 * @info : error flags
231 */
232 static void amdvi_log_devtab_error(AMDVIState *s, uint16_t devid,
233 hwaddr devtab, uint16_t info)
234 {
235 uint64_t evt[4];
236
237 info |= AMDVI_EVENT_DEV_TAB_HW_ERROR;
238
239 amdvi_encode_event(evt, devid, devtab, info);
240 amdvi_log_event(s, evt);
241 pci_word_test_and_set_mask(s->pci.dev.config + PCI_STATUS,
242 PCI_STATUS_SIG_TARGET_ABORT);
243 }
244 /* log an event trying to access command buffer
245 * @addr : address that couldn't be accessed
246 */
247 static void amdvi_log_command_error(AMDVIState *s, hwaddr addr)
248 {
249 uint64_t evt[4], info = AMDVI_EVENT_COMMAND_HW_ERROR;
250
251 amdvi_encode_event(evt, 0, addr, info);
252 amdvi_log_event(s, evt);
253 pci_word_test_and_set_mask(s->pci.dev.config + PCI_STATUS,
254 PCI_STATUS_SIG_TARGET_ABORT);
255 }
256 /* log an illegal comand event
257 * @addr : address of illegal command
258 */
259 static void amdvi_log_illegalcom_error(AMDVIState *s, uint16_t info,
260 hwaddr addr)
261 {
262 uint64_t evt[4];
263
264 info |= AMDVI_EVENT_ILLEGAL_COMMAND_ERROR;
265 amdvi_encode_event(evt, 0, addr, info);
266 amdvi_log_event(s, evt);
267 }
268 /* log an error accessing device table
269 *
270 * @devid : device owning the table entry
271 * @devtab : address of device table entry
272 * @info : error flags
273 */
274 static void amdvi_log_illegaldevtab_error(AMDVIState *s, uint16_t devid,
275 hwaddr addr, uint16_t info)
276 {
277 uint64_t evt[4];
278
279 info |= AMDVI_EVENT_ILLEGAL_DEVTAB_ENTRY;
280 amdvi_encode_event(evt, devid, addr, info);
281 amdvi_log_event(s, evt);
282 }
283 /* log an error accessing a PTE entry
284 * @addr : address that couldn't be accessed
285 */
286 static void amdvi_log_pagetab_error(AMDVIState *s, uint16_t devid,
287 hwaddr addr, uint16_t info)
288 {
289 uint64_t evt[4];
290
291 info |= AMDVI_EVENT_PAGE_TAB_HW_ERROR;
292 amdvi_encode_event(evt, devid, addr, info);
293 amdvi_log_event(s, evt);
294 pci_word_test_and_set_mask(s->pci.dev.config + PCI_STATUS,
295 PCI_STATUS_SIG_TARGET_ABORT);
296 }
297
298 static gboolean amdvi_uint64_equal(gconstpointer v1, gconstpointer v2)
299 {
300 return *((const uint64_t *)v1) == *((const uint64_t *)v2);
301 }
302
303 static guint amdvi_uint64_hash(gconstpointer v)
304 {
305 return (guint)*(const uint64_t *)v;
306 }
307
308 static AMDVIIOTLBEntry *amdvi_iotlb_lookup(AMDVIState *s, hwaddr addr,
309 uint64_t devid)
310 {
311 uint64_t key = (addr >> AMDVI_PAGE_SHIFT_4K) |
312 ((uint64_t)(devid) << AMDVI_DEVID_SHIFT);
313 return g_hash_table_lookup(s->iotlb, &key);
314 }
315
316 static void amdvi_iotlb_reset(AMDVIState *s)
317 {
318 assert(s->iotlb);
319 trace_amdvi_iotlb_reset();
320 g_hash_table_remove_all(s->iotlb);
321 }
322
323 static gboolean amdvi_iotlb_remove_by_devid(gpointer key, gpointer value,
324 gpointer user_data)
325 {
326 AMDVIIOTLBEntry *entry = (AMDVIIOTLBEntry *)value;
327 uint16_t devid = *(uint16_t *)user_data;
328 return entry->devid == devid;
329 }
330
331 static void amdvi_iotlb_remove_page(AMDVIState *s, hwaddr addr,
332 uint64_t devid)
333 {
334 uint64_t key = (addr >> AMDVI_PAGE_SHIFT_4K) |
335 ((uint64_t)(devid) << AMDVI_DEVID_SHIFT);
336 g_hash_table_remove(s->iotlb, &key);
337 }
338
339 static void amdvi_update_iotlb(AMDVIState *s, uint16_t devid,
340 uint64_t gpa, IOMMUTLBEntry to_cache,
341 uint16_t domid)
342 {
343 AMDVIIOTLBEntry *entry = g_new(AMDVIIOTLBEntry, 1);
344 uint64_t *key = g_new(uint64_t, 1);
345 uint64_t gfn = gpa >> AMDVI_PAGE_SHIFT_4K;
346
347 /* don't cache erroneous translations */
348 if (to_cache.perm != IOMMU_NONE) {
349 trace_amdvi_cache_update(domid, PCI_BUS_NUM(devid), PCI_SLOT(devid),
350 PCI_FUNC(devid), gpa, to_cache.translated_addr);
351
352 if (g_hash_table_size(s->iotlb) >= AMDVI_IOTLB_MAX_SIZE) {
353 amdvi_iotlb_reset(s);
354 }
355
356 entry->domid = domid;
357 entry->perms = to_cache.perm;
358 entry->translated_addr = to_cache.translated_addr;
359 entry->page_mask = to_cache.addr_mask;
360 *key = gfn | ((uint64_t)(devid) << AMDVI_DEVID_SHIFT);
361 g_hash_table_replace(s->iotlb, key, entry);
362 }
363 }
364
365 static void amdvi_completion_wait(AMDVIState *s, uint64_t *cmd)
366 {
367 /* pad the last 3 bits */
368 hwaddr addr = cpu_to_le64(extract64(cmd[0], 3, 49)) << 3;
369 uint64_t data = cpu_to_le64(cmd[1]);
370
371 if (extract64(cmd[0], 51, 8)) {
372 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
373 s->cmdbuf + s->cmdbuf_head);
374 }
375 if (extract64(cmd[0], 0, 1)) {
376 if (dma_memory_write(&address_space_memory, addr, &data,
377 AMDVI_COMPLETION_DATA_SIZE)) {
378 trace_amdvi_completion_wait_fail(addr);
379 }
380 }
381 /* set completion interrupt */
382 if (extract64(cmd[0], 1, 1)) {
383 amdvi_test_mask(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_COMP_INT);
384 /* generate interrupt */
385 amdvi_generate_msi_interrupt(s);
386 }
387 trace_amdvi_completion_wait(addr, data);
388 }
389
390 /* log error without aborting since linux seems to be using reserved bits */
391 static void amdvi_inval_devtab_entry(AMDVIState *s, uint64_t *cmd)
392 {
393 uint16_t devid = cpu_to_le16((uint16_t)extract64(cmd[0], 0, 16));
394
395 /* This command should invalidate internal caches of which there isn't */
396 if (extract64(cmd[0], 15, 16) || cmd[1]) {
397 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
398 s->cmdbuf + s->cmdbuf_head);
399 }
400 trace_amdvi_devtab_inval(PCI_BUS_NUM(devid), PCI_SLOT(devid),
401 PCI_FUNC(devid));
402 }
403
404 static void amdvi_complete_ppr(AMDVIState *s, uint64_t *cmd)
405 {
406 if (extract64(cmd[0], 15, 16) || extract64(cmd[0], 19, 8) ||
407 extract64(cmd[1], 0, 2) || extract64(cmd[1], 3, 29)
408 || extract64(cmd[1], 47, 16)) {
409 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
410 s->cmdbuf + s->cmdbuf_head);
411 }
412 trace_amdvi_ppr_exec();
413 }
414
415 static void amdvi_inval_all(AMDVIState *s, uint64_t *cmd)
416 {
417 if (extract64(cmd[0], 0, 60) || cmd[1]) {
418 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
419 s->cmdbuf + s->cmdbuf_head);
420 }
421
422 amdvi_iotlb_reset(s);
423 trace_amdvi_all_inval();
424 }
425
426 static gboolean amdvi_iotlb_remove_by_domid(gpointer key, gpointer value,
427 gpointer user_data)
428 {
429 AMDVIIOTLBEntry *entry = (AMDVIIOTLBEntry *)value;
430 uint16_t domid = *(uint16_t *)user_data;
431 return entry->domid == domid;
432 }
433
434 /* we don't have devid - we can't remove pages by address */
435 static void amdvi_inval_pages(AMDVIState *s, uint64_t *cmd)
436 {
437 uint16_t domid = cpu_to_le16((uint16_t)extract64(cmd[0], 32, 16));
438
439 if (extract64(cmd[0], 20, 12) || extract64(cmd[0], 16, 12) ||
440 extract64(cmd[0], 3, 10)) {
441 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
442 s->cmdbuf + s->cmdbuf_head);
443 }
444
445 g_hash_table_foreach_remove(s->iotlb, amdvi_iotlb_remove_by_domid,
446 &domid);
447 trace_amdvi_pages_inval(domid);
448 }
449
450 static void amdvi_prefetch_pages(AMDVIState *s, uint64_t *cmd)
451 {
452 if (extract64(cmd[0], 16, 8) || extract64(cmd[0], 20, 8) ||
453 extract64(cmd[1], 1, 1) || extract64(cmd[1], 3, 1) ||
454 extract64(cmd[1], 5, 7)) {
455 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
456 s->cmdbuf + s->cmdbuf_head);
457 }
458
459 trace_amdvi_prefetch_pages();
460 }
461
462 static void amdvi_inval_inttable(AMDVIState *s, uint64_t *cmd)
463 {
464 if (extract64(cmd[0], 16, 16) || cmd[1]) {
465 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
466 s->cmdbuf + s->cmdbuf_head);
467 return;
468 }
469
470 trace_amdvi_intr_inval();
471 }
472
473 /* FIXME: Try to work with the specified size instead of all the pages
474 * when the S bit is on
475 */
476 static void iommu_inval_iotlb(AMDVIState *s, uint64_t *cmd)
477 {
478
479 uint16_t devid = extract64(cmd[0], 0, 16);
480 if (extract64(cmd[1], 1, 1) || extract64(cmd[1], 3, 9)) {
481 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4),
482 s->cmdbuf + s->cmdbuf_head);
483 return;
484 }
485
486 if (extract64(cmd[1], 0, 1)) {
487 g_hash_table_foreach_remove(s->iotlb, amdvi_iotlb_remove_by_devid,
488 &devid);
489 } else {
490 amdvi_iotlb_remove_page(s, cpu_to_le64(extract64(cmd[1], 12, 52)) << 12,
491 cpu_to_le16(extract64(cmd[1], 0, 16)));
492 }
493 trace_amdvi_iotlb_inval();
494 }
495
496 /* not honouring reserved bits is regarded as an illegal command */
497 static void amdvi_cmdbuf_exec(AMDVIState *s)
498 {
499 uint64_t cmd[2];
500
501 if (dma_memory_read(&address_space_memory, s->cmdbuf + s->cmdbuf_head,
502 cmd, AMDVI_COMMAND_SIZE)) {
503 trace_amdvi_command_read_fail(s->cmdbuf, s->cmdbuf_head);
504 amdvi_log_command_error(s, s->cmdbuf + s->cmdbuf_head);
505 return;
506 }
507
508 switch (extract64(cmd[0], 60, 4)) {
509 case AMDVI_CMD_COMPLETION_WAIT:
510 amdvi_completion_wait(s, cmd);
511 break;
512 case AMDVI_CMD_INVAL_DEVTAB_ENTRY:
513 amdvi_inval_devtab_entry(s, cmd);
514 break;
515 case AMDVI_CMD_INVAL_AMDVI_PAGES:
516 amdvi_inval_pages(s, cmd);
517 break;
518 case AMDVI_CMD_INVAL_IOTLB_PAGES:
519 iommu_inval_iotlb(s, cmd);
520 break;
521 case AMDVI_CMD_INVAL_INTR_TABLE:
522 amdvi_inval_inttable(s, cmd);
523 break;
524 case AMDVI_CMD_PREFETCH_AMDVI_PAGES:
525 amdvi_prefetch_pages(s, cmd);
526 break;
527 case AMDVI_CMD_COMPLETE_PPR_REQUEST:
528 amdvi_complete_ppr(s, cmd);
529 break;
530 case AMDVI_CMD_INVAL_AMDVI_ALL:
531 amdvi_inval_all(s, cmd);
532 break;
533 default:
534 trace_amdvi_unhandled_command(extract64(cmd[1], 60, 4));
535 /* log illegal command */
536 amdvi_log_illegalcom_error(s, extract64(cmd[1], 60, 4),
537 s->cmdbuf + s->cmdbuf_head);
538 }
539 }
540
541 static void amdvi_cmdbuf_run(AMDVIState *s)
542 {
543 if (!s->cmdbuf_enabled) {
544 trace_amdvi_command_error(amdvi_readq(s, AMDVI_MMIO_CONTROL));
545 return;
546 }
547
548 /* check if there is work to do. */
549 while (s->cmdbuf_head != s->cmdbuf_tail) {
550 trace_amdvi_command_exec(s->cmdbuf_head, s->cmdbuf_tail, s->cmdbuf);
551 amdvi_cmdbuf_exec(s);
552 s->cmdbuf_head += AMDVI_COMMAND_SIZE;
553 amdvi_writeq_raw(s, s->cmdbuf_head, AMDVI_MMIO_COMMAND_HEAD);
554
555 /* wrap head pointer */
556 if (s->cmdbuf_head >= s->cmdbuf_len * AMDVI_COMMAND_SIZE) {
557 s->cmdbuf_head = 0;
558 }
559 }
560 }
561
562 static void amdvi_mmio_trace(hwaddr addr, unsigned size)
563 {
564 uint8_t index = (addr & ~0x2000) / 8;
565
566 if ((addr & 0x2000)) {
567 /* high table */
568 index = index >= AMDVI_MMIO_REGS_HIGH ? AMDVI_MMIO_REGS_HIGH : index;
569 trace_amdvi_mmio_read(amdvi_mmio_high[index], addr, size, addr & ~0x07);
570 } else {
571 index = index >= AMDVI_MMIO_REGS_LOW ? AMDVI_MMIO_REGS_LOW : index;
572 trace_amdvi_mmio_read(amdvi_mmio_low[index], addr, size, addr & ~0x07);
573 }
574 }
575
576 static uint64_t amdvi_mmio_read(void *opaque, hwaddr addr, unsigned size)
577 {
578 AMDVIState *s = opaque;
579
580 uint64_t val = -1;
581 if (addr + size > AMDVI_MMIO_SIZE) {
582 trace_amdvi_mmio_read_invalid(AMDVI_MMIO_SIZE, addr, size);
583 return (uint64_t)-1;
584 }
585
586 if (size == 2) {
587 val = amdvi_readw(s, addr);
588 } else if (size == 4) {
589 val = amdvi_readl(s, addr);
590 } else if (size == 8) {
591 val = amdvi_readq(s, addr);
592 }
593 amdvi_mmio_trace(addr, size);
594
595 return val;
596 }
597
598 static void amdvi_handle_control_write(AMDVIState *s)
599 {
600 unsigned long control = amdvi_readq(s, AMDVI_MMIO_CONTROL);
601 s->enabled = !!(control & AMDVI_MMIO_CONTROL_AMDVIEN);
602
603 s->ats_enabled = !!(control & AMDVI_MMIO_CONTROL_HTTUNEN);
604 s->evtlog_enabled = s->enabled && !!(control &
605 AMDVI_MMIO_CONTROL_EVENTLOGEN);
606
607 s->evtlog_intr = !!(control & AMDVI_MMIO_CONTROL_EVENTINTEN);
608 s->completion_wait_intr = !!(control & AMDVI_MMIO_CONTROL_COMWAITINTEN);
609 s->cmdbuf_enabled = s->enabled && !!(control &
610 AMDVI_MMIO_CONTROL_CMDBUFLEN);
611 s->ga_enabled = !!(control & AMDVI_MMIO_CONTROL_GAEN);
612
613 /* update the flags depending on the control register */
614 if (s->cmdbuf_enabled) {
615 amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_CMDBUF_RUN);
616 } else {
617 amdvi_assign_andq(s, AMDVI_MMIO_STATUS, ~AMDVI_MMIO_STATUS_CMDBUF_RUN);
618 }
619 if (s->evtlog_enabled) {
620 amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_EVT_RUN);
621 } else {
622 amdvi_assign_andq(s, AMDVI_MMIO_STATUS, ~AMDVI_MMIO_STATUS_EVT_RUN);
623 }
624
625 trace_amdvi_control_status(control);
626 amdvi_cmdbuf_run(s);
627 }
628
629 static inline void amdvi_handle_devtab_write(AMDVIState *s)
630
631 {
632 uint64_t val = amdvi_readq(s, AMDVI_MMIO_DEVICE_TABLE);
633 s->devtab = (val & AMDVI_MMIO_DEVTAB_BASE_MASK);
634
635 /* set device table length */
636 s->devtab_len = ((val & AMDVI_MMIO_DEVTAB_SIZE_MASK) + 1 *
637 (AMDVI_MMIO_DEVTAB_SIZE_UNIT /
638 AMDVI_MMIO_DEVTAB_ENTRY_SIZE));
639 }
640
641 static inline void amdvi_handle_cmdhead_write(AMDVIState *s)
642 {
643 s->cmdbuf_head = amdvi_readq(s, AMDVI_MMIO_COMMAND_HEAD)
644 & AMDVI_MMIO_CMDBUF_HEAD_MASK;
645 amdvi_cmdbuf_run(s);
646 }
647
648 static inline void amdvi_handle_cmdbase_write(AMDVIState *s)
649 {
650 s->cmdbuf = amdvi_readq(s, AMDVI_MMIO_COMMAND_BASE)
651 & AMDVI_MMIO_CMDBUF_BASE_MASK;
652 s->cmdbuf_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_CMDBUF_SIZE_BYTE)
653 & AMDVI_MMIO_CMDBUF_SIZE_MASK);
654 s->cmdbuf_head = s->cmdbuf_tail = 0;
655 }
656
657 static inline void amdvi_handle_cmdtail_write(AMDVIState *s)
658 {
659 s->cmdbuf_tail = amdvi_readq(s, AMDVI_MMIO_COMMAND_TAIL)
660 & AMDVI_MMIO_CMDBUF_TAIL_MASK;
661 amdvi_cmdbuf_run(s);
662 }
663
664 static inline void amdvi_handle_excllim_write(AMDVIState *s)
665 {
666 uint64_t val = amdvi_readq(s, AMDVI_MMIO_EXCL_LIMIT);
667 s->excl_limit = (val & AMDVI_MMIO_EXCL_LIMIT_MASK) |
668 AMDVI_MMIO_EXCL_LIMIT_LOW;
669 }
670
671 static inline void amdvi_handle_evtbase_write(AMDVIState *s)
672 {
673 uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_BASE);
674 s->evtlog = val & AMDVI_MMIO_EVTLOG_BASE_MASK;
675 s->evtlog_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_EVTLOG_SIZE_BYTE)
676 & AMDVI_MMIO_EVTLOG_SIZE_MASK);
677 }
678
679 static inline void amdvi_handle_evttail_write(AMDVIState *s)
680 {
681 uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_TAIL);
682 s->evtlog_tail = val & AMDVI_MMIO_EVTLOG_TAIL_MASK;
683 }
684
685 static inline void amdvi_handle_evthead_write(AMDVIState *s)
686 {
687 uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_HEAD);
688 s->evtlog_head = val & AMDVI_MMIO_EVTLOG_HEAD_MASK;
689 }
690
691 static inline void amdvi_handle_pprbase_write(AMDVIState *s)
692 {
693 uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_BASE);
694 s->ppr_log = val & AMDVI_MMIO_PPRLOG_BASE_MASK;
695 s->pprlog_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_PPRLOG_SIZE_BYTE)
696 & AMDVI_MMIO_PPRLOG_SIZE_MASK);
697 }
698
699 static inline void amdvi_handle_pprhead_write(AMDVIState *s)
700 {
701 uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_HEAD);
702 s->pprlog_head = val & AMDVI_MMIO_PPRLOG_HEAD_MASK;
703 }
704
705 static inline void amdvi_handle_pprtail_write(AMDVIState *s)
706 {
707 uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_TAIL);
708 s->pprlog_tail = val & AMDVI_MMIO_PPRLOG_TAIL_MASK;
709 }
710
711 /* FIXME: something might go wrong if System Software writes in chunks
712 * of one byte but linux writes in chunks of 4 bytes so currently it
713 * works correctly with linux but will definitely be busted if software
714 * reads/writes 8 bytes
715 */
716 static void amdvi_mmio_reg_write(AMDVIState *s, unsigned size, uint64_t val,
717 hwaddr addr)
718 {
719 if (size == 2) {
720 amdvi_writew(s, addr, val);
721 } else if (size == 4) {
722 amdvi_writel(s, addr, val);
723 } else if (size == 8) {
724 amdvi_writeq(s, addr, val);
725 }
726 }
727
728 static void amdvi_mmio_write(void *opaque, hwaddr addr, uint64_t val,
729 unsigned size)
730 {
731 AMDVIState *s = opaque;
732 unsigned long offset = addr & 0x07;
733
734 if (addr + size > AMDVI_MMIO_SIZE) {
735 trace_amdvi_mmio_write("error: addr outside region: max ",
736 (uint64_t)AMDVI_MMIO_SIZE, size, val, offset);
737 return;
738 }
739
740 amdvi_mmio_trace(addr, size);
741 switch (addr & ~0x07) {
742 case AMDVI_MMIO_CONTROL:
743 amdvi_mmio_reg_write(s, size, val, addr);
744 amdvi_handle_control_write(s);
745 break;
746 case AMDVI_MMIO_DEVICE_TABLE:
747 amdvi_mmio_reg_write(s, size, val, addr);
748 /* set device table address
749 * This also suffers from inability to tell whether software
750 * is done writing
751 */
752 if (offset || (size == 8)) {
753 amdvi_handle_devtab_write(s);
754 }
755 break;
756 case AMDVI_MMIO_COMMAND_HEAD:
757 amdvi_mmio_reg_write(s, size, val, addr);
758 amdvi_handle_cmdhead_write(s);
759 break;
760 case AMDVI_MMIO_COMMAND_BASE:
761 amdvi_mmio_reg_write(s, size, val, addr);
762 /* FIXME - make sure System Software has finished writing incase
763 * it writes in chucks less than 8 bytes in a robust way.As for
764 * now, this hacks works for the linux driver
765 */
766 if (offset || (size == 8)) {
767 amdvi_handle_cmdbase_write(s);
768 }
769 break;
770 case AMDVI_MMIO_COMMAND_TAIL:
771 amdvi_mmio_reg_write(s, size, val, addr);
772 amdvi_handle_cmdtail_write(s);
773 break;
774 case AMDVI_MMIO_EVENT_BASE:
775 amdvi_mmio_reg_write(s, size, val, addr);
776 amdvi_handle_evtbase_write(s);
777 break;
778 case AMDVI_MMIO_EVENT_HEAD:
779 amdvi_mmio_reg_write(s, size, val, addr);
780 amdvi_handle_evthead_write(s);
781 break;
782 case AMDVI_MMIO_EVENT_TAIL:
783 amdvi_mmio_reg_write(s, size, val, addr);
784 amdvi_handle_evttail_write(s);
785 break;
786 case AMDVI_MMIO_EXCL_LIMIT:
787 amdvi_mmio_reg_write(s, size, val, addr);
788 amdvi_handle_excllim_write(s);
789 break;
790 /* PPR log base - unused for now */
791 case AMDVI_MMIO_PPR_BASE:
792 amdvi_mmio_reg_write(s, size, val, addr);
793 amdvi_handle_pprbase_write(s);
794 break;
795 /* PPR log head - also unused for now */
796 case AMDVI_MMIO_PPR_HEAD:
797 amdvi_mmio_reg_write(s, size, val, addr);
798 amdvi_handle_pprhead_write(s);
799 break;
800 /* PPR log tail - unused for now */
801 case AMDVI_MMIO_PPR_TAIL:
802 amdvi_mmio_reg_write(s, size, val, addr);
803 amdvi_handle_pprtail_write(s);
804 break;
805 }
806 }
807
808 static inline uint64_t amdvi_get_perms(uint64_t entry)
809 {
810 return (entry & (AMDVI_DEV_PERM_READ | AMDVI_DEV_PERM_WRITE)) >>
811 AMDVI_DEV_PERM_SHIFT;
812 }
813
814 /* validate that reserved bits are honoured */
815 static bool amdvi_validate_dte(AMDVIState *s, uint16_t devid,
816 uint64_t *dte)
817 {
818 if ((dte[0] & AMDVI_DTE_LOWER_QUAD_RESERVED)
819 || (dte[1] & AMDVI_DTE_MIDDLE_QUAD_RESERVED)
820 || (dte[2] & AMDVI_DTE_UPPER_QUAD_RESERVED) || dte[3]) {
821 amdvi_log_illegaldevtab_error(s, devid,
822 s->devtab +
823 devid * AMDVI_DEVTAB_ENTRY_SIZE, 0);
824 return false;
825 }
826
827 return true;
828 }
829
830 /* get a device table entry given the devid */
831 static bool amdvi_get_dte(AMDVIState *s, int devid, uint64_t *entry)
832 {
833 uint32_t offset = devid * AMDVI_DEVTAB_ENTRY_SIZE;
834
835 if (dma_memory_read(&address_space_memory, s->devtab + offset, entry,
836 AMDVI_DEVTAB_ENTRY_SIZE)) {
837 trace_amdvi_dte_get_fail(s->devtab, offset);
838 /* log error accessing dte */
839 amdvi_log_devtab_error(s, devid, s->devtab + offset, 0);
840 return false;
841 }
842
843 *entry = le64_to_cpu(*entry);
844 if (!amdvi_validate_dte(s, devid, entry)) {
845 trace_amdvi_invalid_dte(entry[0]);
846 return false;
847 }
848
849 return true;
850 }
851
852 /* get pte translation mode */
853 static inline uint8_t get_pte_translation_mode(uint64_t pte)
854 {
855 return (pte >> AMDVI_DEV_MODE_RSHIFT) & AMDVI_DEV_MODE_MASK;
856 }
857
858 static inline uint64_t pte_override_page_mask(uint64_t pte)
859 {
860 uint8_t page_mask = 12;
861 uint64_t addr = (pte & AMDVI_DEV_PT_ROOT_MASK) ^ AMDVI_DEV_PT_ROOT_MASK;
862 /* find the first zero bit */
863 while (addr & 1) {
864 page_mask++;
865 addr = addr >> 1;
866 }
867
868 return ~((1ULL << page_mask) - 1);
869 }
870
871 static inline uint64_t pte_get_page_mask(uint64_t oldlevel)
872 {
873 return ~((1UL << ((oldlevel * 9) + 3)) - 1);
874 }
875
876 static inline uint64_t amdvi_get_pte_entry(AMDVIState *s, uint64_t pte_addr,
877 uint16_t devid)
878 {
879 uint64_t pte;
880
881 if (dma_memory_read(&address_space_memory, pte_addr, &pte, sizeof(pte))) {
882 trace_amdvi_get_pte_hwerror(pte_addr);
883 amdvi_log_pagetab_error(s, devid, pte_addr, 0);
884 pte = 0;
885 return pte;
886 }
887
888 pte = le64_to_cpu(pte);
889 return pte;
890 }
891
892 static void amdvi_page_walk(AMDVIAddressSpace *as, uint64_t *dte,
893 IOMMUTLBEntry *ret, unsigned perms,
894 hwaddr addr)
895 {
896 unsigned level, present, pte_perms, oldlevel;
897 uint64_t pte = dte[0], pte_addr, page_mask;
898
899 /* make sure the DTE has TV = 1 */
900 if (pte & AMDVI_DEV_TRANSLATION_VALID) {
901 level = get_pte_translation_mode(pte);
902 if (level >= 7) {
903 trace_amdvi_mode_invalid(level, addr);
904 return;
905 }
906 if (level == 0) {
907 goto no_remap;
908 }
909
910 /* we are at the leaf page table or page table encodes a huge page */
911 while (level > 0) {
912 pte_perms = amdvi_get_perms(pte);
913 present = pte & 1;
914 if (!present || perms != (perms & pte_perms)) {
915 amdvi_page_fault(as->iommu_state, as->devfn, addr, perms);
916 trace_amdvi_page_fault(addr);
917 return;
918 }
919
920 /* go to the next lower level */
921 pte_addr = pte & AMDVI_DEV_PT_ROOT_MASK;
922 /* add offset and load pte */
923 pte_addr += ((addr >> (3 + 9 * level)) & 0x1FF) << 3;
924 pte = amdvi_get_pte_entry(as->iommu_state, pte_addr, as->devfn);
925 if (!pte) {
926 return;
927 }
928 oldlevel = level;
929 level = get_pte_translation_mode(pte);
930 if (level == 0x7) {
931 break;
932 }
933 }
934
935 if (level == 0x7) {
936 page_mask = pte_override_page_mask(pte);
937 } else {
938 page_mask = pte_get_page_mask(oldlevel);
939 }
940
941 /* get access permissions from pte */
942 ret->iova = addr & page_mask;
943 ret->translated_addr = (pte & AMDVI_DEV_PT_ROOT_MASK) & page_mask;
944 ret->addr_mask = ~page_mask;
945 ret->perm = amdvi_get_perms(pte);
946 return;
947 }
948 no_remap:
949 ret->iova = addr & AMDVI_PAGE_MASK_4K;
950 ret->translated_addr = addr & AMDVI_PAGE_MASK_4K;
951 ret->addr_mask = ~AMDVI_PAGE_MASK_4K;
952 ret->perm = amdvi_get_perms(pte);
953 }
954
955 static void amdvi_do_translate(AMDVIAddressSpace *as, hwaddr addr,
956 bool is_write, IOMMUTLBEntry *ret)
957 {
958 AMDVIState *s = as->iommu_state;
959 uint16_t devid = PCI_BUILD_BDF(as->bus_num, as->devfn);
960 AMDVIIOTLBEntry *iotlb_entry = amdvi_iotlb_lookup(s, addr, devid);
961 uint64_t entry[4];
962
963 if (iotlb_entry) {
964 trace_amdvi_iotlb_hit(PCI_BUS_NUM(devid), PCI_SLOT(devid),
965 PCI_FUNC(devid), addr, iotlb_entry->translated_addr);
966 ret->iova = addr & ~iotlb_entry->page_mask;
967 ret->translated_addr = iotlb_entry->translated_addr;
968 ret->addr_mask = iotlb_entry->page_mask;
969 ret->perm = iotlb_entry->perms;
970 return;
971 }
972
973 if (!amdvi_get_dte(s, devid, entry)) {
974 return;
975 }
976
977 /* devices with V = 0 are not translated */
978 if (!(entry[0] & AMDVI_DEV_VALID)) {
979 goto out;
980 }
981
982 amdvi_page_walk(as, entry, ret,
983 is_write ? AMDVI_PERM_WRITE : AMDVI_PERM_READ, addr);
984
985 amdvi_update_iotlb(s, devid, addr, *ret,
986 entry[1] & AMDVI_DEV_DOMID_ID_MASK);
987 return;
988
989 out:
990 ret->iova = addr & AMDVI_PAGE_MASK_4K;
991 ret->translated_addr = addr & AMDVI_PAGE_MASK_4K;
992 ret->addr_mask = ~AMDVI_PAGE_MASK_4K;
993 ret->perm = IOMMU_RW;
994 }
995
996 static inline bool amdvi_is_interrupt_addr(hwaddr addr)
997 {
998 return addr >= AMDVI_INT_ADDR_FIRST && addr <= AMDVI_INT_ADDR_LAST;
999 }
1000
1001 static IOMMUTLBEntry amdvi_translate(IOMMUMemoryRegion *iommu, hwaddr addr,
1002 IOMMUAccessFlags flag, int iommu_idx)
1003 {
1004 AMDVIAddressSpace *as = container_of(iommu, AMDVIAddressSpace, iommu);
1005 AMDVIState *s = as->iommu_state;
1006 IOMMUTLBEntry ret = {
1007 .target_as = &address_space_memory,
1008 .iova = addr,
1009 .translated_addr = 0,
1010 .addr_mask = ~(hwaddr)0,
1011 .perm = IOMMU_NONE
1012 };
1013
1014 if (!s->enabled) {
1015 /* AMDVI disabled - corresponds to iommu=off not
1016 * failure to provide any parameter
1017 */
1018 ret.iova = addr & AMDVI_PAGE_MASK_4K;
1019 ret.translated_addr = addr & AMDVI_PAGE_MASK_4K;
1020 ret.addr_mask = ~AMDVI_PAGE_MASK_4K;
1021 ret.perm = IOMMU_RW;
1022 return ret;
1023 } else if (amdvi_is_interrupt_addr(addr)) {
1024 ret.iova = addr & AMDVI_PAGE_MASK_4K;
1025 ret.translated_addr = addr & AMDVI_PAGE_MASK_4K;
1026 ret.addr_mask = ~AMDVI_PAGE_MASK_4K;
1027 ret.perm = IOMMU_WO;
1028 return ret;
1029 }
1030
1031 amdvi_do_translate(as, addr, flag & IOMMU_WO, &ret);
1032 trace_amdvi_translation_result(as->bus_num, PCI_SLOT(as->devfn),
1033 PCI_FUNC(as->devfn), addr, ret.translated_addr);
1034 return ret;
1035 }
1036
1037 static int amdvi_get_irte(AMDVIState *s, MSIMessage *origin, uint64_t *dte,
1038 union irte *irte, uint16_t devid)
1039 {
1040 uint64_t irte_root, offset;
1041
1042 irte_root = dte[2] & AMDVI_IR_PHYS_ADDR_MASK;
1043 offset = (origin->data & AMDVI_IRTE_OFFSET) << 2;
1044
1045 trace_amdvi_ir_irte(irte_root, offset);
1046
1047 if (dma_memory_read(&address_space_memory, irte_root + offset,
1048 irte, sizeof(*irte))) {
1049 trace_amdvi_ir_err("failed to get irte");
1050 return -AMDVI_IR_GET_IRTE;
1051 }
1052
1053 trace_amdvi_ir_irte_val(irte->val);
1054
1055 return 0;
1056 }
1057
1058 static int amdvi_int_remap_legacy(AMDVIState *iommu,
1059 MSIMessage *origin,
1060 MSIMessage *translated,
1061 uint64_t *dte,
1062 X86IOMMUIrq *irq,
1063 uint16_t sid)
1064 {
1065 int ret;
1066 union irte irte;
1067
1068 /* get interrupt remapping table */
1069 ret = amdvi_get_irte(iommu, origin, dte, &irte, sid);
1070 if (ret < 0) {
1071 return ret;
1072 }
1073
1074 if (!irte.fields.valid) {
1075 trace_amdvi_ir_target_abort("RemapEn is disabled");
1076 return -AMDVI_IR_TARGET_ABORT;
1077 }
1078
1079 if (irte.fields.guest_mode) {
1080 error_report_once("guest mode is not zero");
1081 return -AMDVI_IR_ERR;
1082 }
1083
1084 if (irte.fields.int_type > AMDVI_IOAPIC_INT_TYPE_ARBITRATED) {
1085 error_report_once("reserved int_type");
1086 return -AMDVI_IR_ERR;
1087 }
1088
1089 irq->delivery_mode = irte.fields.int_type;
1090 irq->vector = irte.fields.vector;
1091 irq->dest_mode = irte.fields.dm;
1092 irq->redir_hint = irte.fields.rq_eoi;
1093 irq->dest = irte.fields.destination;
1094
1095 return 0;
1096 }
1097
1098 static int amdvi_get_irte_ga(AMDVIState *s, MSIMessage *origin, uint64_t *dte,
1099 struct irte_ga *irte, uint16_t devid)
1100 {
1101 uint64_t irte_root, offset;
1102
1103 irte_root = dte[2] & AMDVI_IR_PHYS_ADDR_MASK;
1104 offset = (origin->data & AMDVI_IRTE_OFFSET) << 4;
1105 trace_amdvi_ir_irte(irte_root, offset);
1106
1107 if (dma_memory_read(&address_space_memory, irte_root + offset,
1108 irte, sizeof(*irte))) {
1109 trace_amdvi_ir_err("failed to get irte_ga");
1110 return -AMDVI_IR_GET_IRTE;
1111 }
1112
1113 trace_amdvi_ir_irte_ga_val(irte->hi.val, irte->lo.val);
1114 return 0;
1115 }
1116
1117 static int amdvi_int_remap_ga(AMDVIState *iommu,
1118 MSIMessage *origin,
1119 MSIMessage *translated,
1120 uint64_t *dte,
1121 X86IOMMUIrq *irq,
1122 uint16_t sid)
1123 {
1124 int ret;
1125 struct irte_ga irte;
1126
1127 /* get interrupt remapping table */
1128 ret = amdvi_get_irte_ga(iommu, origin, dte, &irte, sid);
1129 if (ret < 0) {
1130 return ret;
1131 }
1132
1133 if (!irte.lo.fields_remap.valid) {
1134 trace_amdvi_ir_target_abort("RemapEn is disabled");
1135 return -AMDVI_IR_TARGET_ABORT;
1136 }
1137
1138 if (irte.lo.fields_remap.guest_mode) {
1139 error_report_once("guest mode is not zero");
1140 return -AMDVI_IR_ERR;
1141 }
1142
1143 if (irte.lo.fields_remap.int_type > AMDVI_IOAPIC_INT_TYPE_ARBITRATED) {
1144 error_report_once("reserved int_type is set");
1145 return -AMDVI_IR_ERR;
1146 }
1147
1148 irq->delivery_mode = irte.lo.fields_remap.int_type;
1149 irq->vector = irte.hi.fields.vector;
1150 irq->dest_mode = irte.lo.fields_remap.dm;
1151 irq->redir_hint = irte.lo.fields_remap.rq_eoi;
1152 irq->dest = irte.lo.fields_remap.destination;
1153
1154 return 0;
1155 }
1156
1157 static int __amdvi_int_remap_msi(AMDVIState *iommu,
1158 MSIMessage *origin,
1159 MSIMessage *translated,
1160 uint64_t *dte,
1161 X86IOMMUIrq *irq,
1162 uint16_t sid)
1163 {
1164 int ret;
1165 uint8_t int_ctl;
1166
1167 int_ctl = (dte[2] >> AMDVI_IR_INTCTL_SHIFT) & 3;
1168 trace_amdvi_ir_intctl(int_ctl);
1169
1170 switch (int_ctl) {
1171 case AMDVI_IR_INTCTL_PASS:
1172 memcpy(translated, origin, sizeof(*origin));
1173 return 0;
1174 case AMDVI_IR_INTCTL_REMAP:
1175 break;
1176 case AMDVI_IR_INTCTL_ABORT:
1177 trace_amdvi_ir_target_abort("int_ctl abort");
1178 return -AMDVI_IR_TARGET_ABORT;
1179 default:
1180 trace_amdvi_ir_err("int_ctl reserved");
1181 return -AMDVI_IR_ERR;
1182 }
1183
1184 if (iommu->ga_enabled) {
1185 ret = amdvi_int_remap_ga(iommu, origin, translated, dte, irq, sid);
1186 } else {
1187 ret = amdvi_int_remap_legacy(iommu, origin, translated, dte, irq, sid);
1188 }
1189
1190 return ret;
1191 }
1192
1193 /* Interrupt remapping for MSI/MSI-X entry */
1194 static int amdvi_int_remap_msi(AMDVIState *iommu,
1195 MSIMessage *origin,
1196 MSIMessage *translated,
1197 uint16_t sid)
1198 {
1199 int ret = 0;
1200 uint64_t pass = 0;
1201 uint64_t dte[4] = { 0 };
1202 X86IOMMUIrq irq = { 0 };
1203 uint8_t dest_mode, delivery_mode;
1204
1205 assert(origin && translated);
1206
1207 /*
1208 * When IOMMU is enabled, interrupt remap request will come either from
1209 * IO-APIC or PCI device. If interrupt is from PCI device then it will
1210 * have a valid requester id but if the interrupt is from IO-APIC
1211 * then requester id will be invalid.
1212 */
1213 if (sid == X86_IOMMU_SID_INVALID) {
1214 sid = AMDVI_IOAPIC_SB_DEVID;
1215 }
1216
1217 trace_amdvi_ir_remap_msi_req(origin->address, origin->data, sid);
1218
1219 /* check if device table entry is set before we go further. */
1220 if (!iommu || !iommu->devtab_len) {
1221 memcpy(translated, origin, sizeof(*origin));
1222 goto out;
1223 }
1224
1225 if (!amdvi_get_dte(iommu, sid, dte)) {
1226 return -AMDVI_IR_ERR;
1227 }
1228
1229 /* Check if IR is enabled in DTE */
1230 if (!(dte[2] & AMDVI_IR_REMAP_ENABLE)) {
1231 memcpy(translated, origin, sizeof(*origin));
1232 goto out;
1233 }
1234
1235 /* validate that we are configure with intremap=on */
1236 if (!X86_IOMMU_DEVICE(iommu)->intr_supported) {
1237 trace_amdvi_err("Interrupt remapping is enabled in the guest but "
1238 "not in the host. Use intremap=on to enable interrupt "
1239 "remapping in amd-iommu.");
1240 return -AMDVI_IR_ERR;
1241 }
1242
1243 if (origin->address & AMDVI_MSI_ADDR_HI_MASK) {
1244 trace_amdvi_err("MSI address high 32 bits non-zero when "
1245 "Interrupt Remapping enabled.");
1246 return -AMDVI_IR_ERR;
1247 }
1248
1249 if ((origin->address & AMDVI_MSI_ADDR_LO_MASK) != APIC_DEFAULT_ADDRESS) {
1250 trace_amdvi_err("MSI is not from IOAPIC.");
1251 return -AMDVI_IR_ERR;
1252 }
1253
1254 /*
1255 * The MSI data register [10:8] are used to get the upstream interrupt type.
1256 *
1257 * See MSI/MSI-X format:
1258 * https://pdfs.semanticscholar.org/presentation/9420/c279e942eca568157711ef5c92b800c40a79.pdf
1259 * (page 5)
1260 */
1261 delivery_mode = (origin->data >> MSI_DATA_DELIVERY_MODE_SHIFT) & 7;
1262
1263 switch (delivery_mode) {
1264 case AMDVI_IOAPIC_INT_TYPE_FIXED:
1265 case AMDVI_IOAPIC_INT_TYPE_ARBITRATED:
1266 trace_amdvi_ir_delivery_mode("fixed/arbitrated");
1267 ret = __amdvi_int_remap_msi(iommu, origin, translated, dte, &irq, sid);
1268 if (ret < 0) {
1269 goto remap_fail;
1270 } else {
1271 /* Translate IRQ to MSI messages */
1272 x86_iommu_irq_to_msi_message(&irq, translated);
1273 goto out;
1274 }
1275 break;
1276 case AMDVI_IOAPIC_INT_TYPE_SMI:
1277 error_report("SMI is not supported!");
1278 ret = -AMDVI_IR_ERR;
1279 break;
1280 case AMDVI_IOAPIC_INT_TYPE_NMI:
1281 pass = dte[3] & AMDVI_DEV_NMI_PASS_MASK;
1282 trace_amdvi_ir_delivery_mode("nmi");
1283 break;
1284 case AMDVI_IOAPIC_INT_TYPE_INIT:
1285 pass = dte[3] & AMDVI_DEV_INT_PASS_MASK;
1286 trace_amdvi_ir_delivery_mode("init");
1287 break;
1288 case AMDVI_IOAPIC_INT_TYPE_EINT:
1289 pass = dte[3] & AMDVI_DEV_EINT_PASS_MASK;
1290 trace_amdvi_ir_delivery_mode("eint");
1291 break;
1292 default:
1293 trace_amdvi_ir_delivery_mode("unsupported delivery_mode");
1294 ret = -AMDVI_IR_ERR;
1295 break;
1296 }
1297
1298 if (ret < 0) {
1299 goto remap_fail;
1300 }
1301
1302 /*
1303 * The MSI address register bit[2] is used to get the destination
1304 * mode. The dest_mode 1 is valid for fixed and arbitrated interrupts
1305 * only.
1306 */
1307 dest_mode = (origin->address >> MSI_ADDR_DEST_MODE_SHIFT) & 1;
1308 if (dest_mode) {
1309 trace_amdvi_ir_err("invalid dest_mode");
1310 ret = -AMDVI_IR_ERR;
1311 goto remap_fail;
1312 }
1313
1314 if (pass) {
1315 memcpy(translated, origin, sizeof(*origin));
1316 } else {
1317 trace_amdvi_ir_err("passthrough is not enabled");
1318 ret = -AMDVI_IR_ERR;
1319 goto remap_fail;
1320 }
1321
1322 out:
1323 trace_amdvi_ir_remap_msi(origin->address, origin->data,
1324 translated->address, translated->data);
1325 return 0;
1326
1327 remap_fail:
1328 return ret;
1329 }
1330
1331 static int amdvi_int_remap(X86IOMMUState *iommu,
1332 MSIMessage *origin,
1333 MSIMessage *translated,
1334 uint16_t sid)
1335 {
1336 return amdvi_int_remap_msi(AMD_IOMMU_DEVICE(iommu), origin,
1337 translated, sid);
1338 }
1339
1340 static MemTxResult amdvi_mem_ir_write(void *opaque, hwaddr addr,
1341 uint64_t value, unsigned size,
1342 MemTxAttrs attrs)
1343 {
1344 int ret;
1345 MSIMessage from = { 0, 0 }, to = { 0, 0 };
1346 uint16_t sid = AMDVI_IOAPIC_SB_DEVID;
1347
1348 from.address = (uint64_t) addr + AMDVI_INT_ADDR_FIRST;
1349 from.data = (uint32_t) value;
1350
1351 trace_amdvi_mem_ir_write_req(addr, value, size);
1352
1353 if (!attrs.unspecified) {
1354 /* We have explicit Source ID */
1355 sid = attrs.requester_id;
1356 }
1357
1358 ret = amdvi_int_remap_msi(opaque, &from, &to, sid);
1359 if (ret < 0) {
1360 /* TODO: log the event using IOMMU log event interface */
1361 error_report_once("failed to remap interrupt from devid 0x%x", sid);
1362 return MEMTX_ERROR;
1363 }
1364
1365 apic_get_class()->send_msi(&to);
1366
1367 trace_amdvi_mem_ir_write(to.address, to.data);
1368 return MEMTX_OK;
1369 }
1370
1371 static MemTxResult amdvi_mem_ir_read(void *opaque, hwaddr addr,
1372 uint64_t *data, unsigned size,
1373 MemTxAttrs attrs)
1374 {
1375 return MEMTX_OK;
1376 }
1377
1378 static const MemoryRegionOps amdvi_ir_ops = {
1379 .read_with_attrs = amdvi_mem_ir_read,
1380 .write_with_attrs = amdvi_mem_ir_write,
1381 .endianness = DEVICE_LITTLE_ENDIAN,
1382 .impl = {
1383 .min_access_size = 4,
1384 .max_access_size = 4,
1385 },
1386 .valid = {
1387 .min_access_size = 4,
1388 .max_access_size = 4,
1389 }
1390 };
1391
1392 static AddressSpace *amdvi_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
1393 {
1394 char name[128];
1395 AMDVIState *s = opaque;
1396 AMDVIAddressSpace **iommu_as, *amdvi_dev_as;
1397 int bus_num = pci_bus_num(bus);
1398
1399 iommu_as = s->address_spaces[bus_num];
1400
1401 /* allocate memory during the first run */
1402 if (!iommu_as) {
1403 iommu_as = g_malloc0(sizeof(AMDVIAddressSpace *) * PCI_DEVFN_MAX);
1404 s->address_spaces[bus_num] = iommu_as;
1405 }
1406
1407 /* set up AMD-Vi region */
1408 if (!iommu_as[devfn]) {
1409 snprintf(name, sizeof(name), "amd_iommu_devfn_%d", devfn);
1410
1411 iommu_as[devfn] = g_malloc0(sizeof(AMDVIAddressSpace));
1412 iommu_as[devfn]->bus_num = (uint8_t)bus_num;
1413 iommu_as[devfn]->devfn = (uint8_t)devfn;
1414 iommu_as[devfn]->iommu_state = s;
1415
1416 amdvi_dev_as = iommu_as[devfn];
1417
1418 /*
1419 * Memory region relationships looks like (Address range shows
1420 * only lower 32 bits to make it short in length...):
1421 *
1422 * |-----------------+-------------------+----------|
1423 * | Name | Address range | Priority |
1424 * |-----------------+-------------------+----------+
1425 * | amdvi_root | 00000000-ffffffff | 0 |
1426 * | amdvi_iommu | 00000000-ffffffff | 1 |
1427 * | amdvi_iommu_ir | fee00000-feefffff | 64 |
1428 * |-----------------+-------------------+----------|
1429 */
1430 memory_region_init_iommu(&amdvi_dev_as->iommu,
1431 sizeof(amdvi_dev_as->iommu),
1432 TYPE_AMD_IOMMU_MEMORY_REGION,
1433 OBJECT(s),
1434 "amd_iommu", UINT64_MAX);
1435 memory_region_init(&amdvi_dev_as->root, OBJECT(s),
1436 "amdvi_root", UINT64_MAX);
1437 address_space_init(&amdvi_dev_as->as, &amdvi_dev_as->root, name);
1438 memory_region_init_io(&amdvi_dev_as->iommu_ir, OBJECT(s),
1439 &amdvi_ir_ops, s, "amd_iommu_ir",
1440 AMDVI_INT_ADDR_SIZE);
1441 memory_region_add_subregion_overlap(&amdvi_dev_as->root,
1442 AMDVI_INT_ADDR_FIRST,
1443 &amdvi_dev_as->iommu_ir,
1444 64);
1445 memory_region_add_subregion_overlap(&amdvi_dev_as->root, 0,
1446 MEMORY_REGION(&amdvi_dev_as->iommu),
1447 1);
1448 }
1449 return &iommu_as[devfn]->as;
1450 }
1451
1452 static const MemoryRegionOps mmio_mem_ops = {
1453 .read = amdvi_mmio_read,
1454 .write = amdvi_mmio_write,
1455 .endianness = DEVICE_LITTLE_ENDIAN,
1456 .impl = {
1457 .min_access_size = 1,
1458 .max_access_size = 8,
1459 .unaligned = false,
1460 },
1461 .valid = {
1462 .min_access_size = 1,
1463 .max_access_size = 8,
1464 }
1465 };
1466
1467 static void amdvi_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu,
1468 IOMMUNotifierFlag old,
1469 IOMMUNotifierFlag new)
1470 {
1471 AMDVIAddressSpace *as = container_of(iommu, AMDVIAddressSpace, iommu);
1472
1473 if (new & IOMMU_NOTIFIER_MAP) {
1474 error_report("device %02x.%02x.%x requires iommu notifier which is not "
1475 "currently supported", as->bus_num, PCI_SLOT(as->devfn),
1476 PCI_FUNC(as->devfn));
1477 exit(1);
1478 }
1479 }
1480
1481 static void amdvi_init(AMDVIState *s)
1482 {
1483 amdvi_iotlb_reset(s);
1484
1485 s->devtab_len = 0;
1486 s->cmdbuf_len = 0;
1487 s->cmdbuf_head = 0;
1488 s->cmdbuf_tail = 0;
1489 s->evtlog_head = 0;
1490 s->evtlog_tail = 0;
1491 s->excl_enabled = false;
1492 s->excl_allow = false;
1493 s->mmio_enabled = false;
1494 s->enabled = false;
1495 s->ats_enabled = false;
1496 s->cmdbuf_enabled = false;
1497
1498 /* reset MMIO */
1499 memset(s->mmior, 0, AMDVI_MMIO_SIZE);
1500 amdvi_set_quad(s, AMDVI_MMIO_EXT_FEATURES, AMDVI_EXT_FEATURES,
1501 0xffffffffffffffef, 0);
1502 amdvi_set_quad(s, AMDVI_MMIO_STATUS, 0, 0x98, 0x67);
1503
1504 /* reset device ident */
1505 pci_config_set_vendor_id(s->pci.dev.config, PCI_VENDOR_ID_AMD);
1506 pci_config_set_prog_interface(s->pci.dev.config, 00);
1507 pci_config_set_device_id(s->pci.dev.config, s->devid);
1508 pci_config_set_class(s->pci.dev.config, 0x0806);
1509
1510 /* reset AMDVI specific capabilities, all r/o */
1511 pci_set_long(s->pci.dev.config + s->capab_offset, AMDVI_CAPAB_FEATURES);
1512 pci_set_long(s->pci.dev.config + s->capab_offset + AMDVI_CAPAB_BAR_LOW,
1513 s->mmio.addr & ~(0xffff0000));
1514 pci_set_long(s->pci.dev.config + s->capab_offset + AMDVI_CAPAB_BAR_HIGH,
1515 (s->mmio.addr & ~(0xffff)) >> 16);
1516 pci_set_long(s->pci.dev.config + s->capab_offset + AMDVI_CAPAB_RANGE,
1517 0xff000000);
1518 pci_set_long(s->pci.dev.config + s->capab_offset + AMDVI_CAPAB_MISC, 0);
1519 pci_set_long(s->pci.dev.config + s->capab_offset + AMDVI_CAPAB_MISC,
1520 AMDVI_MAX_PH_ADDR | AMDVI_MAX_GVA_ADDR | AMDVI_MAX_VA_ADDR);
1521 }
1522
1523 static void amdvi_reset(DeviceState *dev)
1524 {
1525 AMDVIState *s = AMD_IOMMU_DEVICE(dev);
1526
1527 msi_reset(&s->pci.dev);
1528 amdvi_init(s);
1529 }
1530
1531 static void amdvi_realize(DeviceState *dev, Error **err)
1532 {
1533 int ret = 0;
1534 AMDVIState *s = AMD_IOMMU_DEVICE(dev);
1535 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(dev);
1536 MachineState *ms = MACHINE(qdev_get_machine());
1537 PCMachineState *pcms = PC_MACHINE(ms);
1538 PCIBus *bus = pcms->bus;
1539
1540 s->iotlb = g_hash_table_new_full(amdvi_uint64_hash,
1541 amdvi_uint64_equal, g_free, g_free);
1542
1543 /* This device should take care of IOMMU PCI properties */
1544 x86_iommu->type = TYPE_AMD;
1545 qdev_set_parent_bus(DEVICE(&s->pci), &bus->qbus);
1546 object_property_set_bool(OBJECT(&s->pci), true, "realized", err);
1547 ret = pci_add_capability(&s->pci.dev, AMDVI_CAPAB_ID_SEC, 0,
1548 AMDVI_CAPAB_SIZE, err);
1549 if (ret < 0) {
1550 return;
1551 }
1552 s->capab_offset = ret;
1553
1554 ret = pci_add_capability(&s->pci.dev, PCI_CAP_ID_MSI, 0,
1555 AMDVI_CAPAB_REG_SIZE, err);
1556 if (ret < 0) {
1557 return;
1558 }
1559 ret = pci_add_capability(&s->pci.dev, PCI_CAP_ID_HT, 0,
1560 AMDVI_CAPAB_REG_SIZE, err);
1561 if (ret < 0) {
1562 return;
1563 }
1564
1565 /* Pseudo address space under root PCI bus. */
1566 pcms->ioapic_as = amdvi_host_dma_iommu(bus, s, AMDVI_IOAPIC_SB_DEVID);
1567
1568 /* set up MMIO */
1569 memory_region_init_io(&s->mmio, OBJECT(s), &mmio_mem_ops, s, "amdvi-mmio",
1570 AMDVI_MMIO_SIZE);
1571
1572 sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->mmio);
1573 sysbus_mmio_map(SYS_BUS_DEVICE(s), 0, AMDVI_BASE_ADDR);
1574 pci_setup_iommu(bus, amdvi_host_dma_iommu, s);
1575 s->devid = object_property_get_int(OBJECT(&s->pci), "addr", err);
1576 msi_init(&s->pci.dev, 0, 1, true, false, err);
1577 amdvi_init(s);
1578 }
1579
1580 static const VMStateDescription vmstate_amdvi = {
1581 .name = "amd-iommu",
1582 .unmigratable = 1
1583 };
1584
1585 static void amdvi_instance_init(Object *klass)
1586 {
1587 AMDVIState *s = AMD_IOMMU_DEVICE(klass);
1588
1589 object_initialize(&s->pci, sizeof(s->pci), TYPE_AMD_IOMMU_PCI);
1590 }
1591
1592 static void amdvi_class_init(ObjectClass *klass, void* data)
1593 {
1594 DeviceClass *dc = DEVICE_CLASS(klass);
1595 X86IOMMUClass *dc_class = X86_IOMMU_CLASS(klass);
1596
1597 dc->reset = amdvi_reset;
1598 dc->vmsd = &vmstate_amdvi;
1599 dc->hotpluggable = false;
1600 dc_class->realize = amdvi_realize;
1601 dc_class->int_remap = amdvi_int_remap;
1602 /* Supported by the pc-q35-* machine types */
1603 dc->user_creatable = true;
1604 }
1605
1606 static const TypeInfo amdvi = {
1607 .name = TYPE_AMD_IOMMU_DEVICE,
1608 .parent = TYPE_X86_IOMMU_DEVICE,
1609 .instance_size = sizeof(AMDVIState),
1610 .instance_init = amdvi_instance_init,
1611 .class_init = amdvi_class_init
1612 };
1613
1614 static const TypeInfo amdviPCI = {
1615 .name = "AMDVI-PCI",
1616 .parent = TYPE_PCI_DEVICE,
1617 .instance_size = sizeof(AMDVIPCIState),
1618 .interfaces = (InterfaceInfo[]) {
1619 { INTERFACE_CONVENTIONAL_PCI_DEVICE },
1620 { },
1621 },
1622 };
1623
1624 static void amdvi_iommu_memory_region_class_init(ObjectClass *klass, void *data)
1625 {
1626 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
1627
1628 imrc->translate = amdvi_translate;
1629 imrc->notify_flag_changed = amdvi_iommu_notify_flag_changed;
1630 }
1631
1632 static const TypeInfo amdvi_iommu_memory_region_info = {
1633 .parent = TYPE_IOMMU_MEMORY_REGION,
1634 .name = TYPE_AMD_IOMMU_MEMORY_REGION,
1635 .class_init = amdvi_iommu_memory_region_class_init,
1636 };
1637
1638 static void amdviPCI_register_types(void)
1639 {
1640 type_register_static(&amdviPCI);
1641 type_register_static(&amdvi);
1642 type_register_static(&amdvi_iommu_memory_region_info);
1643 }
1644
1645 type_init(amdviPCI_register_types);
QEmu