2 * QEMU sPAPR IOMMU (TCE) code
4 * Copyright (c) 2010 David Gibson, IBM Corporation <dwg@au1.ibm.com>
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "qemu/osdep.h"
20 #include "qemu/error-report.h"
23 #include "sysemu/kvm.h"
26 #include "sysemu/dma.h"
27 #include "exec/address-spaces.h"
30 #include "hw/ppc/spapr.h"
31 #include "hw/ppc/spapr_vio.h"
42 #define IOMMU_PAGE_SIZE(shift) (1ULL << (shift))
43 #define IOMMU_PAGE_MASK(shift) (~(IOMMU_PAGE_SIZE(shift) - 1))
45 static QLIST_HEAD(spapr_tce_tables
, sPAPRTCETable
) spapr_tce_tables
;
47 sPAPRTCETable
*spapr_tce_find_by_liobn(target_ulong liobn
)
51 if (liobn
& 0xFFFFFFFF00000000ULL
) {
52 hcall_dprintf("Request for out-of-bounds LIOBN 0x" TARGET_FMT_lx
"\n",
57 QLIST_FOREACH(tcet
, &spapr_tce_tables
, list
) {
58 if (tcet
->liobn
== (uint32_t)liobn
) {
66 static IOMMUAccessFlags
spapr_tce_iommu_access_flags(uint64_t tce
)
68 switch (tce
& SPAPR_TCE_RW
) {
75 default: /* SPAPR_TCE_RW */
80 static uint64_t *spapr_tce_alloc_table(uint32_t liobn
,
86 uint64_t *table
= NULL
;
87 uint64_t window_size
= (uint64_t)nb_table
<< page_shift
;
89 if (kvm_enabled() && !(window_size
>> 32)) {
90 table
= kvmppc_create_spapr_tce(liobn
, window_size
, fd
, need_vfio
);
95 table
= g_malloc0(nb_table
* sizeof(uint64_t));
98 trace_spapr_iommu_new_table(liobn
, table
, *fd
);
103 static void spapr_tce_free_table(uint64_t *table
, int fd
, uint32_t nb_table
)
105 if (!kvm_enabled() ||
106 (kvmppc_remove_spapr_tce(table
, fd
, nb_table
) != 0)) {
111 /* Called from RCU critical section */
112 static IOMMUTLBEntry
spapr_tce_translate_iommu(MemoryRegion
*iommu
, hwaddr addr
,
115 sPAPRTCETable
*tcet
= container_of(iommu
, sPAPRTCETable
, iommu
);
117 IOMMUTLBEntry ret
= {
118 .target_as
= &address_space_memory
,
120 .translated_addr
= 0,
121 .addr_mask
= ~(hwaddr
)0,
125 if ((addr
>> tcet
->page_shift
) < tcet
->nb_table
) {
126 /* Check if we are in bound */
127 hwaddr page_mask
= IOMMU_PAGE_MASK(tcet
->page_shift
);
129 tce
= tcet
->table
[addr
>> tcet
->page_shift
];
130 ret
.iova
= addr
& page_mask
;
131 ret
.translated_addr
= tce
& page_mask
;
132 ret
.addr_mask
= ~page_mask
;
133 ret
.perm
= spapr_tce_iommu_access_flags(tce
);
135 trace_spapr_iommu_xlate(tcet
->liobn
, addr
, ret
.iova
, ret
.perm
,
141 static void spapr_tce_table_pre_save(void *opaque
)
143 sPAPRTCETable
*tcet
= SPAPR_TCE_TABLE(opaque
);
145 tcet
->mig_table
= tcet
->table
;
146 tcet
->mig_nb_table
= tcet
->nb_table
;
148 trace_spapr_iommu_pre_save(tcet
->liobn
, tcet
->mig_nb_table
,
149 tcet
->bus_offset
, tcet
->page_shift
);
152 static uint64_t spapr_tce_get_min_page_size(MemoryRegion
*iommu
)
154 sPAPRTCETable
*tcet
= container_of(iommu
, sPAPRTCETable
, iommu
);
156 return 1ULL << tcet
->page_shift
;
159 static void spapr_tce_notify_started(MemoryRegion
*iommu
)
161 spapr_tce_set_need_vfio(container_of(iommu
, sPAPRTCETable
, iommu
), true);
164 static void spapr_tce_notify_stopped(MemoryRegion
*iommu
)
166 spapr_tce_set_need_vfio(container_of(iommu
, sPAPRTCETable
, iommu
), false);
169 static int spapr_tce_table_post_load(void *opaque
, int version_id
)
171 sPAPRTCETable
*tcet
= SPAPR_TCE_TABLE(opaque
);
172 uint32_t old_nb_table
= tcet
->nb_table
;
173 uint64_t old_bus_offset
= tcet
->bus_offset
;
174 uint32_t old_page_shift
= tcet
->page_shift
;
177 spapr_vio_set_bypass(tcet
->vdev
, tcet
->bypass
);
180 if (tcet
->mig_nb_table
!= tcet
->nb_table
) {
181 spapr_tce_table_disable(tcet
);
184 if (tcet
->mig_nb_table
) {
185 if (!tcet
->nb_table
) {
186 spapr_tce_table_enable(tcet
, old_page_shift
, old_bus_offset
,
190 memcpy(tcet
->table
, tcet
->mig_table
,
191 tcet
->nb_table
* sizeof(tcet
->table
[0]));
193 free(tcet
->mig_table
);
194 tcet
->mig_table
= NULL
;
197 trace_spapr_iommu_post_load(tcet
->liobn
, old_nb_table
, tcet
->nb_table
,
198 tcet
->bus_offset
, tcet
->page_shift
);
203 static bool spapr_tce_table_ex_needed(void *opaque
)
205 sPAPRTCETable
*tcet
= opaque
;
207 return tcet
->bus_offset
|| tcet
->page_shift
!= 0xC;
210 static const VMStateDescription vmstate_spapr_tce_table_ex
= {
211 .name
= "spapr_iommu_ex",
213 .minimum_version_id
= 1,
214 .needed
= spapr_tce_table_ex_needed
,
215 .fields
= (VMStateField
[]) {
216 VMSTATE_UINT64(bus_offset
, sPAPRTCETable
),
217 VMSTATE_UINT32(page_shift
, sPAPRTCETable
),
218 VMSTATE_END_OF_LIST()
222 static const VMStateDescription vmstate_spapr_tce_table
= {
223 .name
= "spapr_iommu",
225 .minimum_version_id
= 2,
226 .pre_save
= spapr_tce_table_pre_save
,
227 .post_load
= spapr_tce_table_post_load
,
228 .fields
= (VMStateField
[]) {
230 VMSTATE_UINT32_EQUAL(liobn
, sPAPRTCETable
),
233 VMSTATE_UINT32(mig_nb_table
, sPAPRTCETable
),
234 VMSTATE_BOOL(bypass
, sPAPRTCETable
),
235 VMSTATE_VARRAY_UINT32_ALLOC(mig_table
, sPAPRTCETable
, mig_nb_table
, 0,
236 vmstate_info_uint64
, uint64_t),
238 VMSTATE_END_OF_LIST()
240 .subsections
= (const VMStateDescription
*[]) {
241 &vmstate_spapr_tce_table_ex
,
246 static MemoryRegionIOMMUOps spapr_iommu_ops
= {
247 .translate
= spapr_tce_translate_iommu
,
248 .get_min_page_size
= spapr_tce_get_min_page_size
,
249 .notify_started
= spapr_tce_notify_started
,
250 .notify_stopped
= spapr_tce_notify_stopped
,
253 static int spapr_tce_table_realize(DeviceState
*dev
)
255 sPAPRTCETable
*tcet
= SPAPR_TCE_TABLE(dev
);
256 Object
*tcetobj
= OBJECT(tcet
);
260 tcet
->need_vfio
= false;
261 snprintf(tmp
, sizeof(tmp
), "tce-root-%x", tcet
->liobn
);
262 memory_region_init(&tcet
->root
, tcetobj
, tmp
, UINT64_MAX
);
264 snprintf(tmp
, sizeof(tmp
), "tce-iommu-%x", tcet
->liobn
);
265 memory_region_init_iommu(&tcet
->iommu
, tcetobj
, &spapr_iommu_ops
, tmp
, 0);
267 QLIST_INSERT_HEAD(&spapr_tce_tables
, tcet
, list
);
269 vmstate_register(DEVICE(tcet
), tcet
->liobn
, &vmstate_spapr_tce_table
,
275 void spapr_tce_set_need_vfio(sPAPRTCETable
*tcet
, bool need_vfio
)
277 size_t table_size
= tcet
->nb_table
* sizeof(uint64_t);
280 if (need_vfio
== tcet
->need_vfio
) {
286 /* FIXME: We don't support transition back to KVM accelerated
291 tcet
->need_vfio
= true;
294 /* Table is already in userspace, nothing to be do */
298 newtable
= g_malloc(table_size
);
299 memcpy(newtable
, tcet
->table
, table_size
);
301 kvmppc_remove_spapr_tce(tcet
->table
, tcet
->fd
, tcet
->nb_table
);
304 tcet
->table
= newtable
;
307 sPAPRTCETable
*spapr_tce_new_table(DeviceState
*owner
, uint32_t liobn
)
312 if (spapr_tce_find_by_liobn(liobn
)) {
313 fprintf(stderr
, "Attempted to create TCE table with duplicate"
314 " LIOBN 0x%x\n", liobn
);
318 tcet
= SPAPR_TCE_TABLE(object_new(TYPE_SPAPR_TCE_TABLE
));
321 snprintf(tmp
, sizeof(tmp
), "tce-table-%x", liobn
);
322 object_property_add_child(OBJECT(owner
), tmp
, OBJECT(tcet
), NULL
);
324 object_property_set_bool(OBJECT(tcet
), true, "realized", NULL
);
329 void spapr_tce_table_enable(sPAPRTCETable
*tcet
,
330 uint32_t page_shift
, uint64_t bus_offset
,
333 if (tcet
->nb_table
) {
334 error_report("Warning: trying to enable already enabled TCE table");
338 tcet
->bus_offset
= bus_offset
;
339 tcet
->page_shift
= page_shift
;
340 tcet
->nb_table
= nb_table
;
341 tcet
->table
= spapr_tce_alloc_table(tcet
->liobn
,
347 memory_region_set_size(&tcet
->iommu
,
348 (uint64_t)tcet
->nb_table
<< tcet
->page_shift
);
349 memory_region_add_subregion(&tcet
->root
, tcet
->bus_offset
, &tcet
->iommu
);
352 void spapr_tce_table_disable(sPAPRTCETable
*tcet
)
354 if (!tcet
->nb_table
) {
358 memory_region_del_subregion(&tcet
->root
, &tcet
->iommu
);
359 memory_region_set_size(&tcet
->iommu
, 0);
361 spapr_tce_free_table(tcet
->table
, tcet
->fd
, tcet
->nb_table
);
364 tcet
->bus_offset
= 0;
365 tcet
->page_shift
= 0;
369 static void spapr_tce_table_unrealize(DeviceState
*dev
, Error
**errp
)
371 sPAPRTCETable
*tcet
= SPAPR_TCE_TABLE(dev
);
373 QLIST_REMOVE(tcet
, list
);
375 spapr_tce_table_disable(tcet
);
378 MemoryRegion
*spapr_tce_get_iommu(sPAPRTCETable
*tcet
)
383 static void spapr_tce_reset(DeviceState
*dev
)
385 sPAPRTCETable
*tcet
= SPAPR_TCE_TABLE(dev
);
386 size_t table_size
= tcet
->nb_table
* sizeof(uint64_t);
388 memset(tcet
->table
, 0, table_size
);
391 static target_ulong
put_tce_emu(sPAPRTCETable
*tcet
, target_ulong ioba
,
395 hwaddr page_mask
= IOMMU_PAGE_MASK(tcet
->page_shift
);
396 unsigned long index
= (ioba
- tcet
->bus_offset
) >> tcet
->page_shift
;
398 if (index
>= tcet
->nb_table
) {
399 hcall_dprintf("spapr_vio_put_tce on out-of-bounds IOBA 0x"
400 TARGET_FMT_lx
"\n", ioba
);
404 tcet
->table
[index
] = tce
;
406 entry
.target_as
= &address_space_memory
,
407 entry
.iova
= (ioba
- tcet
->bus_offset
) & page_mask
;
408 entry
.translated_addr
= tce
& page_mask
;
409 entry
.addr_mask
= ~page_mask
;
410 entry
.perm
= spapr_tce_iommu_access_flags(tce
);
411 memory_region_notify_iommu(&tcet
->iommu
, entry
);
416 static target_ulong
h_put_tce_indirect(PowerPCCPU
*cpu
,
417 sPAPRMachineState
*spapr
,
418 target_ulong opcode
, target_ulong
*args
)
421 target_ulong liobn
= args
[0];
422 target_ulong ioba
= args
[1];
423 target_ulong ioba1
= ioba
;
424 target_ulong tce_list
= args
[2];
425 target_ulong npages
= args
[3];
426 target_ulong ret
= H_PARAMETER
, tce
= 0;
427 sPAPRTCETable
*tcet
= spapr_tce_find_by_liobn(liobn
);
428 CPUState
*cs
= CPU(cpu
);
429 hwaddr page_mask
, page_size
;
435 if ((npages
> 512) || (tce_list
& SPAPR_TCE_PAGE_MASK
)) {
439 page_mask
= IOMMU_PAGE_MASK(tcet
->page_shift
);
440 page_size
= IOMMU_PAGE_SIZE(tcet
->page_shift
);
443 for (i
= 0; i
< npages
; ++i
, ioba
+= page_size
) {
444 tce
= ldq_be_phys(cs
->as
, tce_list
+ i
* sizeof(target_ulong
));
446 ret
= put_tce_emu(tcet
, ioba
, tce
);
452 /* Trace last successful or the first problematic entry */
454 if (SPAPR_IS_PCI_LIOBN(liobn
)) {
455 trace_spapr_iommu_pci_indirect(liobn
, ioba1
, tce_list
, i
, tce
, ret
);
457 trace_spapr_iommu_indirect(liobn
, ioba1
, tce_list
, i
, tce
, ret
);
462 static target_ulong
h_stuff_tce(PowerPCCPU
*cpu
, sPAPRMachineState
*spapr
,
463 target_ulong opcode
, target_ulong
*args
)
466 target_ulong liobn
= args
[0];
467 target_ulong ioba
= args
[1];
468 target_ulong tce_value
= args
[2];
469 target_ulong npages
= args
[3];
470 target_ulong ret
= H_PARAMETER
;
471 sPAPRTCETable
*tcet
= spapr_tce_find_by_liobn(liobn
);
472 hwaddr page_mask
, page_size
;
478 if (npages
> tcet
->nb_table
) {
482 page_mask
= IOMMU_PAGE_MASK(tcet
->page_shift
);
483 page_size
= IOMMU_PAGE_SIZE(tcet
->page_shift
);
486 for (i
= 0; i
< npages
; ++i
, ioba
+= page_size
) {
487 ret
= put_tce_emu(tcet
, ioba
, tce_value
);
492 if (SPAPR_IS_PCI_LIOBN(liobn
)) {
493 trace_spapr_iommu_pci_stuff(liobn
, ioba
, tce_value
, npages
, ret
);
495 trace_spapr_iommu_stuff(liobn
, ioba
, tce_value
, npages
, ret
);
501 static target_ulong
h_put_tce(PowerPCCPU
*cpu
, sPAPRMachineState
*spapr
,
502 target_ulong opcode
, target_ulong
*args
)
504 target_ulong liobn
= args
[0];
505 target_ulong ioba
= args
[1];
506 target_ulong tce
= args
[2];
507 target_ulong ret
= H_PARAMETER
;
508 sPAPRTCETable
*tcet
= spapr_tce_find_by_liobn(liobn
);
511 hwaddr page_mask
= IOMMU_PAGE_MASK(tcet
->page_shift
);
515 ret
= put_tce_emu(tcet
, ioba
, tce
);
517 if (SPAPR_IS_PCI_LIOBN(liobn
)) {
518 trace_spapr_iommu_pci_put(liobn
, ioba
, tce
, ret
);
520 trace_spapr_iommu_put(liobn
, ioba
, tce
, ret
);
526 static target_ulong
get_tce_emu(sPAPRTCETable
*tcet
, target_ulong ioba
,
529 unsigned long index
= (ioba
- tcet
->bus_offset
) >> tcet
->page_shift
;
531 if (index
>= tcet
->nb_table
) {
532 hcall_dprintf("spapr_iommu_get_tce on out-of-bounds IOBA 0x"
533 TARGET_FMT_lx
"\n", ioba
);
537 *tce
= tcet
->table
[index
];
542 static target_ulong
h_get_tce(PowerPCCPU
*cpu
, sPAPRMachineState
*spapr
,
543 target_ulong opcode
, target_ulong
*args
)
545 target_ulong liobn
= args
[0];
546 target_ulong ioba
= args
[1];
547 target_ulong tce
= 0;
548 target_ulong ret
= H_PARAMETER
;
549 sPAPRTCETable
*tcet
= spapr_tce_find_by_liobn(liobn
);
552 hwaddr page_mask
= IOMMU_PAGE_MASK(tcet
->page_shift
);
556 ret
= get_tce_emu(tcet
, ioba
, &tce
);
561 if (SPAPR_IS_PCI_LIOBN(liobn
)) {
562 trace_spapr_iommu_pci_get(liobn
, ioba
, ret
, tce
);
564 trace_spapr_iommu_get(liobn
, ioba
, ret
, tce
);
570 int spapr_dma_dt(void *fdt
, int node_off
, const char *propname
,
571 uint32_t liobn
, uint64_t window
, uint32_t size
)
573 uint32_t dma_prop
[5];
576 dma_prop
[0] = cpu_to_be32(liobn
);
577 dma_prop
[1] = cpu_to_be32(window
>> 32);
578 dma_prop
[2] = cpu_to_be32(window
& 0xFFFFFFFF);
579 dma_prop
[3] = 0; /* window size is 32 bits */
580 dma_prop
[4] = cpu_to_be32(size
);
582 ret
= fdt_setprop_cell(fdt
, node_off
, "ibm,#dma-address-cells", 2);
587 ret
= fdt_setprop_cell(fdt
, node_off
, "ibm,#dma-size-cells", 2);
592 ret
= fdt_setprop(fdt
, node_off
, propname
, dma_prop
, sizeof(dma_prop
));
600 int spapr_tcet_dma_dt(void *fdt
, int node_off
, const char *propname
,
607 return spapr_dma_dt(fdt
, node_off
, propname
,
608 tcet
->liobn
, 0, tcet
->nb_table
<< tcet
->page_shift
);
611 static void spapr_tce_table_class_init(ObjectClass
*klass
, void *data
)
613 DeviceClass
*dc
= DEVICE_CLASS(klass
);
614 dc
->init
= spapr_tce_table_realize
;
615 dc
->reset
= spapr_tce_reset
;
616 dc
->unrealize
= spapr_tce_table_unrealize
;
618 QLIST_INIT(&spapr_tce_tables
);
621 spapr_register_hypercall(H_PUT_TCE
, h_put_tce
);
622 spapr_register_hypercall(H_GET_TCE
, h_get_tce
);
623 spapr_register_hypercall(H_PUT_TCE_INDIRECT
, h_put_tce_indirect
);
624 spapr_register_hypercall(H_STUFF_TCE
, h_stuff_tce
);
627 static TypeInfo spapr_tce_table_info
= {
628 .name
= TYPE_SPAPR_TCE_TABLE
,
629 .parent
= TYPE_DEVICE
,
630 .instance_size
= sizeof(sPAPRTCETable
),
631 .class_init
= spapr_tce_table_class_init
,
634 static void register_types(void)
636 type_register_static(&spapr_tce_table_info
);
639 type_init(register_types
);