2 * IOMMU API for s390 PCI devices
4 * Copyright IBM Corp. 2015
5 * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
9 #include <linux/iommu.h>
10 #include <linux/iommu-helper.h>
11 #include <linux/sizes.h>
12 #include <asm/pci_dma.h>
15 * Physically contiguous memory regions can be mapped with 4 KiB alignment,
16 * we allow all page sizes that are an order of 4KiB (no special large page
19 #define S390_IOMMU_PGSIZES (~0xFFFUL)
21 static const struct iommu_ops s390_iommu_ops
;
24 struct iommu_domain domain
;
25 struct list_head devices
;
26 unsigned long *dma_table
;
27 spinlock_t dma_table_lock
;
31 struct s390_domain_device
{
32 struct list_head list
;
33 struct zpci_dev
*zdev
;
36 static struct s390_domain
*to_s390_domain(struct iommu_domain
*dom
)
38 return container_of(dom
, struct s390_domain
, domain
);
41 static bool s390_iommu_capable(enum iommu_cap cap
)
44 case IOMMU_CAP_CACHE_COHERENCY
:
46 case IOMMU_CAP_INTR_REMAP
:
53 static struct iommu_domain
*s390_domain_alloc(unsigned domain_type
)
55 struct s390_domain
*s390_domain
;
57 if (domain_type
!= IOMMU_DOMAIN_UNMANAGED
)
60 s390_domain
= kzalloc(sizeof(*s390_domain
), GFP_KERNEL
);
64 s390_domain
->dma_table
= dma_alloc_cpu_table();
65 if (!s390_domain
->dma_table
) {
70 spin_lock_init(&s390_domain
->dma_table_lock
);
71 spin_lock_init(&s390_domain
->list_lock
);
72 INIT_LIST_HEAD(&s390_domain
->devices
);
74 return &s390_domain
->domain
;
77 static void s390_domain_free(struct iommu_domain
*domain
)
79 struct s390_domain
*s390_domain
= to_s390_domain(domain
);
81 dma_cleanup_tables(s390_domain
->dma_table
);
85 static int s390_iommu_attach_device(struct iommu_domain
*domain
,
88 struct s390_domain
*s390_domain
= to_s390_domain(domain
);
89 struct zpci_dev
*zdev
= to_pci_dev(dev
)->sysdata
;
90 struct s390_domain_device
*domain_device
;
97 domain_device
= kzalloc(sizeof(*domain_device
), GFP_KERNEL
);
102 zpci_dma_exit_device(zdev
);
104 zdev
->dma_table
= s390_domain
->dma_table
;
105 rc
= zpci_register_ioat(zdev
, 0, zdev
->start_dma
, zdev
->end_dma
,
106 (u64
) zdev
->dma_table
);
110 spin_lock_irqsave(&s390_domain
->list_lock
, flags
);
111 /* First device defines the DMA range limits */
112 if (list_empty(&s390_domain
->devices
)) {
113 domain
->geometry
.aperture_start
= zdev
->start_dma
;
114 domain
->geometry
.aperture_end
= zdev
->end_dma
;
115 domain
->geometry
.force_aperture
= true;
116 /* Allow only devices with identical DMA range limits */
117 } else if (domain
->geometry
.aperture_start
!= zdev
->start_dma
||
118 domain
->geometry
.aperture_end
!= zdev
->end_dma
) {
120 spin_unlock_irqrestore(&s390_domain
->list_lock
, flags
);
123 domain_device
->zdev
= zdev
;
124 zdev
->s390_domain
= s390_domain
;
125 list_add(&domain_device
->list
, &s390_domain
->devices
);
126 spin_unlock_irqrestore(&s390_domain
->list_lock
, flags
);
131 zpci_dma_init_device(zdev
);
132 kfree(domain_device
);
137 static void s390_iommu_detach_device(struct iommu_domain
*domain
,
140 struct s390_domain
*s390_domain
= to_s390_domain(domain
);
141 struct zpci_dev
*zdev
= to_pci_dev(dev
)->sysdata
;
142 struct s390_domain_device
*domain_device
, *tmp
;
149 spin_lock_irqsave(&s390_domain
->list_lock
, flags
);
150 list_for_each_entry_safe(domain_device
, tmp
, &s390_domain
->devices
,
152 if (domain_device
->zdev
== zdev
) {
153 list_del(&domain_device
->list
);
154 kfree(domain_device
);
159 spin_unlock_irqrestore(&s390_domain
->list_lock
, flags
);
162 zdev
->s390_domain
= NULL
;
163 zpci_unregister_ioat(zdev
, 0);
164 zpci_dma_init_device(zdev
);
168 static int s390_iommu_add_device(struct device
*dev
)
170 struct iommu_group
*group
= iommu_group_get_for_dev(dev
);
171 struct zpci_dev
*zdev
= to_pci_dev(dev
)->sysdata
;
174 return PTR_ERR(group
);
176 iommu_group_put(group
);
177 iommu_device_link(&zdev
->iommu_dev
, dev
);
182 static void s390_iommu_remove_device(struct device
*dev
)
184 struct zpci_dev
*zdev
= to_pci_dev(dev
)->sysdata
;
185 struct iommu_domain
*domain
;
188 * This is a workaround for a scenario where the IOMMU API common code
189 * "forgets" to call the detach_dev callback: After binding a device
190 * to vfio-pci and completing the VFIO_SET_IOMMU ioctl (which triggers
191 * the attach_dev), removing the device via
192 * "echo 1 > /sys/bus/pci/devices/.../remove" won't trigger detach_dev,
193 * only remove_device will be called via the BUS_NOTIFY_REMOVED_DEVICE
196 * So let's call detach_dev from here if it hasn't been called before.
198 if (zdev
&& zdev
->s390_domain
) {
199 domain
= iommu_get_domain_for_dev(dev
);
201 s390_iommu_detach_device(domain
, dev
);
204 iommu_device_unlink(&zdev
->iommu_dev
, dev
);
205 iommu_group_remove_device(dev
);
208 static int s390_iommu_update_trans(struct s390_domain
*s390_domain
,
209 unsigned long pa
, dma_addr_t dma_addr
,
210 size_t size
, int flags
)
212 struct s390_domain_device
*domain_device
;
213 u8
*page_addr
= (u8
*) (pa
& PAGE_MASK
);
214 dma_addr_t start_dma_addr
= dma_addr
;
215 unsigned long irq_flags
, nr_pages
, i
;
216 unsigned long *entry
;
219 if (dma_addr
< s390_domain
->domain
.geometry
.aperture_start
||
220 dma_addr
+ size
> s390_domain
->domain
.geometry
.aperture_end
)
223 nr_pages
= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
227 spin_lock_irqsave(&s390_domain
->dma_table_lock
, irq_flags
);
228 for (i
= 0; i
< nr_pages
; i
++) {
229 entry
= dma_walk_cpu_trans(s390_domain
->dma_table
, dma_addr
);
234 dma_update_cpu_trans(entry
, page_addr
, flags
);
235 page_addr
+= PAGE_SIZE
;
236 dma_addr
+= PAGE_SIZE
;
239 spin_lock(&s390_domain
->list_lock
);
240 list_for_each_entry(domain_device
, &s390_domain
->devices
, list
) {
241 rc
= zpci_refresh_trans((u64
) domain_device
->zdev
->fh
<< 32,
242 start_dma_addr
, nr_pages
* PAGE_SIZE
);
246 spin_unlock(&s390_domain
->list_lock
);
249 if (rc
&& ((flags
& ZPCI_PTE_VALID_MASK
) == ZPCI_PTE_VALID
)) {
250 flags
= ZPCI_PTE_INVALID
;
252 page_addr
-= PAGE_SIZE
;
253 dma_addr
-= PAGE_SIZE
;
254 entry
= dma_walk_cpu_trans(s390_domain
->dma_table
,
258 dma_update_cpu_trans(entry
, page_addr
, flags
);
261 spin_unlock_irqrestore(&s390_domain
->dma_table_lock
, irq_flags
);
266 static int s390_iommu_map(struct iommu_domain
*domain
, unsigned long iova
,
267 phys_addr_t paddr
, size_t size
, int prot
)
269 struct s390_domain
*s390_domain
= to_s390_domain(domain
);
270 int flags
= ZPCI_PTE_VALID
, rc
= 0;
272 if (!(prot
& IOMMU_READ
))
275 if (!(prot
& IOMMU_WRITE
))
276 flags
|= ZPCI_TABLE_PROTECTED
;
278 rc
= s390_iommu_update_trans(s390_domain
, (unsigned long) paddr
, iova
,
284 static phys_addr_t
s390_iommu_iova_to_phys(struct iommu_domain
*domain
,
287 struct s390_domain
*s390_domain
= to_s390_domain(domain
);
288 unsigned long *sto
, *pto
, *rto
, flags
;
289 unsigned int rtx
, sx
, px
;
290 phys_addr_t phys
= 0;
292 if (iova
< domain
->geometry
.aperture_start
||
293 iova
> domain
->geometry
.aperture_end
)
296 rtx
= calc_rtx(iova
);
299 rto
= s390_domain
->dma_table
;
301 spin_lock_irqsave(&s390_domain
->dma_table_lock
, flags
);
302 if (rto
&& reg_entry_isvalid(rto
[rtx
])) {
303 sto
= get_rt_sto(rto
[rtx
]);
304 if (sto
&& reg_entry_isvalid(sto
[sx
])) {
305 pto
= get_st_pto(sto
[sx
]);
306 if (pto
&& pt_entry_isvalid(pto
[px
]))
307 phys
= pto
[px
] & ZPCI_PTE_ADDR_MASK
;
310 spin_unlock_irqrestore(&s390_domain
->dma_table_lock
, flags
);
315 static size_t s390_iommu_unmap(struct iommu_domain
*domain
,
316 unsigned long iova
, size_t size
)
318 struct s390_domain
*s390_domain
= to_s390_domain(domain
);
319 int flags
= ZPCI_PTE_INVALID
;
323 paddr
= s390_iommu_iova_to_phys(domain
, iova
);
327 rc
= s390_iommu_update_trans(s390_domain
, (unsigned long) paddr
, iova
,
335 int zpci_init_iommu(struct zpci_dev
*zdev
)
339 rc
= iommu_device_sysfs_add(&zdev
->iommu_dev
, NULL
, NULL
,
340 "s390-iommu.%08x", zdev
->fid
);
344 iommu_device_set_ops(&zdev
->iommu_dev
, &s390_iommu_ops
);
346 rc
= iommu_device_register(&zdev
->iommu_dev
);
353 iommu_device_sysfs_remove(&zdev
->iommu_dev
);
359 void zpci_destroy_iommu(struct zpci_dev
*zdev
)
361 iommu_device_unregister(&zdev
->iommu_dev
);
362 iommu_device_sysfs_remove(&zdev
->iommu_dev
);
365 static const struct iommu_ops s390_iommu_ops
= {
366 .capable
= s390_iommu_capable
,
367 .domain_alloc
= s390_domain_alloc
,
368 .domain_free
= s390_domain_free
,
369 .attach_dev
= s390_iommu_attach_device
,
370 .detach_dev
= s390_iommu_detach_device
,
371 .map
= s390_iommu_map
,
372 .unmap
= s390_iommu_unmap
,
373 .iova_to_phys
= s390_iommu_iova_to_phys
,
374 .add_device
= s390_iommu_add_device
,
375 .remove_device
= s390_iommu_remove_device
,
376 .device_group
= generic_device_group
,
377 .pgsize_bitmap
= S390_IOMMU_PGSIZES
,
380 static int __init
s390_iommu_init(void)
382 return bus_set_iommu(&pci_bus_type
, &s390_iommu_ops
);
384 subsys_initcall(s390_iommu_init
);